4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
56 mlx5_flow_create_eth(const struct rte_flow_item *item,
57 const void *default_mask,
61 mlx5_flow_create_vlan(const struct rte_flow_item *item,
62 const void *default_mask,
66 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
67 const void *default_mask,
71 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
72 const void *default_mask,
76 mlx5_flow_create_udp(const struct rte_flow_item *item,
77 const void *default_mask,
81 mlx5_flow_create_tcp(const struct rte_flow_item *item,
82 const void *default_mask,
86 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
87 const void *default_mask,
91 LIST_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
92 struct ibv_exp_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
93 struct ibv_exp_rwq_ind_table *ind_table; /**< Indirection table. */
94 struct ibv_qp *qp; /**< Verbs queue pair. */
95 struct ibv_exp_flow *ibv_flow; /**< Verbs flow. */
96 struct ibv_exp_wq *wq; /**< Verbs work queue. */
97 struct ibv_cq *cq; /**< Verbs completion queue. */
98 struct rxq *rxq; /**< Pointer to the queue, NULL if drop queue. */
99 uint32_t mark:1; /**< Set if the flow is marked. */
102 /** Static initializer for items. */
104 (const enum rte_flow_item_type []){ \
105 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
108 /** Structure to generate a simple graph of layers supported by the NIC. */
109 struct mlx5_flow_items {
110 /** List of possible actions for these items. */
111 const enum rte_flow_action_type *const actions;
112 /** Bit-masks corresponding to the possibilities for the item. */
115 * Default bit-masks to use when item->mask is not provided. When
116 * \default_mask is also NULL, the full supported bit-mask (\mask) is
119 const void *default_mask;
120 /** Bit-masks size in bytes. */
121 const unsigned int mask_sz;
123 * Conversion function from rte_flow to NIC specific flow.
126 * rte_flow item to convert.
127 * @param default_mask
128 * Default bit-masks to use when item->mask is not provided.
130 * Internal structure to store the conversion.
133 * 0 on success, negative value otherwise.
135 int (*convert)(const struct rte_flow_item *item,
136 const void *default_mask,
138 /** Size in bytes of the destination structure. */
139 const unsigned int dst_sz;
140 /** List of possible following items. */
141 const enum rte_flow_item_type *const items;
144 /** Valid action for this PMD. */
145 static const enum rte_flow_action_type valid_actions[] = {
146 RTE_FLOW_ACTION_TYPE_DROP,
147 RTE_FLOW_ACTION_TYPE_QUEUE,
148 RTE_FLOW_ACTION_TYPE_MARK,
149 RTE_FLOW_ACTION_TYPE_END,
152 /** Graph of supported items and associated actions. */
153 static const struct mlx5_flow_items mlx5_flow_items[] = {
154 [RTE_FLOW_ITEM_TYPE_END] = {
155 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
156 RTE_FLOW_ITEM_TYPE_VXLAN),
158 [RTE_FLOW_ITEM_TYPE_ETH] = {
159 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
160 RTE_FLOW_ITEM_TYPE_IPV4,
161 RTE_FLOW_ITEM_TYPE_IPV6),
162 .actions = valid_actions,
163 .mask = &(const struct rte_flow_item_eth){
164 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
165 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
167 .default_mask = &rte_flow_item_eth_mask,
168 .mask_sz = sizeof(struct rte_flow_item_eth),
169 .convert = mlx5_flow_create_eth,
170 .dst_sz = sizeof(struct ibv_exp_flow_spec_eth),
172 [RTE_FLOW_ITEM_TYPE_VLAN] = {
173 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
174 RTE_FLOW_ITEM_TYPE_IPV6),
175 .actions = valid_actions,
176 .mask = &(const struct rte_flow_item_vlan){
179 .default_mask = &rte_flow_item_vlan_mask,
180 .mask_sz = sizeof(struct rte_flow_item_vlan),
181 .convert = mlx5_flow_create_vlan,
184 [RTE_FLOW_ITEM_TYPE_IPV4] = {
185 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
186 RTE_FLOW_ITEM_TYPE_TCP),
187 .actions = valid_actions,
188 .mask = &(const struct rte_flow_item_ipv4){
192 .type_of_service = -1,
196 .default_mask = &rte_flow_item_ipv4_mask,
197 .mask_sz = sizeof(struct rte_flow_item_ipv4),
198 .convert = mlx5_flow_create_ipv4,
199 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv4_ext),
201 [RTE_FLOW_ITEM_TYPE_IPV6] = {
202 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
203 RTE_FLOW_ITEM_TYPE_TCP),
204 .actions = valid_actions,
205 .mask = &(const struct rte_flow_item_ipv6){
208 0xff, 0xff, 0xff, 0xff,
209 0xff, 0xff, 0xff, 0xff,
210 0xff, 0xff, 0xff, 0xff,
211 0xff, 0xff, 0xff, 0xff,
214 0xff, 0xff, 0xff, 0xff,
215 0xff, 0xff, 0xff, 0xff,
216 0xff, 0xff, 0xff, 0xff,
217 0xff, 0xff, 0xff, 0xff,
221 .default_mask = &rte_flow_item_ipv6_mask,
222 .mask_sz = sizeof(struct rte_flow_item_ipv6),
223 .convert = mlx5_flow_create_ipv6,
224 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv6),
226 [RTE_FLOW_ITEM_TYPE_UDP] = {
227 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
228 .actions = valid_actions,
229 .mask = &(const struct rte_flow_item_udp){
235 .default_mask = &rte_flow_item_udp_mask,
236 .mask_sz = sizeof(struct rte_flow_item_udp),
237 .convert = mlx5_flow_create_udp,
238 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
240 [RTE_FLOW_ITEM_TYPE_TCP] = {
241 .actions = valid_actions,
242 .mask = &(const struct rte_flow_item_tcp){
248 .default_mask = &rte_flow_item_tcp_mask,
249 .mask_sz = sizeof(struct rte_flow_item_tcp),
250 .convert = mlx5_flow_create_tcp,
251 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
253 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
254 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
255 .actions = valid_actions,
256 .mask = &(const struct rte_flow_item_vxlan){
257 .vni = "\xff\xff\xff",
259 .default_mask = &rte_flow_item_vxlan_mask,
260 .mask_sz = sizeof(struct rte_flow_item_vxlan),
261 .convert = mlx5_flow_create_vxlan,
262 .dst_sz = sizeof(struct ibv_exp_flow_spec_tunnel),
266 /** Structure to pass to the conversion function. */
268 struct ibv_exp_flow_attr *ibv_attr; /**< Verbs attribute. */
269 unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */
270 uint32_t inner; /**< Set once VXLAN is encountered. */
273 struct mlx5_flow_action {
274 uint32_t queue:1; /**< Target is a receive queue. */
275 uint32_t drop:1; /**< Target is a drop queue. */
276 uint32_t mark:1; /**< Mark is present in the flow. */
277 uint32_t queue_id; /**< Identifier of the queue. */
278 uint32_t mark_id; /**< Mark identifier. */
282 * Check support for a given item.
285 * Item specification.
287 * Bit-masks covering supported fields to compare with spec, last and mask in
290 * Bit-Mask size in bytes.
296 mlx5_flow_item_validate(const struct rte_flow_item *item,
297 const uint8_t *mask, unsigned int size)
301 if (!item->spec && (item->mask || item->last))
303 if (item->spec && !item->mask) {
305 const uint8_t *spec = item->spec;
307 for (i = 0; i < size; ++i)
308 if ((spec[i] | mask[i]) != mask[i])
311 if (item->last && !item->mask) {
313 const uint8_t *spec = item->last;
315 for (i = 0; i < size; ++i)
316 if ((spec[i] | mask[i]) != mask[i])
321 const uint8_t *spec = item->mask;
323 for (i = 0; i < size; ++i)
324 if ((spec[i] | mask[i]) != mask[i])
327 if (item->spec && item->last) {
330 const uint8_t *apply = mask;
335 for (i = 0; i < size; ++i) {
336 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
337 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
339 ret = memcmp(spec, last, size);
345 * Validate a flow supported by the NIC.
348 * Pointer to private structure.
350 * Flow rule attributes.
352 * Pattern specification (list terminated by the END pattern item).
354 * Associated actions (list terminated by the END action).
356 * Perform verbose error reporting if not NULL.
357 * @param[in, out] flow
358 * Flow structure to update.
361 * 0 on success, a negative errno value otherwise and rte_errno is set.
364 priv_flow_validate(struct priv *priv,
365 const struct rte_flow_attr *attr,
366 const struct rte_flow_item items[],
367 const struct rte_flow_action actions[],
368 struct rte_flow_error *error,
369 struct mlx5_flow *flow)
371 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
372 struct mlx5_flow_action action = {
380 rte_flow_error_set(error, ENOTSUP,
381 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
383 "groups are not supported");
386 if (attr->priority) {
387 rte_flow_error_set(error, ENOTSUP,
388 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
390 "priorities are not supported");
394 rte_flow_error_set(error, ENOTSUP,
395 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
397 "egress is not supported");
400 if (!attr->ingress) {
401 rte_flow_error_set(error, ENOTSUP,
402 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
404 "only ingress is supported");
407 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
408 const struct mlx5_flow_items *token = NULL;
412 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
416 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
418 if (cur_item->items[i] == items->type) {
419 token = &mlx5_flow_items[items->type];
424 goto exit_item_not_supported;
426 err = mlx5_flow_item_validate(items,
427 (const uint8_t *)cur_item->mask,
430 goto exit_item_not_supported;
431 if (flow->ibv_attr && cur_item->convert) {
432 err = cur_item->convert(items,
433 (cur_item->default_mask ?
434 cur_item->default_mask :
438 goto exit_item_not_supported;
440 flow->offset += cur_item->dst_sz;
442 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
443 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
445 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
447 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
448 const struct rte_flow_action_queue *queue =
449 (const struct rte_flow_action_queue *)
452 if (!queue || (queue->index > (priv->rxqs_n - 1)))
453 goto exit_action_not_supported;
455 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
456 const struct rte_flow_action_mark *mark =
457 (const struct rte_flow_action_mark *)
461 rte_flow_error_set(error, EINVAL,
462 RTE_FLOW_ERROR_TYPE_ACTION,
464 "mark must be defined");
466 } else if (mark->id >= MLX5_FLOW_MARK_MAX) {
467 rte_flow_error_set(error, ENOTSUP,
468 RTE_FLOW_ERROR_TYPE_ACTION,
470 "mark must be between 0"
476 goto exit_action_not_supported;
479 if (action.mark && !flow->ibv_attr && !action.drop)
480 flow->offset += sizeof(struct ibv_exp_flow_spec_action_tag);
481 if (!action.queue && !action.drop) {
482 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
483 NULL, "no valid action");
487 exit_item_not_supported:
488 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
489 items, "item not supported");
491 exit_action_not_supported:
492 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
493 actions, "action not supported");
498 * Validate a flow supported by the NIC.
500 * @see rte_flow_validate()
504 mlx5_flow_validate(struct rte_eth_dev *dev,
505 const struct rte_flow_attr *attr,
506 const struct rte_flow_item items[],
507 const struct rte_flow_action actions[],
508 struct rte_flow_error *error)
510 struct priv *priv = dev->data->dev_private;
512 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr) };
515 ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
521 * Convert Ethernet item to Verbs specification.
524 * Item specification.
525 * @param default_mask[in]
526 * Default bit-masks to use when item->mask is not provided.
527 * @param data[in, out]
531 mlx5_flow_create_eth(const struct rte_flow_item *item,
532 const void *default_mask,
535 const struct rte_flow_item_eth *spec = item->spec;
536 const struct rte_flow_item_eth *mask = item->mask;
537 struct mlx5_flow *flow = (struct mlx5_flow *)data;
538 struct ibv_exp_flow_spec_eth *eth;
539 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
542 ++flow->ibv_attr->num_of_specs;
543 flow->ibv_attr->priority = 2;
544 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
545 *eth = (struct ibv_exp_flow_spec_eth) {
546 .type = flow->inner | IBV_EXP_FLOW_SPEC_ETH,
553 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
554 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
555 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
556 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
557 /* Remove unwanted bits from values. */
558 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
559 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
560 eth->val.src_mac[i] &= eth->mask.src_mac[i];
566 * Convert VLAN item to Verbs specification.
569 * Item specification.
570 * @param default_mask[in]
571 * Default bit-masks to use when item->mask is not provided.
572 * @param data[in, out]
576 mlx5_flow_create_vlan(const struct rte_flow_item *item,
577 const void *default_mask,
580 const struct rte_flow_item_vlan *spec = item->spec;
581 const struct rte_flow_item_vlan *mask = item->mask;
582 struct mlx5_flow *flow = (struct mlx5_flow *)data;
583 struct ibv_exp_flow_spec_eth *eth;
584 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
586 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
591 eth->val.vlan_tag = spec->tci;
592 eth->mask.vlan_tag = mask->tci;
593 eth->val.vlan_tag &= eth->mask.vlan_tag;
598 * Convert IPv4 item to Verbs specification.
601 * Item specification.
602 * @param default_mask[in]
603 * Default bit-masks to use when item->mask is not provided.
604 * @param data[in, out]
608 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
609 const void *default_mask,
612 const struct rte_flow_item_ipv4 *spec = item->spec;
613 const struct rte_flow_item_ipv4 *mask = item->mask;
614 struct mlx5_flow *flow = (struct mlx5_flow *)data;
615 struct ibv_exp_flow_spec_ipv4_ext *ipv4;
616 unsigned int ipv4_size = sizeof(struct ibv_exp_flow_spec_ipv4_ext);
618 ++flow->ibv_attr->num_of_specs;
619 flow->ibv_attr->priority = 1;
620 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
621 *ipv4 = (struct ibv_exp_flow_spec_ipv4_ext) {
622 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV4_EXT,
629 ipv4->val = (struct ibv_exp_flow_ipv4_ext_filter){
630 .src_ip = spec->hdr.src_addr,
631 .dst_ip = spec->hdr.dst_addr,
632 .proto = spec->hdr.next_proto_id,
633 .tos = spec->hdr.type_of_service,
635 ipv4->mask = (struct ibv_exp_flow_ipv4_ext_filter){
636 .src_ip = mask->hdr.src_addr,
637 .dst_ip = mask->hdr.dst_addr,
638 .proto = mask->hdr.next_proto_id,
639 .tos = mask->hdr.type_of_service,
641 /* Remove unwanted bits from values. */
642 ipv4->val.src_ip &= ipv4->mask.src_ip;
643 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
644 ipv4->val.proto &= ipv4->mask.proto;
645 ipv4->val.tos &= ipv4->mask.tos;
650 * Convert IPv6 item to Verbs specification.
653 * Item specification.
654 * @param default_mask[in]
655 * Default bit-masks to use when item->mask is not provided.
656 * @param data[in, out]
660 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
661 const void *default_mask,
664 const struct rte_flow_item_ipv6 *spec = item->spec;
665 const struct rte_flow_item_ipv6 *mask = item->mask;
666 struct mlx5_flow *flow = (struct mlx5_flow *)data;
667 struct ibv_exp_flow_spec_ipv6 *ipv6;
668 unsigned int ipv6_size = sizeof(struct ibv_exp_flow_spec_ipv6);
671 ++flow->ibv_attr->num_of_specs;
672 flow->ibv_attr->priority = 1;
673 ipv6 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
674 *ipv6 = (struct ibv_exp_flow_spec_ipv6) {
675 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV6,
682 memcpy(ipv6->val.src_ip, spec->hdr.src_addr,
683 RTE_DIM(ipv6->val.src_ip));
684 memcpy(ipv6->val.dst_ip, spec->hdr.dst_addr,
685 RTE_DIM(ipv6->val.dst_ip));
686 memcpy(ipv6->mask.src_ip, mask->hdr.src_addr,
687 RTE_DIM(ipv6->mask.src_ip));
688 memcpy(ipv6->mask.dst_ip, mask->hdr.dst_addr,
689 RTE_DIM(ipv6->mask.dst_ip));
690 /* Remove unwanted bits from values. */
691 for (i = 0; i < RTE_DIM(ipv6->val.src_ip); ++i) {
692 ipv6->val.src_ip[i] &= ipv6->mask.src_ip[i];
693 ipv6->val.dst_ip[i] &= ipv6->mask.dst_ip[i];
699 * Convert UDP item to Verbs specification.
702 * Item specification.
703 * @param default_mask[in]
704 * Default bit-masks to use when item->mask is not provided.
705 * @param data[in, out]
709 mlx5_flow_create_udp(const struct rte_flow_item *item,
710 const void *default_mask,
713 const struct rte_flow_item_udp *spec = item->spec;
714 const struct rte_flow_item_udp *mask = item->mask;
715 struct mlx5_flow *flow = (struct mlx5_flow *)data;
716 struct ibv_exp_flow_spec_tcp_udp *udp;
717 unsigned int udp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
719 ++flow->ibv_attr->num_of_specs;
720 flow->ibv_attr->priority = 0;
721 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
722 *udp = (struct ibv_exp_flow_spec_tcp_udp) {
723 .type = flow->inner | IBV_EXP_FLOW_SPEC_UDP,
730 udp->val.dst_port = spec->hdr.dst_port;
731 udp->val.src_port = spec->hdr.src_port;
732 udp->mask.dst_port = mask->hdr.dst_port;
733 udp->mask.src_port = mask->hdr.src_port;
734 /* Remove unwanted bits from values. */
735 udp->val.src_port &= udp->mask.src_port;
736 udp->val.dst_port &= udp->mask.dst_port;
741 * Convert TCP item to Verbs specification.
744 * Item specification.
745 * @param default_mask[in]
746 * Default bit-masks to use when item->mask is not provided.
747 * @param data[in, out]
751 mlx5_flow_create_tcp(const struct rte_flow_item *item,
752 const void *default_mask,
755 const struct rte_flow_item_tcp *spec = item->spec;
756 const struct rte_flow_item_tcp *mask = item->mask;
757 struct mlx5_flow *flow = (struct mlx5_flow *)data;
758 struct ibv_exp_flow_spec_tcp_udp *tcp;
759 unsigned int tcp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
761 ++flow->ibv_attr->num_of_specs;
762 flow->ibv_attr->priority = 0;
763 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
764 *tcp = (struct ibv_exp_flow_spec_tcp_udp) {
765 .type = flow->inner | IBV_EXP_FLOW_SPEC_TCP,
772 tcp->val.dst_port = spec->hdr.dst_port;
773 tcp->val.src_port = spec->hdr.src_port;
774 tcp->mask.dst_port = mask->hdr.dst_port;
775 tcp->mask.src_port = mask->hdr.src_port;
776 /* Remove unwanted bits from values. */
777 tcp->val.src_port &= tcp->mask.src_port;
778 tcp->val.dst_port &= tcp->mask.dst_port;
783 * Convert VXLAN item to Verbs specification.
786 * Item specification.
787 * @param default_mask[in]
788 * Default bit-masks to use when item->mask is not provided.
789 * @param data[in, out]
793 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
794 const void *default_mask,
797 const struct rte_flow_item_vxlan *spec = item->spec;
798 const struct rte_flow_item_vxlan *mask = item->mask;
799 struct mlx5_flow *flow = (struct mlx5_flow *)data;
800 struct ibv_exp_flow_spec_tunnel *vxlan;
801 unsigned int size = sizeof(struct ibv_exp_flow_spec_tunnel);
807 ++flow->ibv_attr->num_of_specs;
808 flow->ibv_attr->priority = 0;
810 vxlan = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
811 *vxlan = (struct ibv_exp_flow_spec_tunnel) {
812 .type = flow->inner | IBV_EXP_FLOW_SPEC_VXLAN_TUNNEL,
815 flow->inner = IBV_EXP_FLOW_SPEC_INNER;
820 memcpy(&id.vni[1], spec->vni, 3);
821 vxlan->val.tunnel_id = id.vlan_id;
822 memcpy(&id.vni[1], mask->vni, 3);
823 vxlan->mask.tunnel_id = id.vlan_id;
824 /* Remove unwanted bits from values. */
825 vxlan->val.tunnel_id &= vxlan->mask.tunnel_id;
830 * Convert mark/flag action to Verbs specification.
833 * Pointer to MLX5 flow structure.
838 mlx5_flow_create_flag_mark(struct mlx5_flow *flow, uint32_t mark_id)
840 struct ibv_exp_flow_spec_action_tag *tag;
841 unsigned int size = sizeof(struct ibv_exp_flow_spec_action_tag);
843 tag = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
844 *tag = (struct ibv_exp_flow_spec_action_tag){
845 .type = IBV_EXP_FLOW_SPEC_ACTION_TAG,
847 .tag_id = mlx5_flow_mark_set(mark_id),
849 ++flow->ibv_attr->num_of_specs;
854 * Complete flow rule creation.
857 * Pointer to private structure.
859 * Verbs flow attributes.
861 * Target action structure.
863 * Perform verbose error reporting if not NULL.
866 * A flow if the rule could be created.
868 static struct rte_flow *
869 priv_flow_create_action_queue(struct priv *priv,
870 struct ibv_exp_flow_attr *ibv_attr,
871 struct mlx5_flow_action *action,
872 struct rte_flow_error *error)
874 struct rxq_ctrl *rxq;
875 struct rte_flow *rte_flow;
879 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
881 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
882 NULL, "cannot allocate flow memory");
887 ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
888 &(struct ibv_exp_cq_init_attr){
892 rte_flow_error_set(error, ENOMEM,
893 RTE_FLOW_ERROR_TYPE_HANDLE,
894 NULL, "cannot allocate CQ");
897 rte_flow->wq = ibv_exp_create_wq(priv->ctx,
898 &(struct ibv_exp_wq_init_attr){
899 .wq_type = IBV_EXP_WQT_RQ,
906 rxq = container_of((*priv->rxqs)[action->queue_id],
907 struct rxq_ctrl, rxq);
908 rte_flow->rxq = &rxq->rxq;
909 rxq->rxq.mark |= action->mark;
910 rte_flow->wq = rxq->wq;
912 rte_flow->mark = action->mark;
913 rte_flow->ibv_attr = ibv_attr;
914 rte_flow->ind_table = ibv_exp_create_rwq_ind_table(
916 &(struct ibv_exp_rwq_ind_table_init_attr){
918 .log_ind_tbl_size = 0,
919 .ind_tbl = &rte_flow->wq,
922 if (!rte_flow->ind_table) {
923 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
924 NULL, "cannot allocate indirection table");
927 rte_flow->qp = ibv_exp_create_qp(
929 &(struct ibv_exp_qp_init_attr){
930 .qp_type = IBV_QPT_RAW_PACKET,
932 IBV_EXP_QP_INIT_ATTR_PD |
933 IBV_EXP_QP_INIT_ATTR_PORT |
934 IBV_EXP_QP_INIT_ATTR_RX_HASH,
936 .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
938 IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
939 .rx_hash_key_len = rss_hash_default_key_len,
940 .rx_hash_key = rss_hash_default_key,
941 .rx_hash_fields_mask = 0,
942 .rwq_ind_tbl = rte_flow->ind_table,
944 .port_num = priv->port,
947 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
948 NULL, "cannot allocate QP");
953 rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp,
955 if (!rte_flow->ibv_flow) {
956 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
957 NULL, "flow rule creation failure");
964 ibv_destroy_qp(rte_flow->qp);
965 if (rte_flow->ind_table)
966 ibv_exp_destroy_rwq_ind_table(rte_flow->ind_table);
967 if (!rte_flow->rxq && rte_flow->wq)
968 ibv_exp_destroy_wq(rte_flow->wq);
969 if (!rte_flow->rxq && rte_flow->cq)
970 ibv_destroy_cq(rte_flow->cq);
971 rte_free(rte_flow->ibv_attr);
980 * Pointer to private structure.
982 * Flow rule attributes.
984 * Pattern specification (list terminated by the END pattern item).
986 * Associated actions (list terminated by the END action).
988 * Perform verbose error reporting if not NULL.
991 * A flow on success, NULL otherwise.
993 static struct rte_flow *
994 priv_flow_create(struct priv *priv,
995 const struct rte_flow_attr *attr,
996 const struct rte_flow_item items[],
997 const struct rte_flow_action actions[],
998 struct rte_flow_error *error)
1000 struct rte_flow *rte_flow;
1001 struct mlx5_flow_action action;
1002 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr), };
1005 err = priv_flow_validate(priv, attr, items, actions, error, &flow);
1008 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
1009 flow.offset = sizeof(struct ibv_exp_flow_attr);
1010 if (!flow.ibv_attr) {
1011 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1012 NULL, "cannot allocate ibv_attr memory");
1015 *flow.ibv_attr = (struct ibv_exp_flow_attr){
1016 .type = IBV_EXP_FLOW_ATTR_NORMAL,
1017 .size = sizeof(struct ibv_exp_flow_attr),
1018 .priority = attr->priority,
1025 claim_zero(priv_flow_validate(priv, attr, items, actions,
1027 action = (struct mlx5_flow_action){
1031 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1033 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1034 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1036 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1039 ((const struct rte_flow_action_queue *)
1040 actions->conf)->index;
1041 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1044 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
1045 const struct rte_flow_action_mark *mark =
1046 (const struct rte_flow_action_mark *)
1050 action.mark_id = mark->id;
1051 action.mark = !action.drop;
1053 rte_flow_error_set(error, ENOTSUP,
1054 RTE_FLOW_ERROR_TYPE_ACTION,
1055 actions, "unsupported action");
1060 mlx5_flow_create_flag_mark(&flow, action.mark_id);
1061 flow.offset += sizeof(struct ibv_exp_flow_spec_action_tag);
1063 rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
1067 rte_free(flow.ibv_attr);
1074 * @see rte_flow_create()
1078 mlx5_flow_create(struct rte_eth_dev *dev,
1079 const struct rte_flow_attr *attr,
1080 const struct rte_flow_item items[],
1081 const struct rte_flow_action actions[],
1082 struct rte_flow_error *error)
1084 struct priv *priv = dev->data->dev_private;
1085 struct rte_flow *flow;
1088 flow = priv_flow_create(priv, attr, items, actions, error);
1090 LIST_INSERT_HEAD(&priv->flows, flow, next);
1091 DEBUG("Flow created %p", (void *)flow);
1101 * Pointer to private structure.
1106 priv_flow_destroy(struct priv *priv,
1107 struct rte_flow *flow)
1110 LIST_REMOVE(flow, next);
1112 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1114 claim_zero(ibv_destroy_qp(flow->qp));
1115 if (flow->ind_table)
1116 claim_zero(ibv_exp_destroy_rwq_ind_table(flow->ind_table));
1117 if (!flow->rxq && flow->wq)
1118 claim_zero(ibv_exp_destroy_wq(flow->wq));
1119 if (!flow->rxq && flow->cq)
1120 claim_zero(ibv_destroy_cq(flow->cq));
1122 struct rte_flow *tmp;
1123 uint32_t mark_n = 0;
1125 for (tmp = LIST_FIRST(&priv->flows);
1127 tmp = LIST_NEXT(tmp, next)) {
1128 if ((flow->rxq == tmp->rxq) && tmp->mark)
1131 flow->rxq->mark = !!mark_n;
1133 rte_free(flow->ibv_attr);
1134 DEBUG("Flow destroyed %p", (void *)flow);
1141 * @see rte_flow_destroy()
1145 mlx5_flow_destroy(struct rte_eth_dev *dev,
1146 struct rte_flow *flow,
1147 struct rte_flow_error *error)
1149 struct priv *priv = dev->data->dev_private;
1153 priv_flow_destroy(priv, flow);
1159 * Destroy all flows.
1162 * Pointer to private structure.
1165 priv_flow_flush(struct priv *priv)
1167 while (!LIST_EMPTY(&priv->flows)) {
1168 struct rte_flow *flow;
1170 flow = LIST_FIRST(&priv->flows);
1171 priv_flow_destroy(priv, flow);
1176 * Destroy all flows.
1178 * @see rte_flow_flush()
1182 mlx5_flow_flush(struct rte_eth_dev *dev,
1183 struct rte_flow_error *error)
1185 struct priv *priv = dev->data->dev_private;
1189 priv_flow_flush(priv);
1197 * Called by dev_stop() to remove all flows.
1200 * Pointer to private structure.
1203 priv_flow_stop(struct priv *priv)
1205 struct rte_flow *flow;
1207 for (flow = LIST_FIRST(&priv->flows);
1209 flow = LIST_NEXT(flow, next)) {
1210 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1211 flow->ibv_flow = NULL;
1213 flow->rxq->mark = 0;
1214 DEBUG("Flow %p removed", (void *)flow);
1222 * Pointer to private structure.
1225 * 0 on success, a errno value otherwise and rte_errno is set.
1228 priv_flow_start(struct priv *priv)
1230 struct rte_flow *flow;
1232 for (flow = LIST_FIRST(&priv->flows);
1234 flow = LIST_NEXT(flow, next)) {
1235 flow->ibv_flow = ibv_exp_create_flow(flow->qp,
1237 if (!flow->ibv_flow) {
1238 DEBUG("Flow %p cannot be applied", (void *)flow);
1242 DEBUG("Flow %p applied", (void *)flow);
1244 flow->rxq->mark |= flow->mark;