4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
56 mlx5_flow_create_eth(const struct rte_flow_item *item,
57 const void *default_mask,
61 mlx5_flow_create_vlan(const struct rte_flow_item *item,
62 const void *default_mask,
66 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
67 const void *default_mask,
71 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
72 const void *default_mask,
76 mlx5_flow_create_udp(const struct rte_flow_item *item,
77 const void *default_mask,
81 mlx5_flow_create_tcp(const struct rte_flow_item *item,
82 const void *default_mask,
86 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
87 const void *default_mask,
91 LIST_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
92 struct ibv_exp_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
93 struct ibv_exp_rwq_ind_table *ind_table; /**< Indirection table. */
94 struct ibv_qp *qp; /**< Verbs queue pair. */
95 struct ibv_exp_flow *ibv_flow; /**< Verbs flow. */
96 struct ibv_exp_wq *wq; /**< Verbs work queue. */
97 struct ibv_cq *cq; /**< Verbs completion queue. */
98 struct rxq *rxq; /**< Pointer to the queue, NULL if drop queue. */
99 uint32_t mark:1; /**< Set if the flow is marked. */
102 /** Static initializer for items. */
104 (const enum rte_flow_item_type []){ \
105 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
108 /** Structure to generate a simple graph of layers supported by the NIC. */
109 struct mlx5_flow_items {
110 /** List of possible actions for these items. */
111 const enum rte_flow_action_type *const actions;
112 /** Bit-masks corresponding to the possibilities for the item. */
115 * Default bit-masks to use when item->mask is not provided. When
116 * \default_mask is also NULL, the full supported bit-mask (\mask) is
119 const void *default_mask;
120 /** Bit-masks size in bytes. */
121 const unsigned int mask_sz;
123 * Conversion function from rte_flow to NIC specific flow.
126 * rte_flow item to convert.
127 * @param default_mask
128 * Default bit-masks to use when item->mask is not provided.
130 * Internal structure to store the conversion.
133 * 0 on success, negative value otherwise.
135 int (*convert)(const struct rte_flow_item *item,
136 const void *default_mask,
138 /** Size in bytes of the destination structure. */
139 const unsigned int dst_sz;
140 /** List of possible following items. */
141 const enum rte_flow_item_type *const items;
144 /** Valid action for this PMD. */
145 static const enum rte_flow_action_type valid_actions[] = {
146 RTE_FLOW_ACTION_TYPE_DROP,
147 RTE_FLOW_ACTION_TYPE_QUEUE,
148 RTE_FLOW_ACTION_TYPE_MARK,
149 RTE_FLOW_ACTION_TYPE_END,
152 /** Graph of supported items and associated actions. */
153 static const struct mlx5_flow_items mlx5_flow_items[] = {
154 [RTE_FLOW_ITEM_TYPE_END] = {
155 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
156 RTE_FLOW_ITEM_TYPE_VXLAN),
158 [RTE_FLOW_ITEM_TYPE_ETH] = {
159 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
160 RTE_FLOW_ITEM_TYPE_IPV4,
161 RTE_FLOW_ITEM_TYPE_IPV6),
162 .actions = valid_actions,
163 .mask = &(const struct rte_flow_item_eth){
164 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
165 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
167 .mask_sz = sizeof(struct rte_flow_item_eth),
168 .convert = mlx5_flow_create_eth,
169 .dst_sz = sizeof(struct ibv_exp_flow_spec_eth),
171 [RTE_FLOW_ITEM_TYPE_VLAN] = {
172 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
173 RTE_FLOW_ITEM_TYPE_IPV6),
174 .actions = valid_actions,
175 .mask = &(const struct rte_flow_item_vlan){
178 .mask_sz = sizeof(struct rte_flow_item_vlan),
179 .convert = mlx5_flow_create_vlan,
182 [RTE_FLOW_ITEM_TYPE_IPV4] = {
183 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
184 RTE_FLOW_ITEM_TYPE_TCP),
185 .actions = valid_actions,
186 .mask = &(const struct rte_flow_item_ipv4){
190 .type_of_service = -1,
194 .default_mask = &(const struct rte_flow_item_ipv4){
200 .mask_sz = sizeof(struct rte_flow_item_ipv4),
201 .convert = mlx5_flow_create_ipv4,
202 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv4_ext),
204 [RTE_FLOW_ITEM_TYPE_IPV6] = {
205 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
206 RTE_FLOW_ITEM_TYPE_TCP),
207 .actions = valid_actions,
208 .mask = &(const struct rte_flow_item_ipv6){
211 0xff, 0xff, 0xff, 0xff,
212 0xff, 0xff, 0xff, 0xff,
213 0xff, 0xff, 0xff, 0xff,
214 0xff, 0xff, 0xff, 0xff,
217 0xff, 0xff, 0xff, 0xff,
218 0xff, 0xff, 0xff, 0xff,
219 0xff, 0xff, 0xff, 0xff,
220 0xff, 0xff, 0xff, 0xff,
224 .mask_sz = sizeof(struct rte_flow_item_ipv6),
225 .convert = mlx5_flow_create_ipv6,
226 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv6),
228 [RTE_FLOW_ITEM_TYPE_UDP] = {
229 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
230 .actions = valid_actions,
231 .mask = &(const struct rte_flow_item_udp){
237 .mask_sz = sizeof(struct rte_flow_item_udp),
238 .convert = mlx5_flow_create_udp,
239 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
241 [RTE_FLOW_ITEM_TYPE_TCP] = {
242 .actions = valid_actions,
243 .mask = &(const struct rte_flow_item_tcp){
249 .mask_sz = sizeof(struct rte_flow_item_tcp),
250 .convert = mlx5_flow_create_tcp,
251 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
253 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
254 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
255 .actions = valid_actions,
256 .mask = &(const struct rte_flow_item_vxlan){
257 .vni = "\xff\xff\xff",
259 .mask_sz = sizeof(struct rte_flow_item_vxlan),
260 .convert = mlx5_flow_create_vxlan,
261 .dst_sz = sizeof(struct ibv_exp_flow_spec_tunnel),
265 /** Structure to pass to the conversion function. */
267 struct ibv_exp_flow_attr *ibv_attr; /**< Verbs attribute. */
268 unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */
269 uint32_t inner; /**< Set once VXLAN is encountered. */
272 struct mlx5_flow_action {
273 uint32_t queue:1; /**< Target is a receive queue. */
274 uint32_t drop:1; /**< Target is a drop queue. */
275 uint32_t mark:1; /**< Mark is present in the flow. */
276 uint32_t queue_id; /**< Identifier of the queue. */
277 uint32_t mark_id; /**< Mark identifier. */
281 * Check support for a given item.
284 * Item specification.
286 * Bit-masks covering supported fields to compare with spec, last and mask in
289 * Bit-Mask size in bytes.
295 mlx5_flow_item_validate(const struct rte_flow_item *item,
296 const uint8_t *mask, unsigned int size)
300 if (!item->spec && (item->mask || item->last))
302 if (item->spec && !item->mask) {
304 const uint8_t *spec = item->spec;
306 for (i = 0; i < size; ++i)
307 if ((spec[i] | mask[i]) != mask[i])
310 if (item->last && !item->mask) {
312 const uint8_t *spec = item->last;
314 for (i = 0; i < size; ++i)
315 if ((spec[i] | mask[i]) != mask[i])
320 const uint8_t *spec = item->mask;
322 for (i = 0; i < size; ++i)
323 if ((spec[i] | mask[i]) != mask[i])
326 if (item->spec && item->last) {
329 const uint8_t *apply = mask;
334 for (i = 0; i < size; ++i) {
335 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
336 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
338 ret = memcmp(spec, last, size);
344 * Validate a flow supported by the NIC.
347 * Pointer to private structure.
349 * Flow rule attributes.
351 * Pattern specification (list terminated by the END pattern item).
353 * Associated actions (list terminated by the END action).
355 * Perform verbose error reporting if not NULL.
356 * @param[in, out] flow
357 * Flow structure to update.
360 * 0 on success, a negative errno value otherwise and rte_errno is set.
363 priv_flow_validate(struct priv *priv,
364 const struct rte_flow_attr *attr,
365 const struct rte_flow_item items[],
366 const struct rte_flow_action actions[],
367 struct rte_flow_error *error,
368 struct mlx5_flow *flow)
370 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
371 struct mlx5_flow_action action = {
379 rte_flow_error_set(error, ENOTSUP,
380 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
382 "groups are not supported");
385 if (attr->priority) {
386 rte_flow_error_set(error, ENOTSUP,
387 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
389 "priorities are not supported");
393 rte_flow_error_set(error, ENOTSUP,
394 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
396 "egress is not supported");
399 if (!attr->ingress) {
400 rte_flow_error_set(error, ENOTSUP,
401 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
403 "only ingress is supported");
406 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
407 const struct mlx5_flow_items *token = NULL;
411 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
415 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
417 if (cur_item->items[i] == items->type) {
418 token = &mlx5_flow_items[items->type];
423 goto exit_item_not_supported;
425 err = mlx5_flow_item_validate(items,
426 (const uint8_t *)cur_item->mask,
427 sizeof(cur_item->mask_sz));
429 goto exit_item_not_supported;
430 if (flow->ibv_attr && cur_item->convert) {
431 err = cur_item->convert(items,
432 (cur_item->default_mask ?
433 cur_item->default_mask :
437 goto exit_item_not_supported;
439 flow->offset += cur_item->dst_sz;
441 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
442 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
444 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
446 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
447 const struct rte_flow_action_queue *queue =
448 (const struct rte_flow_action_queue *)
451 if (!queue || (queue->index > (priv->rxqs_n - 1)))
452 goto exit_action_not_supported;
454 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
455 const struct rte_flow_action_mark *mark =
456 (const struct rte_flow_action_mark *)
459 if (mark && (mark->id >= MLX5_FLOW_MARK_MAX)) {
460 rte_flow_error_set(error, ENOTSUP,
461 RTE_FLOW_ERROR_TYPE_ACTION,
463 "mark must be between 0"
469 goto exit_action_not_supported;
472 if (action.mark && !flow->ibv_attr && !action.drop)
473 flow->offset += sizeof(struct ibv_exp_flow_spec_action_tag);
474 if (!action.queue && !action.drop) {
475 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
476 NULL, "no valid action");
480 exit_item_not_supported:
481 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
482 items, "item not supported");
484 exit_action_not_supported:
485 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
486 actions, "action not supported");
491 * Validate a flow supported by the NIC.
493 * @see rte_flow_validate()
497 mlx5_flow_validate(struct rte_eth_dev *dev,
498 const struct rte_flow_attr *attr,
499 const struct rte_flow_item items[],
500 const struct rte_flow_action actions[],
501 struct rte_flow_error *error)
503 struct priv *priv = dev->data->dev_private;
505 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr) };
508 ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
514 * Convert Ethernet item to Verbs specification.
517 * Item specification.
518 * @param default_mask[in]
519 * Default bit-masks to use when item->mask is not provided.
520 * @param data[in, out]
524 mlx5_flow_create_eth(const struct rte_flow_item *item,
525 const void *default_mask,
528 const struct rte_flow_item_eth *spec = item->spec;
529 const struct rte_flow_item_eth *mask = item->mask;
530 struct mlx5_flow *flow = (struct mlx5_flow *)data;
531 struct ibv_exp_flow_spec_eth *eth;
532 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
535 ++flow->ibv_attr->num_of_specs;
536 flow->ibv_attr->priority = 2;
537 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
538 *eth = (struct ibv_exp_flow_spec_eth) {
539 .type = flow->inner | IBV_EXP_FLOW_SPEC_ETH,
546 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
547 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
548 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
549 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
550 /* Remove unwanted bits from values. */
551 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
552 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
553 eth->val.src_mac[i] &= eth->mask.src_mac[i];
559 * Convert VLAN item to Verbs specification.
562 * Item specification.
563 * @param default_mask[in]
564 * Default bit-masks to use when item->mask is not provided.
565 * @param data[in, out]
569 mlx5_flow_create_vlan(const struct rte_flow_item *item,
570 const void *default_mask,
573 const struct rte_flow_item_vlan *spec = item->spec;
574 const struct rte_flow_item_vlan *mask = item->mask;
575 struct mlx5_flow *flow = (struct mlx5_flow *)data;
576 struct ibv_exp_flow_spec_eth *eth;
577 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
579 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
584 eth->val.vlan_tag = spec->tci;
585 eth->mask.vlan_tag = mask->tci;
586 eth->val.vlan_tag &= eth->mask.vlan_tag;
591 * Convert IPv4 item to Verbs specification.
594 * Item specification.
595 * @param default_mask[in]
596 * Default bit-masks to use when item->mask is not provided.
597 * @param data[in, out]
601 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
602 const void *default_mask,
605 const struct rte_flow_item_ipv4 *spec = item->spec;
606 const struct rte_flow_item_ipv4 *mask = item->mask;
607 struct mlx5_flow *flow = (struct mlx5_flow *)data;
608 struct ibv_exp_flow_spec_ipv4_ext *ipv4;
609 unsigned int ipv4_size = sizeof(struct ibv_exp_flow_spec_ipv4_ext);
611 ++flow->ibv_attr->num_of_specs;
612 flow->ibv_attr->priority = 1;
613 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
614 *ipv4 = (struct ibv_exp_flow_spec_ipv4_ext) {
615 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV4_EXT,
622 ipv4->val = (struct ibv_exp_flow_ipv4_ext_filter){
623 .src_ip = spec->hdr.src_addr,
624 .dst_ip = spec->hdr.dst_addr,
625 .proto = spec->hdr.next_proto_id,
626 .tos = spec->hdr.type_of_service,
628 ipv4->mask = (struct ibv_exp_flow_ipv4_ext_filter){
629 .src_ip = mask->hdr.src_addr,
630 .dst_ip = mask->hdr.dst_addr,
631 .proto = mask->hdr.next_proto_id,
632 .tos = mask->hdr.type_of_service,
634 /* Remove unwanted bits from values. */
635 ipv4->val.src_ip &= ipv4->mask.src_ip;
636 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
637 ipv4->val.proto &= ipv4->mask.proto;
638 ipv4->val.tos &= ipv4->mask.tos;
643 * Convert IPv6 item to Verbs specification.
646 * Item specification.
647 * @param default_mask[in]
648 * Default bit-masks to use when item->mask is not provided.
649 * @param data[in, out]
653 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
654 const void *default_mask,
657 const struct rte_flow_item_ipv6 *spec = item->spec;
658 const struct rte_flow_item_ipv6 *mask = item->mask;
659 struct mlx5_flow *flow = (struct mlx5_flow *)data;
660 struct ibv_exp_flow_spec_ipv6 *ipv6;
661 unsigned int ipv6_size = sizeof(struct ibv_exp_flow_spec_ipv6);
664 ++flow->ibv_attr->num_of_specs;
665 flow->ibv_attr->priority = 1;
666 ipv6 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
667 *ipv6 = (struct ibv_exp_flow_spec_ipv6) {
668 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV6,
675 memcpy(ipv6->val.src_ip, spec->hdr.src_addr,
676 RTE_DIM(ipv6->val.src_ip));
677 memcpy(ipv6->val.dst_ip, spec->hdr.dst_addr,
678 RTE_DIM(ipv6->val.dst_ip));
679 memcpy(ipv6->mask.src_ip, mask->hdr.src_addr,
680 RTE_DIM(ipv6->mask.src_ip));
681 memcpy(ipv6->mask.dst_ip, mask->hdr.dst_addr,
682 RTE_DIM(ipv6->mask.dst_ip));
683 /* Remove unwanted bits from values. */
684 for (i = 0; i < RTE_DIM(ipv6->val.src_ip); ++i) {
685 ipv6->val.src_ip[i] &= ipv6->mask.src_ip[i];
686 ipv6->val.dst_ip[i] &= ipv6->mask.dst_ip[i];
692 * Convert UDP item to Verbs specification.
695 * Item specification.
696 * @param default_mask[in]
697 * Default bit-masks to use when item->mask is not provided.
698 * @param data[in, out]
702 mlx5_flow_create_udp(const struct rte_flow_item *item,
703 const void *default_mask,
706 const struct rte_flow_item_udp *spec = item->spec;
707 const struct rte_flow_item_udp *mask = item->mask;
708 struct mlx5_flow *flow = (struct mlx5_flow *)data;
709 struct ibv_exp_flow_spec_tcp_udp *udp;
710 unsigned int udp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
712 ++flow->ibv_attr->num_of_specs;
713 flow->ibv_attr->priority = 0;
714 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
715 *udp = (struct ibv_exp_flow_spec_tcp_udp) {
716 .type = flow->inner | IBV_EXP_FLOW_SPEC_UDP,
723 udp->val.dst_port = spec->hdr.dst_port;
724 udp->val.src_port = spec->hdr.src_port;
725 udp->mask.dst_port = mask->hdr.dst_port;
726 udp->mask.src_port = mask->hdr.src_port;
727 /* Remove unwanted bits from values. */
728 udp->val.src_port &= udp->mask.src_port;
729 udp->val.dst_port &= udp->mask.dst_port;
734 * Convert TCP item to Verbs specification.
737 * Item specification.
738 * @param default_mask[in]
739 * Default bit-masks to use when item->mask is not provided.
740 * @param data[in, out]
744 mlx5_flow_create_tcp(const struct rte_flow_item *item,
745 const void *default_mask,
748 const struct rte_flow_item_tcp *spec = item->spec;
749 const struct rte_flow_item_tcp *mask = item->mask;
750 struct mlx5_flow *flow = (struct mlx5_flow *)data;
751 struct ibv_exp_flow_spec_tcp_udp *tcp;
752 unsigned int tcp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
754 ++flow->ibv_attr->num_of_specs;
755 flow->ibv_attr->priority = 0;
756 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
757 *tcp = (struct ibv_exp_flow_spec_tcp_udp) {
758 .type = flow->inner | IBV_EXP_FLOW_SPEC_TCP,
765 tcp->val.dst_port = spec->hdr.dst_port;
766 tcp->val.src_port = spec->hdr.src_port;
767 tcp->mask.dst_port = mask->hdr.dst_port;
768 tcp->mask.src_port = mask->hdr.src_port;
769 /* Remove unwanted bits from values. */
770 tcp->val.src_port &= tcp->mask.src_port;
771 tcp->val.dst_port &= tcp->mask.dst_port;
776 * Convert VXLAN item to Verbs specification.
779 * Item specification.
780 * @param default_mask[in]
781 * Default bit-masks to use when item->mask is not provided.
782 * @param data[in, out]
786 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
787 const void *default_mask,
790 const struct rte_flow_item_vxlan *spec = item->spec;
791 const struct rte_flow_item_vxlan *mask = item->mask;
792 struct mlx5_flow *flow = (struct mlx5_flow *)data;
793 struct ibv_exp_flow_spec_tunnel *vxlan;
794 unsigned int size = sizeof(struct ibv_exp_flow_spec_tunnel);
800 ++flow->ibv_attr->num_of_specs;
801 flow->ibv_attr->priority = 0;
803 vxlan = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
804 *vxlan = (struct ibv_exp_flow_spec_tunnel) {
805 .type = flow->inner | IBV_EXP_FLOW_SPEC_VXLAN_TUNNEL,
808 flow->inner = IBV_EXP_FLOW_SPEC_INNER;
813 memcpy(&id.vni[1], spec->vni, 3);
814 vxlan->val.tunnel_id = id.vlan_id;
815 memcpy(&id.vni[1], mask->vni, 3);
816 vxlan->mask.tunnel_id = id.vlan_id;
817 /* Remove unwanted bits from values. */
818 vxlan->val.tunnel_id &= vxlan->mask.tunnel_id;
823 * Convert mark/flag action to Verbs specification.
826 * Pointer to MLX5 flow structure.
831 mlx5_flow_create_flag_mark(struct mlx5_flow *flow, uint32_t mark_id)
833 struct ibv_exp_flow_spec_action_tag *tag;
834 unsigned int size = sizeof(struct ibv_exp_flow_spec_action_tag);
836 tag = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
837 *tag = (struct ibv_exp_flow_spec_action_tag){
838 .type = IBV_EXP_FLOW_SPEC_ACTION_TAG,
840 .tag_id = mlx5_flow_mark_set(mark_id),
842 ++flow->ibv_attr->num_of_specs;
847 * Complete flow rule creation.
850 * Pointer to private structure.
852 * Verbs flow attributes.
854 * Target action structure.
856 * Perform verbose error reporting if not NULL.
859 * A flow if the rule could be created.
861 static struct rte_flow *
862 priv_flow_create_action_queue(struct priv *priv,
863 struct ibv_exp_flow_attr *ibv_attr,
864 struct mlx5_flow_action *action,
865 struct rte_flow_error *error)
867 struct rxq_ctrl *rxq;
868 struct rte_flow *rte_flow;
872 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
874 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
875 NULL, "cannot allocate flow memory");
880 ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
881 &(struct ibv_exp_cq_init_attr){
885 rte_flow_error_set(error, ENOMEM,
886 RTE_FLOW_ERROR_TYPE_HANDLE,
887 NULL, "cannot allocate CQ");
890 rte_flow->wq = ibv_exp_create_wq(priv->ctx,
891 &(struct ibv_exp_wq_init_attr){
892 .wq_type = IBV_EXP_WQT_RQ,
899 rxq = container_of((*priv->rxqs)[action->queue_id],
900 struct rxq_ctrl, rxq);
901 rte_flow->rxq = &rxq->rxq;
902 rxq->rxq.mark |= action->mark;
903 rte_flow->wq = rxq->wq;
905 rte_flow->mark = action->mark;
906 rte_flow->ibv_attr = ibv_attr;
907 rte_flow->ind_table = ibv_exp_create_rwq_ind_table(
909 &(struct ibv_exp_rwq_ind_table_init_attr){
911 .log_ind_tbl_size = 0,
912 .ind_tbl = &rte_flow->wq,
915 if (!rte_flow->ind_table) {
916 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
917 NULL, "cannot allocate indirection table");
920 rte_flow->qp = ibv_exp_create_qp(
922 &(struct ibv_exp_qp_init_attr){
923 .qp_type = IBV_QPT_RAW_PACKET,
925 IBV_EXP_QP_INIT_ATTR_PD |
926 IBV_EXP_QP_INIT_ATTR_PORT |
927 IBV_EXP_QP_INIT_ATTR_RX_HASH,
929 .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
931 IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
932 .rx_hash_key_len = rss_hash_default_key_len,
933 .rx_hash_key = rss_hash_default_key,
934 .rx_hash_fields_mask = 0,
935 .rwq_ind_tbl = rte_flow->ind_table,
937 .port_num = priv->port,
940 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
941 NULL, "cannot allocate QP");
944 rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp,
946 if (!rte_flow->ibv_flow) {
947 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
948 NULL, "flow rule creation failure");
955 ibv_destroy_qp(rte_flow->qp);
956 if (rte_flow->ind_table)
957 ibv_exp_destroy_rwq_ind_table(rte_flow->ind_table);
958 if (!rte_flow->rxq && rte_flow->wq)
959 ibv_exp_destroy_wq(rte_flow->wq);
960 if (!rte_flow->rxq && rte_flow->cq)
961 ibv_destroy_cq(rte_flow->cq);
962 rte_free(rte_flow->ibv_attr);
971 * Pointer to private structure.
973 * Flow rule attributes.
975 * Pattern specification (list terminated by the END pattern item).
977 * Associated actions (list terminated by the END action).
979 * Perform verbose error reporting if not NULL.
982 * A flow on success, NULL otherwise.
984 static struct rte_flow *
985 priv_flow_create(struct priv *priv,
986 const struct rte_flow_attr *attr,
987 const struct rte_flow_item items[],
988 const struct rte_flow_action actions[],
989 struct rte_flow_error *error)
991 struct rte_flow *rte_flow;
992 struct mlx5_flow_action action;
993 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr), };
996 err = priv_flow_validate(priv, attr, items, actions, error, &flow);
999 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
1000 flow.offset = sizeof(struct ibv_exp_flow_attr);
1001 if (!flow.ibv_attr) {
1002 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1003 NULL, "cannot allocate ibv_attr memory");
1006 *flow.ibv_attr = (struct ibv_exp_flow_attr){
1007 .type = IBV_EXP_FLOW_ATTR_NORMAL,
1008 .size = sizeof(struct ibv_exp_flow_attr),
1009 .priority = attr->priority,
1016 claim_zero(priv_flow_validate(priv, attr, items, actions,
1018 action = (struct mlx5_flow_action){
1022 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1024 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1025 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1027 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1030 ((const struct rte_flow_action_queue *)
1031 actions->conf)->index;
1032 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1035 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
1036 const struct rte_flow_action_mark *mark =
1037 (const struct rte_flow_action_mark *)
1041 action.mark_id = mark->id;
1042 action.mark = !action.drop;
1044 rte_flow_error_set(error, ENOTSUP,
1045 RTE_FLOW_ERROR_TYPE_ACTION,
1046 actions, "unsupported action");
1051 mlx5_flow_create_flag_mark(&flow, action.mark_id);
1052 flow.offset += sizeof(struct ibv_exp_flow_spec_action_tag);
1054 rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
1058 rte_free(flow.ibv_attr);
1065 * @see rte_flow_create()
1069 mlx5_flow_create(struct rte_eth_dev *dev,
1070 const struct rte_flow_attr *attr,
1071 const struct rte_flow_item items[],
1072 const struct rte_flow_action actions[],
1073 struct rte_flow_error *error)
1075 struct priv *priv = dev->data->dev_private;
1076 struct rte_flow *flow;
1079 flow = priv_flow_create(priv, attr, items, actions, error);
1081 LIST_INSERT_HEAD(&priv->flows, flow, next);
1082 DEBUG("Flow created %p", (void *)flow);
1092 * Pointer to private structure.
1097 priv_flow_destroy(struct priv *priv,
1098 struct rte_flow *flow)
1101 LIST_REMOVE(flow, next);
1103 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1105 claim_zero(ibv_destroy_qp(flow->qp));
1106 if (flow->ind_table)
1107 claim_zero(ibv_exp_destroy_rwq_ind_table(flow->ind_table));
1108 if (!flow->rxq && flow->wq)
1109 claim_zero(ibv_exp_destroy_wq(flow->wq));
1110 if (!flow->rxq && flow->cq)
1111 claim_zero(ibv_destroy_cq(flow->cq));
1113 struct rte_flow *tmp;
1114 uint32_t mark_n = 0;
1116 for (tmp = LIST_FIRST(&priv->flows);
1118 tmp = LIST_NEXT(tmp, next)) {
1119 if ((flow->rxq == tmp->rxq) && tmp->mark)
1122 flow->rxq->mark = !!mark_n;
1124 rte_free(flow->ibv_attr);
1125 DEBUG("Flow destroyed %p", (void *)flow);
1132 * @see rte_flow_destroy()
1136 mlx5_flow_destroy(struct rte_eth_dev *dev,
1137 struct rte_flow *flow,
1138 struct rte_flow_error *error)
1140 struct priv *priv = dev->data->dev_private;
1144 priv_flow_destroy(priv, flow);
1150 * Destroy all flows.
1153 * Pointer to private structure.
1156 priv_flow_flush(struct priv *priv)
1158 while (!LIST_EMPTY(&priv->flows)) {
1159 struct rte_flow *flow;
1161 flow = LIST_FIRST(&priv->flows);
1162 priv_flow_destroy(priv, flow);
1167 * Destroy all flows.
1169 * @see rte_flow_flush()
1173 mlx5_flow_flush(struct rte_eth_dev *dev,
1174 struct rte_flow_error *error)
1176 struct priv *priv = dev->data->dev_private;
1180 priv_flow_flush(priv);
1188 * Called by dev_stop() to remove all flows.
1191 * Pointer to private structure.
1194 priv_flow_stop(struct priv *priv)
1196 struct rte_flow *flow;
1198 for (flow = LIST_FIRST(&priv->flows);
1200 flow = LIST_NEXT(flow, next)) {
1201 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1202 flow->ibv_flow = NULL;
1204 flow->rxq->mark = 0;
1205 DEBUG("Flow %p removed", (void *)flow);
1213 * Pointer to private structure.
1216 * 0 on success, a errno value otherwise and rte_errno is set.
1219 priv_flow_start(struct priv *priv)
1221 struct rte_flow *flow;
1223 for (flow = LIST_FIRST(&priv->flows);
1225 flow = LIST_NEXT(flow, next)) {
1226 flow->ibv_flow = ibv_exp_create_flow(flow->qp,
1228 if (!flow->ibv_flow) {
1229 DEBUG("Flow %p cannot be applied", (void *)flow);
1233 DEBUG("Flow %p applied", (void *)flow);
1235 flow->rxq->mark |= flow->mark;