4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 mlx5_flow_create_eth(const struct rte_flow_item *item,
56 const void *default_mask,
60 mlx5_flow_create_vlan(const struct rte_flow_item *item,
61 const void *default_mask,
65 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
66 const void *default_mask,
70 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
71 const void *default_mask,
75 mlx5_flow_create_udp(const struct rte_flow_item *item,
76 const void *default_mask,
80 mlx5_flow_create_tcp(const struct rte_flow_item *item,
81 const void *default_mask,
85 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
86 const void *default_mask,
90 LIST_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
91 struct ibv_exp_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
92 struct ibv_exp_rwq_ind_table *ind_table; /**< Indirection table. */
93 struct ibv_qp *qp; /**< Verbs queue pair. */
94 struct ibv_exp_flow *ibv_flow; /**< Verbs flow. */
95 struct ibv_exp_wq *wq; /**< Verbs work queue. */
96 struct ibv_cq *cq; /**< Verbs completion queue. */
97 struct rxq *rxq; /**< Pointer to the queue, NULL if drop queue. */
100 /** Static initializer for items. */
102 (const enum rte_flow_item_type []){ \
103 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
106 /** Structure to generate a simple graph of layers supported by the NIC. */
107 struct mlx5_flow_items {
108 /** List of possible actions for these items. */
109 const enum rte_flow_action_type *const actions;
110 /** Bit-masks corresponding to the possibilities for the item. */
112 /** Bit-masks size in bytes. */
113 const unsigned int mask_sz;
115 * Conversion function from rte_flow to NIC specific flow.
118 * rte_flow item to convert.
119 * @param default_mask
120 * Default bit-masks to use when item->mask is not provided.
122 * Internal structure to store the conversion.
125 * 0 on success, negative value otherwise.
127 int (*convert)(const struct rte_flow_item *item,
128 const void *default_mask,
130 /** Size in bytes of the destination structure. */
131 const unsigned int dst_sz;
132 /** List of possible following items. */
133 const enum rte_flow_item_type *const items;
136 /** Valid action for this PMD. */
137 static const enum rte_flow_action_type valid_actions[] = {
138 RTE_FLOW_ACTION_TYPE_DROP,
139 RTE_FLOW_ACTION_TYPE_QUEUE,
140 RTE_FLOW_ACTION_TYPE_END,
143 /** Graph of supported items and associated actions. */
144 static const struct mlx5_flow_items mlx5_flow_items[] = {
145 [RTE_FLOW_ITEM_TYPE_END] = {
146 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
147 RTE_FLOW_ITEM_TYPE_VXLAN),
149 [RTE_FLOW_ITEM_TYPE_ETH] = {
150 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
151 RTE_FLOW_ITEM_TYPE_IPV4,
152 RTE_FLOW_ITEM_TYPE_IPV6),
153 .actions = valid_actions,
154 .mask = &(const struct rte_flow_item_eth){
155 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
156 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
158 .mask_sz = sizeof(struct rte_flow_item_eth),
159 .convert = mlx5_flow_create_eth,
160 .dst_sz = sizeof(struct ibv_exp_flow_spec_eth),
162 [RTE_FLOW_ITEM_TYPE_VLAN] = {
163 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
164 RTE_FLOW_ITEM_TYPE_IPV6),
165 .actions = valid_actions,
166 .mask = &(const struct rte_flow_item_vlan){
169 .mask_sz = sizeof(struct rte_flow_item_vlan),
170 .convert = mlx5_flow_create_vlan,
173 [RTE_FLOW_ITEM_TYPE_IPV4] = {
174 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
175 RTE_FLOW_ITEM_TYPE_TCP),
176 .actions = valid_actions,
177 .mask = &(const struct rte_flow_item_ipv4){
183 .mask_sz = sizeof(struct rte_flow_item_ipv4),
184 .convert = mlx5_flow_create_ipv4,
185 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv4),
187 [RTE_FLOW_ITEM_TYPE_IPV6] = {
188 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
189 RTE_FLOW_ITEM_TYPE_TCP),
190 .actions = valid_actions,
191 .mask = &(const struct rte_flow_item_ipv6){
194 0xff, 0xff, 0xff, 0xff,
195 0xff, 0xff, 0xff, 0xff,
196 0xff, 0xff, 0xff, 0xff,
197 0xff, 0xff, 0xff, 0xff,
200 0xff, 0xff, 0xff, 0xff,
201 0xff, 0xff, 0xff, 0xff,
202 0xff, 0xff, 0xff, 0xff,
203 0xff, 0xff, 0xff, 0xff,
207 .mask_sz = sizeof(struct rte_flow_item_ipv6),
208 .convert = mlx5_flow_create_ipv6,
209 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv6),
211 [RTE_FLOW_ITEM_TYPE_UDP] = {
212 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
213 .actions = valid_actions,
214 .mask = &(const struct rte_flow_item_udp){
220 .mask_sz = sizeof(struct rte_flow_item_udp),
221 .convert = mlx5_flow_create_udp,
222 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
224 [RTE_FLOW_ITEM_TYPE_TCP] = {
225 .actions = valid_actions,
226 .mask = &(const struct rte_flow_item_tcp){
232 .mask_sz = sizeof(struct rte_flow_item_tcp),
233 .convert = mlx5_flow_create_tcp,
234 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
236 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
237 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
238 .actions = valid_actions,
239 .mask = &(const struct rte_flow_item_vxlan){
240 .vni = "\xff\xff\xff",
242 .mask_sz = sizeof(struct rte_flow_item_vxlan),
243 .convert = mlx5_flow_create_vxlan,
244 .dst_sz = sizeof(struct ibv_exp_flow_spec_tunnel),
248 /** Structure to pass to the conversion function. */
250 struct ibv_exp_flow_attr *ibv_attr; /**< Verbs attribute. */
251 unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */
252 uint32_t inner; /**< Set once VXLAN is encountered. */
255 struct mlx5_flow_action {
256 uint32_t queue:1; /**< Target is a receive queue. */
257 uint32_t drop:1; /**< Target is a drop queue. */
258 uint32_t queue_id; /**< Identifier of the queue. */
262 * Check support for a given item.
265 * Item specification.
267 * Bit-masks covering supported fields to compare with spec, last and mask in
270 * Bit-Mask size in bytes.
276 mlx5_flow_item_validate(const struct rte_flow_item *item,
277 const uint8_t *mask, unsigned int size)
281 if (!item->spec && (item->mask || item->last))
283 if (item->spec && !item->mask) {
285 const uint8_t *spec = item->spec;
287 for (i = 0; i < size; ++i)
288 if ((spec[i] | mask[i]) != mask[i])
291 if (item->last && !item->mask) {
293 const uint8_t *spec = item->last;
295 for (i = 0; i < size; ++i)
296 if ((spec[i] | mask[i]) != mask[i])
301 const uint8_t *spec = item->mask;
303 for (i = 0; i < size; ++i)
304 if ((spec[i] | mask[i]) != mask[i])
307 if (item->spec && item->last) {
310 const uint8_t *apply = mask;
315 for (i = 0; i < size; ++i) {
316 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
317 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
319 ret = memcmp(spec, last, size);
325 * Validate a flow supported by the NIC.
328 * Pointer to private structure.
330 * Flow rule attributes.
332 * Pattern specification (list terminated by the END pattern item).
334 * Associated actions (list terminated by the END action).
336 * Perform verbose error reporting if not NULL.
337 * @param[in, out] flow
338 * Flow structure to update.
341 * 0 on success, a negative errno value otherwise and rte_errno is set.
344 priv_flow_validate(struct priv *priv,
345 const struct rte_flow_attr *attr,
346 const struct rte_flow_item items[],
347 const struct rte_flow_action actions[],
348 struct rte_flow_error *error,
349 struct mlx5_flow *flow)
351 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
352 struct mlx5_flow_action action = {
359 rte_flow_error_set(error, ENOTSUP,
360 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
362 "groups are not supported");
365 if (attr->priority) {
366 rte_flow_error_set(error, ENOTSUP,
367 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
369 "priorities are not supported");
373 rte_flow_error_set(error, ENOTSUP,
374 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
376 "egress is not supported");
379 if (!attr->ingress) {
380 rte_flow_error_set(error, ENOTSUP,
381 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
383 "only ingress is supported");
386 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
387 const struct mlx5_flow_items *token = NULL;
391 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
393 /* Handle special situation for VLAN. */
394 if (items->type == RTE_FLOW_ITEM_TYPE_VLAN) {
395 if (((const struct rte_flow_item_vlan *)items)->tci >
397 rte_flow_error_set(error, ENOTSUP,
398 RTE_FLOW_ERROR_TYPE_ITEM,
400 "wrong VLAN id value");
406 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
408 if (cur_item->items[i] == items->type) {
409 token = &mlx5_flow_items[items->type];
414 goto exit_item_not_supported;
416 err = mlx5_flow_item_validate(items,
417 (const uint8_t *)cur_item->mask,
418 sizeof(cur_item->mask_sz));
420 goto exit_item_not_supported;
421 if (flow->ibv_attr && cur_item->convert) {
422 err = cur_item->convert(items, cur_item->mask, flow);
424 goto exit_item_not_supported;
426 flow->offset += cur_item->dst_sz;
428 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
429 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
431 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
433 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
434 const struct rte_flow_action_queue *queue =
435 (const struct rte_flow_action_queue *)
438 if (!queue || (queue->index > (priv->rxqs_n - 1)))
439 goto exit_action_not_supported;
442 goto exit_action_not_supported;
445 if (!action.queue && !action.drop) {
446 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
447 NULL, "no valid action");
451 exit_item_not_supported:
452 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
453 items, "item not supported");
455 exit_action_not_supported:
456 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
457 actions, "action not supported");
462 * Validate a flow supported by the NIC.
464 * @see rte_flow_validate()
468 mlx5_flow_validate(struct rte_eth_dev *dev,
469 const struct rte_flow_attr *attr,
470 const struct rte_flow_item items[],
471 const struct rte_flow_action actions[],
472 struct rte_flow_error *error)
474 struct priv *priv = dev->data->dev_private;
476 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr) };
479 ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
485 * Convert Ethernet item to Verbs specification.
488 * Item specification.
489 * @param default_mask[in]
490 * Default bit-masks to use when item->mask is not provided.
491 * @param data[in, out]
495 mlx5_flow_create_eth(const struct rte_flow_item *item,
496 const void *default_mask,
499 const struct rte_flow_item_eth *spec = item->spec;
500 const struct rte_flow_item_eth *mask = item->mask;
501 struct mlx5_flow *flow = (struct mlx5_flow *)data;
502 struct ibv_exp_flow_spec_eth *eth;
503 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
506 ++flow->ibv_attr->num_of_specs;
507 flow->ibv_attr->priority = 2;
508 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
509 *eth = (struct ibv_exp_flow_spec_eth) {
510 .type = flow->inner | IBV_EXP_FLOW_SPEC_ETH,
517 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
518 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
519 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
520 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
521 /* Remove unwanted bits from values. */
522 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
523 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
524 eth->val.src_mac[i] &= eth->mask.src_mac[i];
530 * Convert VLAN item to Verbs specification.
533 * Item specification.
534 * @param default_mask[in]
535 * Default bit-masks to use when item->mask is not provided.
536 * @param data[in, out]
540 mlx5_flow_create_vlan(const struct rte_flow_item *item,
541 const void *default_mask,
544 const struct rte_flow_item_vlan *spec = item->spec;
545 const struct rte_flow_item_vlan *mask = item->mask;
546 struct mlx5_flow *flow = (struct mlx5_flow *)data;
547 struct ibv_exp_flow_spec_eth *eth;
548 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
550 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
555 eth->val.vlan_tag = spec->tci;
556 eth->mask.vlan_tag = mask->tci;
557 eth->val.vlan_tag &= eth->mask.vlan_tag;
562 * Convert IPv4 item to Verbs specification.
565 * Item specification.
566 * @param default_mask[in]
567 * Default bit-masks to use when item->mask is not provided.
568 * @param data[in, out]
572 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
573 const void *default_mask,
576 const struct rte_flow_item_ipv4 *spec = item->spec;
577 const struct rte_flow_item_ipv4 *mask = item->mask;
578 struct mlx5_flow *flow = (struct mlx5_flow *)data;
579 struct ibv_exp_flow_spec_ipv4 *ipv4;
580 unsigned int ipv4_size = sizeof(struct ibv_exp_flow_spec_ipv4);
582 ++flow->ibv_attr->num_of_specs;
583 flow->ibv_attr->priority = 1;
584 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
585 *ipv4 = (struct ibv_exp_flow_spec_ipv4) {
586 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV4,
593 ipv4->val = (struct ibv_exp_flow_ipv4_filter){
594 .src_ip = spec->hdr.src_addr,
595 .dst_ip = spec->hdr.dst_addr,
597 ipv4->mask = (struct ibv_exp_flow_ipv4_filter){
598 .src_ip = mask->hdr.src_addr,
599 .dst_ip = mask->hdr.dst_addr,
601 /* Remove unwanted bits from values. */
602 ipv4->val.src_ip &= ipv4->mask.src_ip;
603 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
608 * Convert IPv6 item to Verbs specification.
611 * Item specification.
612 * @param default_mask[in]
613 * Default bit-masks to use when item->mask is not provided.
614 * @param data[in, out]
618 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
619 const void *default_mask,
622 const struct rte_flow_item_ipv6 *spec = item->spec;
623 const struct rte_flow_item_ipv6 *mask = item->mask;
624 struct mlx5_flow *flow = (struct mlx5_flow *)data;
625 struct ibv_exp_flow_spec_ipv6 *ipv6;
626 unsigned int ipv6_size = sizeof(struct ibv_exp_flow_spec_ipv6);
629 ++flow->ibv_attr->num_of_specs;
630 flow->ibv_attr->priority = 1;
631 ipv6 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
632 *ipv6 = (struct ibv_exp_flow_spec_ipv6) {
633 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV6,
640 memcpy(ipv6->val.src_ip, spec->hdr.src_addr,
641 RTE_DIM(ipv6->val.src_ip));
642 memcpy(ipv6->val.dst_ip, spec->hdr.dst_addr,
643 RTE_DIM(ipv6->val.dst_ip));
644 memcpy(ipv6->mask.src_ip, mask->hdr.src_addr,
645 RTE_DIM(ipv6->mask.src_ip));
646 memcpy(ipv6->mask.dst_ip, mask->hdr.dst_addr,
647 RTE_DIM(ipv6->mask.dst_ip));
648 /* Remove unwanted bits from values. */
649 for (i = 0; i < RTE_DIM(ipv6->val.src_ip); ++i) {
650 ipv6->val.src_ip[i] &= ipv6->mask.src_ip[i];
651 ipv6->val.dst_ip[i] &= ipv6->mask.dst_ip[i];
657 * Convert UDP item to Verbs specification.
660 * Item specification.
661 * @param default_mask[in]
662 * Default bit-masks to use when item->mask is not provided.
663 * @param data[in, out]
667 mlx5_flow_create_udp(const struct rte_flow_item *item,
668 const void *default_mask,
671 const struct rte_flow_item_udp *spec = item->spec;
672 const struct rte_flow_item_udp *mask = item->mask;
673 struct mlx5_flow *flow = (struct mlx5_flow *)data;
674 struct ibv_exp_flow_spec_tcp_udp *udp;
675 unsigned int udp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
677 ++flow->ibv_attr->num_of_specs;
678 flow->ibv_attr->priority = 0;
679 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
680 *udp = (struct ibv_exp_flow_spec_tcp_udp) {
681 .type = flow->inner | IBV_EXP_FLOW_SPEC_UDP,
688 udp->val.dst_port = spec->hdr.dst_port;
689 udp->val.src_port = spec->hdr.src_port;
690 udp->mask.dst_port = mask->hdr.dst_port;
691 udp->mask.src_port = mask->hdr.src_port;
692 /* Remove unwanted bits from values. */
693 udp->val.src_port &= udp->mask.src_port;
694 udp->val.dst_port &= udp->mask.dst_port;
699 * Convert TCP item to Verbs specification.
702 * Item specification.
703 * @param default_mask[in]
704 * Default bit-masks to use when item->mask is not provided.
705 * @param data[in, out]
709 mlx5_flow_create_tcp(const struct rte_flow_item *item,
710 const void *default_mask,
713 const struct rte_flow_item_tcp *spec = item->spec;
714 const struct rte_flow_item_tcp *mask = item->mask;
715 struct mlx5_flow *flow = (struct mlx5_flow *)data;
716 struct ibv_exp_flow_spec_tcp_udp *tcp;
717 unsigned int tcp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
719 ++flow->ibv_attr->num_of_specs;
720 flow->ibv_attr->priority = 0;
721 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
722 *tcp = (struct ibv_exp_flow_spec_tcp_udp) {
723 .type = flow->inner | IBV_EXP_FLOW_SPEC_TCP,
730 tcp->val.dst_port = spec->hdr.dst_port;
731 tcp->val.src_port = spec->hdr.src_port;
732 tcp->mask.dst_port = mask->hdr.dst_port;
733 tcp->mask.src_port = mask->hdr.src_port;
734 /* Remove unwanted bits from values. */
735 tcp->val.src_port &= tcp->mask.src_port;
736 tcp->val.dst_port &= tcp->mask.dst_port;
741 * Convert VXLAN item to Verbs specification.
744 * Item specification.
745 * @param default_mask[in]
746 * Default bit-masks to use when item->mask is not provided.
747 * @param data[in, out]
751 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
752 const void *default_mask,
755 const struct rte_flow_item_vxlan *spec = item->spec;
756 const struct rte_flow_item_vxlan *mask = item->mask;
757 struct mlx5_flow *flow = (struct mlx5_flow *)data;
758 struct ibv_exp_flow_spec_tunnel *vxlan;
759 unsigned int size = sizeof(struct ibv_exp_flow_spec_tunnel);
765 ++flow->ibv_attr->num_of_specs;
766 flow->ibv_attr->priority = 0;
768 vxlan = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
769 *vxlan = (struct ibv_exp_flow_spec_tunnel) {
770 .type = flow->inner | IBV_EXP_FLOW_SPEC_VXLAN_TUNNEL,
773 flow->inner = IBV_EXP_FLOW_SPEC_INNER;
778 memcpy(&id.vni[1], spec->vni, 3);
779 vxlan->val.tunnel_id = id.vlan_id;
780 memcpy(&id.vni[1], mask->vni, 3);
781 vxlan->mask.tunnel_id = id.vlan_id;
782 /* Remove unwanted bits from values. */
783 vxlan->val.tunnel_id &= vxlan->mask.tunnel_id;
788 * Complete flow rule creation.
791 * Pointer to private structure.
793 * Verbs flow attributes.
795 * Target action structure.
797 * Perform verbose error reporting if not NULL.
800 * A flow if the rule could be created.
802 static struct rte_flow *
803 priv_flow_create_action_queue(struct priv *priv,
804 struct ibv_exp_flow_attr *ibv_attr,
805 struct mlx5_flow_action *action,
806 struct rte_flow_error *error)
808 struct rxq_ctrl *rxq;
809 struct rte_flow *rte_flow;
813 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
815 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
816 NULL, "cannot allocate flow memory");
821 ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
822 &(struct ibv_exp_cq_init_attr){
826 rte_flow_error_set(error, ENOMEM,
827 RTE_FLOW_ERROR_TYPE_HANDLE,
828 NULL, "cannot allocate CQ");
831 rte_flow->wq = ibv_exp_create_wq(priv->ctx,
832 &(struct ibv_exp_wq_init_attr){
833 .wq_type = IBV_EXP_WQT_RQ,
840 rxq = container_of((*priv->rxqs)[action->queue_id],
841 struct rxq_ctrl, rxq);
842 rte_flow->rxq = &rxq->rxq;
843 rte_flow->wq = rxq->wq;
845 rte_flow->ibv_attr = ibv_attr;
846 rte_flow->ind_table = ibv_exp_create_rwq_ind_table(
848 &(struct ibv_exp_rwq_ind_table_init_attr){
850 .log_ind_tbl_size = 0,
851 .ind_tbl = &rte_flow->wq,
854 if (!rte_flow->ind_table) {
855 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
856 NULL, "cannot allocate indirection table");
859 rte_flow->qp = ibv_exp_create_qp(
861 &(struct ibv_exp_qp_init_attr){
862 .qp_type = IBV_QPT_RAW_PACKET,
864 IBV_EXP_QP_INIT_ATTR_PD |
865 IBV_EXP_QP_INIT_ATTR_PORT |
866 IBV_EXP_QP_INIT_ATTR_RX_HASH,
868 .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
870 IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
871 .rx_hash_key_len = rss_hash_default_key_len,
872 .rx_hash_key = rss_hash_default_key,
873 .rx_hash_fields_mask = 0,
874 .rwq_ind_tbl = rte_flow->ind_table,
876 .port_num = priv->port,
879 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
880 NULL, "cannot allocate QP");
883 rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp,
885 if (!rte_flow->ibv_flow) {
886 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
887 NULL, "flow rule creation failure");
894 ibv_destroy_qp(rte_flow->qp);
895 if (rte_flow->ind_table)
896 ibv_exp_destroy_rwq_ind_table(rte_flow->ind_table);
897 if (!rte_flow->rxq && rte_flow->wq)
898 ibv_exp_destroy_wq(rte_flow->wq);
899 if (!rte_flow->rxq && rte_flow->cq)
900 ibv_destroy_cq(rte_flow->cq);
901 rte_free(rte_flow->ibv_attr);
910 * Pointer to private structure.
912 * Flow rule attributes.
914 * Pattern specification (list terminated by the END pattern item).
916 * Associated actions (list terminated by the END action).
918 * Perform verbose error reporting if not NULL.
921 * A flow on success, NULL otherwise.
923 static struct rte_flow *
924 priv_flow_create(struct priv *priv,
925 const struct rte_flow_attr *attr,
926 const struct rte_flow_item items[],
927 const struct rte_flow_action actions[],
928 struct rte_flow_error *error)
930 struct rte_flow *rte_flow;
931 struct mlx5_flow_action action;
932 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr), };
935 err = priv_flow_validate(priv, attr, items, actions, error, &flow);
938 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
939 flow.offset = sizeof(struct ibv_exp_flow_attr);
940 if (!flow.ibv_attr) {
941 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
942 NULL, "cannot allocate ibv_attr memory");
945 *flow.ibv_attr = (struct ibv_exp_flow_attr){
946 .type = IBV_EXP_FLOW_ATTR_NORMAL,
947 .size = sizeof(struct ibv_exp_flow_attr),
948 .priority = attr->priority,
955 claim_zero(priv_flow_validate(priv, attr, items, actions,
957 action = (struct mlx5_flow_action){
961 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
962 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
964 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
967 ((const struct rte_flow_action_queue *)
968 actions->conf)->index;
969 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
972 rte_flow_error_set(error, ENOTSUP,
973 RTE_FLOW_ERROR_TYPE_ACTION,
974 actions, "unsupported action");
978 rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
982 rte_free(flow.ibv_attr);
989 * @see rte_flow_create()
993 mlx5_flow_create(struct rte_eth_dev *dev,
994 const struct rte_flow_attr *attr,
995 const struct rte_flow_item items[],
996 const struct rte_flow_action actions[],
997 struct rte_flow_error *error)
999 struct priv *priv = dev->data->dev_private;
1000 struct rte_flow *flow;
1003 flow = priv_flow_create(priv, attr, items, actions, error);
1005 LIST_INSERT_HEAD(&priv->flows, flow, next);
1006 DEBUG("Flow created %p", (void *)flow);
1016 * Pointer to private structure.
1021 priv_flow_destroy(struct priv *priv,
1022 struct rte_flow *flow)
1025 LIST_REMOVE(flow, next);
1027 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1029 claim_zero(ibv_destroy_qp(flow->qp));
1030 if (flow->ind_table)
1031 claim_zero(ibv_exp_destroy_rwq_ind_table(flow->ind_table));
1032 if (!flow->rxq && flow->wq)
1033 claim_zero(ibv_exp_destroy_wq(flow->wq));
1034 if (!flow->rxq && flow->cq)
1035 claim_zero(ibv_destroy_cq(flow->cq));
1036 rte_free(flow->ibv_attr);
1037 DEBUG("Flow destroyed %p", (void *)flow);
1044 * @see rte_flow_destroy()
1048 mlx5_flow_destroy(struct rte_eth_dev *dev,
1049 struct rte_flow *flow,
1050 struct rte_flow_error *error)
1052 struct priv *priv = dev->data->dev_private;
1056 priv_flow_destroy(priv, flow);
1062 * Destroy all flows.
1065 * Pointer to private structure.
1068 priv_flow_flush(struct priv *priv)
1070 while (!LIST_EMPTY(&priv->flows)) {
1071 struct rte_flow *flow;
1073 flow = LIST_FIRST(&priv->flows);
1074 priv_flow_destroy(priv, flow);
1079 * Destroy all flows.
1081 * @see rte_flow_flush()
1085 mlx5_flow_flush(struct rte_eth_dev *dev,
1086 struct rte_flow_error *error)
1088 struct priv *priv = dev->data->dev_private;
1092 priv_flow_flush(priv);
1100 * Called by dev_stop() to remove all flows.
1103 * Pointer to private structure.
1106 priv_flow_stop(struct priv *priv)
1108 struct rte_flow *flow;
1110 for (flow = LIST_FIRST(&priv->flows);
1112 flow = LIST_NEXT(flow, next)) {
1113 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1114 flow->ibv_flow = NULL;
1115 DEBUG("Flow %p removed", (void *)flow);
1123 * Pointer to private structure.
1126 * 0 on success, a errno value otherwise and rte_errno is set.
1129 priv_flow_start(struct priv *priv)
1131 struct rte_flow *flow;
1133 for (flow = LIST_FIRST(&priv->flows);
1135 flow = LIST_NEXT(flow, next)) {
1136 flow->ibv_flow = ibv_exp_create_flow(flow->qp,
1138 if (!flow->ibv_flow) {
1139 DEBUG("Flow %p cannot be applied", (void *)flow);
1143 DEBUG("Flow %p applied", (void *)flow);