4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
56 mlx5_flow_create_eth(const struct rte_flow_item *item,
57 const void *default_mask,
61 mlx5_flow_create_vlan(const struct rte_flow_item *item,
62 const void *default_mask,
66 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
67 const void *default_mask,
71 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
72 const void *default_mask,
76 mlx5_flow_create_udp(const struct rte_flow_item *item,
77 const void *default_mask,
81 mlx5_flow_create_tcp(const struct rte_flow_item *item,
82 const void *default_mask,
86 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
87 const void *default_mask,
91 LIST_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
92 struct ibv_exp_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
93 struct ibv_exp_rwq_ind_table *ind_table; /**< Indirection table. */
94 struct ibv_qp *qp; /**< Verbs queue pair. */
95 struct ibv_exp_flow *ibv_flow; /**< Verbs flow. */
96 struct ibv_exp_wq *wq; /**< Verbs work queue. */
97 struct ibv_cq *cq; /**< Verbs completion queue. */
98 struct rxq *rxq; /**< Pointer to the queue, NULL if drop queue. */
99 uint32_t mark:1; /**< Set if the flow is marked. */
102 /** Static initializer for items. */
104 (const enum rte_flow_item_type []){ \
105 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
108 /** Structure to generate a simple graph of layers supported by the NIC. */
109 struct mlx5_flow_items {
110 /** List of possible actions for these items. */
111 const enum rte_flow_action_type *const actions;
112 /** Bit-masks corresponding to the possibilities for the item. */
114 /** Bit-masks size in bytes. */
115 const unsigned int mask_sz;
117 * Conversion function from rte_flow to NIC specific flow.
120 * rte_flow item to convert.
121 * @param default_mask
122 * Default bit-masks to use when item->mask is not provided.
124 * Internal structure to store the conversion.
127 * 0 on success, negative value otherwise.
129 int (*convert)(const struct rte_flow_item *item,
130 const void *default_mask,
132 /** Size in bytes of the destination structure. */
133 const unsigned int dst_sz;
134 /** List of possible following items. */
135 const enum rte_flow_item_type *const items;
138 /** Valid action for this PMD. */
139 static const enum rte_flow_action_type valid_actions[] = {
140 RTE_FLOW_ACTION_TYPE_DROP,
141 RTE_FLOW_ACTION_TYPE_QUEUE,
142 RTE_FLOW_ACTION_TYPE_MARK,
143 RTE_FLOW_ACTION_TYPE_END,
146 /** Graph of supported items and associated actions. */
147 static const struct mlx5_flow_items mlx5_flow_items[] = {
148 [RTE_FLOW_ITEM_TYPE_END] = {
149 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH,
150 RTE_FLOW_ITEM_TYPE_VXLAN),
152 [RTE_FLOW_ITEM_TYPE_ETH] = {
153 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
154 RTE_FLOW_ITEM_TYPE_IPV4,
155 RTE_FLOW_ITEM_TYPE_IPV6),
156 .actions = valid_actions,
157 .mask = &(const struct rte_flow_item_eth){
158 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
159 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
161 .mask_sz = sizeof(struct rte_flow_item_eth),
162 .convert = mlx5_flow_create_eth,
163 .dst_sz = sizeof(struct ibv_exp_flow_spec_eth),
165 [RTE_FLOW_ITEM_TYPE_VLAN] = {
166 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
167 RTE_FLOW_ITEM_TYPE_IPV6),
168 .actions = valid_actions,
169 .mask = &(const struct rte_flow_item_vlan){
172 .mask_sz = sizeof(struct rte_flow_item_vlan),
173 .convert = mlx5_flow_create_vlan,
176 [RTE_FLOW_ITEM_TYPE_IPV4] = {
177 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
178 RTE_FLOW_ITEM_TYPE_TCP),
179 .actions = valid_actions,
180 .mask = &(const struct rte_flow_item_ipv4){
186 .mask_sz = sizeof(struct rte_flow_item_ipv4),
187 .convert = mlx5_flow_create_ipv4,
188 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv4),
190 [RTE_FLOW_ITEM_TYPE_IPV6] = {
191 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
192 RTE_FLOW_ITEM_TYPE_TCP),
193 .actions = valid_actions,
194 .mask = &(const struct rte_flow_item_ipv6){
197 0xff, 0xff, 0xff, 0xff,
198 0xff, 0xff, 0xff, 0xff,
199 0xff, 0xff, 0xff, 0xff,
200 0xff, 0xff, 0xff, 0xff,
203 0xff, 0xff, 0xff, 0xff,
204 0xff, 0xff, 0xff, 0xff,
205 0xff, 0xff, 0xff, 0xff,
206 0xff, 0xff, 0xff, 0xff,
210 .mask_sz = sizeof(struct rte_flow_item_ipv6),
211 .convert = mlx5_flow_create_ipv6,
212 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv6),
214 [RTE_FLOW_ITEM_TYPE_UDP] = {
215 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VXLAN),
216 .actions = valid_actions,
217 .mask = &(const struct rte_flow_item_udp){
223 .mask_sz = sizeof(struct rte_flow_item_udp),
224 .convert = mlx5_flow_create_udp,
225 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
227 [RTE_FLOW_ITEM_TYPE_TCP] = {
228 .actions = valid_actions,
229 .mask = &(const struct rte_flow_item_tcp){
235 .mask_sz = sizeof(struct rte_flow_item_tcp),
236 .convert = mlx5_flow_create_tcp,
237 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
239 [RTE_FLOW_ITEM_TYPE_VXLAN] = {
240 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
241 .actions = valid_actions,
242 .mask = &(const struct rte_flow_item_vxlan){
243 .vni = "\xff\xff\xff",
245 .mask_sz = sizeof(struct rte_flow_item_vxlan),
246 .convert = mlx5_flow_create_vxlan,
247 .dst_sz = sizeof(struct ibv_exp_flow_spec_tunnel),
251 /** Structure to pass to the conversion function. */
253 struct ibv_exp_flow_attr *ibv_attr; /**< Verbs attribute. */
254 unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */
255 uint32_t inner; /**< Set once VXLAN is encountered. */
258 struct mlx5_flow_action {
259 uint32_t queue:1; /**< Target is a receive queue. */
260 uint32_t drop:1; /**< Target is a drop queue. */
261 uint32_t mark:1; /**< Mark is present in the flow. */
262 uint32_t queue_id; /**< Identifier of the queue. */
263 uint32_t mark_id; /**< Mark identifier. */
267 * Check support for a given item.
270 * Item specification.
272 * Bit-masks covering supported fields to compare with spec, last and mask in
275 * Bit-Mask size in bytes.
281 mlx5_flow_item_validate(const struct rte_flow_item *item,
282 const uint8_t *mask, unsigned int size)
286 if (!item->spec && (item->mask || item->last))
288 if (item->spec && !item->mask) {
290 const uint8_t *spec = item->spec;
292 for (i = 0; i < size; ++i)
293 if ((spec[i] | mask[i]) != mask[i])
296 if (item->last && !item->mask) {
298 const uint8_t *spec = item->last;
300 for (i = 0; i < size; ++i)
301 if ((spec[i] | mask[i]) != mask[i])
306 const uint8_t *spec = item->mask;
308 for (i = 0; i < size; ++i)
309 if ((spec[i] | mask[i]) != mask[i])
312 if (item->spec && item->last) {
315 const uint8_t *apply = mask;
320 for (i = 0; i < size; ++i) {
321 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
322 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
324 ret = memcmp(spec, last, size);
330 * Validate a flow supported by the NIC.
333 * Pointer to private structure.
335 * Flow rule attributes.
337 * Pattern specification (list terminated by the END pattern item).
339 * Associated actions (list terminated by the END action).
341 * Perform verbose error reporting if not NULL.
342 * @param[in, out] flow
343 * Flow structure to update.
346 * 0 on success, a negative errno value otherwise and rte_errno is set.
349 priv_flow_validate(struct priv *priv,
350 const struct rte_flow_attr *attr,
351 const struct rte_flow_item items[],
352 const struct rte_flow_action actions[],
353 struct rte_flow_error *error,
354 struct mlx5_flow *flow)
356 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
357 struct mlx5_flow_action action = {
365 rte_flow_error_set(error, ENOTSUP,
366 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
368 "groups are not supported");
371 if (attr->priority) {
372 rte_flow_error_set(error, ENOTSUP,
373 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
375 "priorities are not supported");
379 rte_flow_error_set(error, ENOTSUP,
380 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
382 "egress is not supported");
385 if (!attr->ingress) {
386 rte_flow_error_set(error, ENOTSUP,
387 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
389 "only ingress is supported");
392 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
393 const struct mlx5_flow_items *token = NULL;
397 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
399 /* Handle special situation for VLAN. */
400 if (items->type == RTE_FLOW_ITEM_TYPE_VLAN) {
401 if (((const struct rte_flow_item_vlan *)items)->tci >
403 rte_flow_error_set(error, ENOTSUP,
404 RTE_FLOW_ERROR_TYPE_ITEM,
406 "wrong VLAN id value");
412 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
414 if (cur_item->items[i] == items->type) {
415 token = &mlx5_flow_items[items->type];
420 goto exit_item_not_supported;
422 err = mlx5_flow_item_validate(items,
423 (const uint8_t *)cur_item->mask,
424 sizeof(cur_item->mask_sz));
426 goto exit_item_not_supported;
427 if (flow->ibv_attr && cur_item->convert) {
428 err = cur_item->convert(items, cur_item->mask, flow);
430 goto exit_item_not_supported;
432 flow->offset += cur_item->dst_sz;
434 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
435 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
437 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
439 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
440 const struct rte_flow_action_queue *queue =
441 (const struct rte_flow_action_queue *)
444 if (!queue || (queue->index > (priv->rxqs_n - 1)))
445 goto exit_action_not_supported;
447 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
448 const struct rte_flow_action_mark *mark =
449 (const struct rte_flow_action_mark *)
452 if (mark && (mark->id >= MLX5_FLOW_MARK_MAX)) {
453 rte_flow_error_set(error, ENOTSUP,
454 RTE_FLOW_ERROR_TYPE_ACTION,
456 "mark must be between 0"
462 goto exit_action_not_supported;
465 if (action.mark && !flow->ibv_attr)
466 flow->offset += sizeof(struct ibv_exp_flow_spec_action_tag);
467 if (!action.queue && !action.drop) {
468 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
469 NULL, "no valid action");
473 exit_item_not_supported:
474 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
475 items, "item not supported");
477 exit_action_not_supported:
478 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
479 actions, "action not supported");
484 * Validate a flow supported by the NIC.
486 * @see rte_flow_validate()
490 mlx5_flow_validate(struct rte_eth_dev *dev,
491 const struct rte_flow_attr *attr,
492 const struct rte_flow_item items[],
493 const struct rte_flow_action actions[],
494 struct rte_flow_error *error)
496 struct priv *priv = dev->data->dev_private;
498 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr) };
501 ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
507 * Convert Ethernet item to Verbs specification.
510 * Item specification.
511 * @param default_mask[in]
512 * Default bit-masks to use when item->mask is not provided.
513 * @param data[in, out]
517 mlx5_flow_create_eth(const struct rte_flow_item *item,
518 const void *default_mask,
521 const struct rte_flow_item_eth *spec = item->spec;
522 const struct rte_flow_item_eth *mask = item->mask;
523 struct mlx5_flow *flow = (struct mlx5_flow *)data;
524 struct ibv_exp_flow_spec_eth *eth;
525 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
528 ++flow->ibv_attr->num_of_specs;
529 flow->ibv_attr->priority = 2;
530 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
531 *eth = (struct ibv_exp_flow_spec_eth) {
532 .type = flow->inner | IBV_EXP_FLOW_SPEC_ETH,
539 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
540 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
541 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
542 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
543 /* Remove unwanted bits from values. */
544 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
545 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
546 eth->val.src_mac[i] &= eth->mask.src_mac[i];
552 * Convert VLAN item to Verbs specification.
555 * Item specification.
556 * @param default_mask[in]
557 * Default bit-masks to use when item->mask is not provided.
558 * @param data[in, out]
562 mlx5_flow_create_vlan(const struct rte_flow_item *item,
563 const void *default_mask,
566 const struct rte_flow_item_vlan *spec = item->spec;
567 const struct rte_flow_item_vlan *mask = item->mask;
568 struct mlx5_flow *flow = (struct mlx5_flow *)data;
569 struct ibv_exp_flow_spec_eth *eth;
570 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
572 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
577 eth->val.vlan_tag = spec->tci;
578 eth->mask.vlan_tag = mask->tci;
579 eth->val.vlan_tag &= eth->mask.vlan_tag;
584 * Convert IPv4 item to Verbs specification.
587 * Item specification.
588 * @param default_mask[in]
589 * Default bit-masks to use when item->mask is not provided.
590 * @param data[in, out]
594 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
595 const void *default_mask,
598 const struct rte_flow_item_ipv4 *spec = item->spec;
599 const struct rte_flow_item_ipv4 *mask = item->mask;
600 struct mlx5_flow *flow = (struct mlx5_flow *)data;
601 struct ibv_exp_flow_spec_ipv4 *ipv4;
602 unsigned int ipv4_size = sizeof(struct ibv_exp_flow_spec_ipv4);
604 ++flow->ibv_attr->num_of_specs;
605 flow->ibv_attr->priority = 1;
606 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
607 *ipv4 = (struct ibv_exp_flow_spec_ipv4) {
608 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV4,
615 ipv4->val = (struct ibv_exp_flow_ipv4_filter){
616 .src_ip = spec->hdr.src_addr,
617 .dst_ip = spec->hdr.dst_addr,
619 ipv4->mask = (struct ibv_exp_flow_ipv4_filter){
620 .src_ip = mask->hdr.src_addr,
621 .dst_ip = mask->hdr.dst_addr,
623 /* Remove unwanted bits from values. */
624 ipv4->val.src_ip &= ipv4->mask.src_ip;
625 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
630 * Convert IPv6 item to Verbs specification.
633 * Item specification.
634 * @param default_mask[in]
635 * Default bit-masks to use when item->mask is not provided.
636 * @param data[in, out]
640 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
641 const void *default_mask,
644 const struct rte_flow_item_ipv6 *spec = item->spec;
645 const struct rte_flow_item_ipv6 *mask = item->mask;
646 struct mlx5_flow *flow = (struct mlx5_flow *)data;
647 struct ibv_exp_flow_spec_ipv6 *ipv6;
648 unsigned int ipv6_size = sizeof(struct ibv_exp_flow_spec_ipv6);
651 ++flow->ibv_attr->num_of_specs;
652 flow->ibv_attr->priority = 1;
653 ipv6 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
654 *ipv6 = (struct ibv_exp_flow_spec_ipv6) {
655 .type = flow->inner | IBV_EXP_FLOW_SPEC_IPV6,
662 memcpy(ipv6->val.src_ip, spec->hdr.src_addr,
663 RTE_DIM(ipv6->val.src_ip));
664 memcpy(ipv6->val.dst_ip, spec->hdr.dst_addr,
665 RTE_DIM(ipv6->val.dst_ip));
666 memcpy(ipv6->mask.src_ip, mask->hdr.src_addr,
667 RTE_DIM(ipv6->mask.src_ip));
668 memcpy(ipv6->mask.dst_ip, mask->hdr.dst_addr,
669 RTE_DIM(ipv6->mask.dst_ip));
670 /* Remove unwanted bits from values. */
671 for (i = 0; i < RTE_DIM(ipv6->val.src_ip); ++i) {
672 ipv6->val.src_ip[i] &= ipv6->mask.src_ip[i];
673 ipv6->val.dst_ip[i] &= ipv6->mask.dst_ip[i];
679 * Convert UDP item to Verbs specification.
682 * Item specification.
683 * @param default_mask[in]
684 * Default bit-masks to use when item->mask is not provided.
685 * @param data[in, out]
689 mlx5_flow_create_udp(const struct rte_flow_item *item,
690 const void *default_mask,
693 const struct rte_flow_item_udp *spec = item->spec;
694 const struct rte_flow_item_udp *mask = item->mask;
695 struct mlx5_flow *flow = (struct mlx5_flow *)data;
696 struct ibv_exp_flow_spec_tcp_udp *udp;
697 unsigned int udp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
699 ++flow->ibv_attr->num_of_specs;
700 flow->ibv_attr->priority = 0;
701 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
702 *udp = (struct ibv_exp_flow_spec_tcp_udp) {
703 .type = flow->inner | IBV_EXP_FLOW_SPEC_UDP,
710 udp->val.dst_port = spec->hdr.dst_port;
711 udp->val.src_port = spec->hdr.src_port;
712 udp->mask.dst_port = mask->hdr.dst_port;
713 udp->mask.src_port = mask->hdr.src_port;
714 /* Remove unwanted bits from values. */
715 udp->val.src_port &= udp->mask.src_port;
716 udp->val.dst_port &= udp->mask.dst_port;
721 * Convert TCP item to Verbs specification.
724 * Item specification.
725 * @param default_mask[in]
726 * Default bit-masks to use when item->mask is not provided.
727 * @param data[in, out]
731 mlx5_flow_create_tcp(const struct rte_flow_item *item,
732 const void *default_mask,
735 const struct rte_flow_item_tcp *spec = item->spec;
736 const struct rte_flow_item_tcp *mask = item->mask;
737 struct mlx5_flow *flow = (struct mlx5_flow *)data;
738 struct ibv_exp_flow_spec_tcp_udp *tcp;
739 unsigned int tcp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
741 ++flow->ibv_attr->num_of_specs;
742 flow->ibv_attr->priority = 0;
743 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
744 *tcp = (struct ibv_exp_flow_spec_tcp_udp) {
745 .type = flow->inner | IBV_EXP_FLOW_SPEC_TCP,
752 tcp->val.dst_port = spec->hdr.dst_port;
753 tcp->val.src_port = spec->hdr.src_port;
754 tcp->mask.dst_port = mask->hdr.dst_port;
755 tcp->mask.src_port = mask->hdr.src_port;
756 /* Remove unwanted bits from values. */
757 tcp->val.src_port &= tcp->mask.src_port;
758 tcp->val.dst_port &= tcp->mask.dst_port;
763 * Convert VXLAN item to Verbs specification.
766 * Item specification.
767 * @param default_mask[in]
768 * Default bit-masks to use when item->mask is not provided.
769 * @param data[in, out]
773 mlx5_flow_create_vxlan(const struct rte_flow_item *item,
774 const void *default_mask,
777 const struct rte_flow_item_vxlan *spec = item->spec;
778 const struct rte_flow_item_vxlan *mask = item->mask;
779 struct mlx5_flow *flow = (struct mlx5_flow *)data;
780 struct ibv_exp_flow_spec_tunnel *vxlan;
781 unsigned int size = sizeof(struct ibv_exp_flow_spec_tunnel);
787 ++flow->ibv_attr->num_of_specs;
788 flow->ibv_attr->priority = 0;
790 vxlan = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
791 *vxlan = (struct ibv_exp_flow_spec_tunnel) {
792 .type = flow->inner | IBV_EXP_FLOW_SPEC_VXLAN_TUNNEL,
795 flow->inner = IBV_EXP_FLOW_SPEC_INNER;
800 memcpy(&id.vni[1], spec->vni, 3);
801 vxlan->val.tunnel_id = id.vlan_id;
802 memcpy(&id.vni[1], mask->vni, 3);
803 vxlan->mask.tunnel_id = id.vlan_id;
804 /* Remove unwanted bits from values. */
805 vxlan->val.tunnel_id &= vxlan->mask.tunnel_id;
810 * Convert mark/flag action to Verbs specification.
813 * Pointer to MLX5 flow structure.
818 mlx5_flow_create_flag_mark(struct mlx5_flow *flow, uint32_t mark_id)
820 struct ibv_exp_flow_spec_action_tag *tag;
821 unsigned int size = sizeof(struct ibv_exp_flow_spec_action_tag);
823 tag = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
824 *tag = (struct ibv_exp_flow_spec_action_tag){
825 .type = IBV_EXP_FLOW_SPEC_ACTION_TAG,
827 .tag_id = mlx5_flow_mark_set(mark_id),
829 ++flow->ibv_attr->num_of_specs;
834 * Complete flow rule creation.
837 * Pointer to private structure.
839 * Verbs flow attributes.
841 * Target action structure.
843 * Perform verbose error reporting if not NULL.
846 * A flow if the rule could be created.
848 static struct rte_flow *
849 priv_flow_create_action_queue(struct priv *priv,
850 struct ibv_exp_flow_attr *ibv_attr,
851 struct mlx5_flow_action *action,
852 struct rte_flow_error *error)
854 struct rxq_ctrl *rxq;
855 struct rte_flow *rte_flow;
859 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
861 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
862 NULL, "cannot allocate flow memory");
867 ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
868 &(struct ibv_exp_cq_init_attr){
872 rte_flow_error_set(error, ENOMEM,
873 RTE_FLOW_ERROR_TYPE_HANDLE,
874 NULL, "cannot allocate CQ");
877 rte_flow->wq = ibv_exp_create_wq(priv->ctx,
878 &(struct ibv_exp_wq_init_attr){
879 .wq_type = IBV_EXP_WQT_RQ,
886 rxq = container_of((*priv->rxqs)[action->queue_id],
887 struct rxq_ctrl, rxq);
888 rte_flow->rxq = &rxq->rxq;
889 rxq->rxq.mark |= action->mark;
890 rte_flow->wq = rxq->wq;
892 rte_flow->mark = action->mark;
893 rte_flow->ibv_attr = ibv_attr;
894 rte_flow->ind_table = ibv_exp_create_rwq_ind_table(
896 &(struct ibv_exp_rwq_ind_table_init_attr){
898 .log_ind_tbl_size = 0,
899 .ind_tbl = &rte_flow->wq,
902 if (!rte_flow->ind_table) {
903 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
904 NULL, "cannot allocate indirection table");
907 rte_flow->qp = ibv_exp_create_qp(
909 &(struct ibv_exp_qp_init_attr){
910 .qp_type = IBV_QPT_RAW_PACKET,
912 IBV_EXP_QP_INIT_ATTR_PD |
913 IBV_EXP_QP_INIT_ATTR_PORT |
914 IBV_EXP_QP_INIT_ATTR_RX_HASH,
916 .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
918 IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
919 .rx_hash_key_len = rss_hash_default_key_len,
920 .rx_hash_key = rss_hash_default_key,
921 .rx_hash_fields_mask = 0,
922 .rwq_ind_tbl = rte_flow->ind_table,
924 .port_num = priv->port,
927 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
928 NULL, "cannot allocate QP");
931 rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp,
933 if (!rte_flow->ibv_flow) {
934 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
935 NULL, "flow rule creation failure");
942 ibv_destroy_qp(rte_flow->qp);
943 if (rte_flow->ind_table)
944 ibv_exp_destroy_rwq_ind_table(rte_flow->ind_table);
945 if (!rte_flow->rxq && rte_flow->wq)
946 ibv_exp_destroy_wq(rte_flow->wq);
947 if (!rte_flow->rxq && rte_flow->cq)
948 ibv_destroy_cq(rte_flow->cq);
949 rte_free(rte_flow->ibv_attr);
958 * Pointer to private structure.
960 * Flow rule attributes.
962 * Pattern specification (list terminated by the END pattern item).
964 * Associated actions (list terminated by the END action).
966 * Perform verbose error reporting if not NULL.
969 * A flow on success, NULL otherwise.
971 static struct rte_flow *
972 priv_flow_create(struct priv *priv,
973 const struct rte_flow_attr *attr,
974 const struct rte_flow_item items[],
975 const struct rte_flow_action actions[],
976 struct rte_flow_error *error)
978 struct rte_flow *rte_flow;
979 struct mlx5_flow_action action;
980 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr), };
983 err = priv_flow_validate(priv, attr, items, actions, error, &flow);
986 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
987 flow.offset = sizeof(struct ibv_exp_flow_attr);
988 if (!flow.ibv_attr) {
989 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
990 NULL, "cannot allocate ibv_attr memory");
993 *flow.ibv_attr = (struct ibv_exp_flow_attr){
994 .type = IBV_EXP_FLOW_ATTR_NORMAL,
995 .size = sizeof(struct ibv_exp_flow_attr),
996 .priority = attr->priority,
1003 claim_zero(priv_flow_validate(priv, attr, items, actions,
1005 action = (struct mlx5_flow_action){
1009 .mark_id = MLX5_FLOW_MARK_DEFAULT,
1011 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1012 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1014 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1017 ((const struct rte_flow_action_queue *)
1018 actions->conf)->index;
1019 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1021 } else if (actions->type == RTE_FLOW_ACTION_TYPE_MARK) {
1022 const struct rte_flow_action_mark *mark =
1023 (const struct rte_flow_action_mark *)
1027 action.mark_id = mark->id;
1030 rte_flow_error_set(error, ENOTSUP,
1031 RTE_FLOW_ERROR_TYPE_ACTION,
1032 actions, "unsupported action");
1037 mlx5_flow_create_flag_mark(&flow, action.mark_id);
1038 flow.offset += sizeof(struct ibv_exp_flow_spec_action_tag);
1040 rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
1044 rte_free(flow.ibv_attr);
1051 * @see rte_flow_create()
1055 mlx5_flow_create(struct rte_eth_dev *dev,
1056 const struct rte_flow_attr *attr,
1057 const struct rte_flow_item items[],
1058 const struct rte_flow_action actions[],
1059 struct rte_flow_error *error)
1061 struct priv *priv = dev->data->dev_private;
1062 struct rte_flow *flow;
1065 flow = priv_flow_create(priv, attr, items, actions, error);
1067 LIST_INSERT_HEAD(&priv->flows, flow, next);
1068 DEBUG("Flow created %p", (void *)flow);
1078 * Pointer to private structure.
1083 priv_flow_destroy(struct priv *priv,
1084 struct rte_flow *flow)
1087 LIST_REMOVE(flow, next);
1089 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1091 claim_zero(ibv_destroy_qp(flow->qp));
1092 if (flow->ind_table)
1093 claim_zero(ibv_exp_destroy_rwq_ind_table(flow->ind_table));
1094 if (!flow->rxq && flow->wq)
1095 claim_zero(ibv_exp_destroy_wq(flow->wq));
1096 if (!flow->rxq && flow->cq)
1097 claim_zero(ibv_destroy_cq(flow->cq));
1099 struct rte_flow *tmp;
1100 uint32_t mark_n = 0;
1102 for (tmp = LIST_FIRST(&priv->flows);
1104 tmp = LIST_NEXT(tmp, next)) {
1105 if ((flow->rxq == tmp->rxq) && tmp->mark)
1108 flow->rxq->mark = !!mark_n;
1110 rte_free(flow->ibv_attr);
1111 DEBUG("Flow destroyed %p", (void *)flow);
1118 * @see rte_flow_destroy()
1122 mlx5_flow_destroy(struct rte_eth_dev *dev,
1123 struct rte_flow *flow,
1124 struct rte_flow_error *error)
1126 struct priv *priv = dev->data->dev_private;
1130 priv_flow_destroy(priv, flow);
1136 * Destroy all flows.
1139 * Pointer to private structure.
1142 priv_flow_flush(struct priv *priv)
1144 while (!LIST_EMPTY(&priv->flows)) {
1145 struct rte_flow *flow;
1147 flow = LIST_FIRST(&priv->flows);
1148 priv_flow_destroy(priv, flow);
1153 * Destroy all flows.
1155 * @see rte_flow_flush()
1159 mlx5_flow_flush(struct rte_eth_dev *dev,
1160 struct rte_flow_error *error)
1162 struct priv *priv = dev->data->dev_private;
1166 priv_flow_flush(priv);
1174 * Called by dev_stop() to remove all flows.
1177 * Pointer to private structure.
1180 priv_flow_stop(struct priv *priv)
1182 struct rte_flow *flow;
1184 for (flow = LIST_FIRST(&priv->flows);
1186 flow = LIST_NEXT(flow, next)) {
1187 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1188 flow->ibv_flow = NULL;
1190 flow->rxq->mark = 0;
1191 DEBUG("Flow %p removed", (void *)flow);
1199 * Pointer to private structure.
1202 * 0 on success, a errno value otherwise and rte_errno is set.
1205 priv_flow_start(struct priv *priv)
1207 struct rte_flow *flow;
1209 for (flow = LIST_FIRST(&priv->flows);
1211 flow = LIST_NEXT(flow, next)) {
1212 flow->ibv_flow = ibv_exp_create_flow(flow->qp,
1214 if (!flow->ibv_flow) {
1215 DEBUG("Flow %p cannot be applied", (void *)flow);
1219 DEBUG("Flow %p applied", (void *)flow);
1221 flow->rxq->mark |= flow->mark;