4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Flow API operations for mlx4 driver.
39 #include <arpa/inet.h>
46 #include <sys/queue.h>
48 /* Verbs headers do not support -pedantic. */
50 #pragma GCC diagnostic ignored "-Wpedantic"
52 #include <infiniband/verbs.h>
54 #pragma GCC diagnostic error "-Wpedantic"
57 #include <rte_byteorder.h>
58 #include <rte_errno.h>
59 #include <rte_eth_ctrl.h>
60 #include <rte_ethdev.h>
62 #include <rte_flow_driver.h>
63 #include <rte_malloc.h>
67 #include "mlx4_flow.h"
68 #include "mlx4_rxtx.h"
69 #include "mlx4_utils.h"
71 /** Static initializer for a list of subsequent item types. */
72 #define NEXT_ITEM(...) \
73 (const enum rte_flow_item_type []){ \
74 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
77 /** Processor structure associated with a flow item. */
78 struct mlx4_flow_proc_item {
79 /** Bit-masks corresponding to the possibilities for the item. */
82 * Default bit-masks to use when item->mask is not provided. When
83 * \default_mask is also NULL, the full supported bit-mask (\mask) is
86 const void *default_mask;
87 /** Bit-masks size in bytes. */
88 const unsigned int mask_sz;
90 * Check support for a given item.
95 * Bit-masks covering supported fields to compare with spec,
99 * Bit-Mask size in bytes.
102 * 0 on success, negative value otherwise.
104 int (*validate)(const struct rte_flow_item *item,
105 const uint8_t *mask, unsigned int size);
107 * Conversion function from rte_flow to NIC specific flow.
110 * rte_flow item to convert.
111 * @param default_mask
112 * Default bit-masks to use when item->mask is not provided.
114 * Flow rule handle to update.
117 * 0 on success, negative value otherwise.
119 int (*convert)(const struct rte_flow_item *item,
120 const void *default_mask,
121 struct rte_flow *flow);
122 /** Size in bytes of the destination structure. */
123 const unsigned int dst_sz;
124 /** List of possible subsequent items. */
125 const enum rte_flow_item_type *const next_item;
128 /** Shared resources for drop flow rules. */
130 struct ibv_qp *qp; /**< QP target. */
131 struct ibv_cq *cq; /**< CQ associated with above QP. */
132 struct priv *priv; /**< Back pointer to private data. */
133 uint32_t refcnt; /**< Reference count. */
137 * Convert Ethernet item to Verbs specification.
140 * Item specification.
141 * @param default_mask[in]
142 * Default bit-masks to use when item->mask is not provided.
143 * @param flow[in, out]
144 * Flow rule handle to update.
147 mlx4_flow_create_eth(const struct rte_flow_item *item,
148 const void *default_mask,
149 struct rte_flow *flow)
151 const struct rte_flow_item_eth *spec = item->spec;
152 const struct rte_flow_item_eth *mask = item->mask;
153 struct ibv_flow_spec_eth *eth;
154 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
157 ++flow->ibv_attr->num_of_specs;
158 eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
159 *eth = (struct ibv_flow_spec_eth) {
160 .type = IBV_FLOW_SPEC_ETH,
164 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
169 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
170 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
171 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
172 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
173 /* Remove unwanted bits from values. */
174 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
175 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
176 eth->val.src_mac[i] &= eth->mask.src_mac[i];
182 * Convert VLAN item to Verbs specification.
185 * Item specification.
186 * @param default_mask[in]
187 * Default bit-masks to use when item->mask is not provided.
188 * @param flow[in, out]
189 * Flow rule handle to update.
192 mlx4_flow_create_vlan(const struct rte_flow_item *item,
193 const void *default_mask,
194 struct rte_flow *flow)
196 const struct rte_flow_item_vlan *spec = item->spec;
197 const struct rte_flow_item_vlan *mask = item->mask;
198 struct ibv_flow_spec_eth *eth;
199 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
201 eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size -
207 eth->val.vlan_tag = spec->tci;
208 eth->mask.vlan_tag = mask->tci;
209 eth->val.vlan_tag &= eth->mask.vlan_tag;
214 * Convert IPv4 item to Verbs specification.
217 * Item specification.
218 * @param default_mask[in]
219 * Default bit-masks to use when item->mask is not provided.
220 * @param flow[in, out]
221 * Flow rule handle to update.
224 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
225 const void *default_mask,
226 struct rte_flow *flow)
228 const struct rte_flow_item_ipv4 *spec = item->spec;
229 const struct rte_flow_item_ipv4 *mask = item->mask;
230 struct ibv_flow_spec_ipv4 *ipv4;
231 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
233 ++flow->ibv_attr->num_of_specs;
234 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
235 *ipv4 = (struct ibv_flow_spec_ipv4) {
236 .type = IBV_FLOW_SPEC_IPV4,
241 ipv4->val = (struct ibv_flow_ipv4_filter) {
242 .src_ip = spec->hdr.src_addr,
243 .dst_ip = spec->hdr.dst_addr,
247 ipv4->mask = (struct ibv_flow_ipv4_filter) {
248 .src_ip = mask->hdr.src_addr,
249 .dst_ip = mask->hdr.dst_addr,
251 /* Remove unwanted bits from values. */
252 ipv4->val.src_ip &= ipv4->mask.src_ip;
253 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
258 * Convert UDP item to Verbs specification.
261 * Item specification.
262 * @param default_mask[in]
263 * Default bit-masks to use when item->mask is not provided.
264 * @param flow[in, out]
265 * Flow rule handle to update.
268 mlx4_flow_create_udp(const struct rte_flow_item *item,
269 const void *default_mask,
270 struct rte_flow *flow)
272 const struct rte_flow_item_udp *spec = item->spec;
273 const struct rte_flow_item_udp *mask = item->mask;
274 struct ibv_flow_spec_tcp_udp *udp;
275 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
277 ++flow->ibv_attr->num_of_specs;
278 udp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
279 *udp = (struct ibv_flow_spec_tcp_udp) {
280 .type = IBV_FLOW_SPEC_UDP,
285 udp->val.dst_port = spec->hdr.dst_port;
286 udp->val.src_port = spec->hdr.src_port;
289 udp->mask.dst_port = mask->hdr.dst_port;
290 udp->mask.src_port = mask->hdr.src_port;
291 /* Remove unwanted bits from values. */
292 udp->val.src_port &= udp->mask.src_port;
293 udp->val.dst_port &= udp->mask.dst_port;
298 * Convert TCP item to Verbs specification.
301 * Item specification.
302 * @param default_mask[in]
303 * Default bit-masks to use when item->mask is not provided.
304 * @param flow[in, out]
305 * Flow rule handle to update.
308 mlx4_flow_create_tcp(const struct rte_flow_item *item,
309 const void *default_mask,
310 struct rte_flow *flow)
312 const struct rte_flow_item_tcp *spec = item->spec;
313 const struct rte_flow_item_tcp *mask = item->mask;
314 struct ibv_flow_spec_tcp_udp *tcp;
315 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
317 ++flow->ibv_attr->num_of_specs;
318 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
319 *tcp = (struct ibv_flow_spec_tcp_udp) {
320 .type = IBV_FLOW_SPEC_TCP,
325 tcp->val.dst_port = spec->hdr.dst_port;
326 tcp->val.src_port = spec->hdr.src_port;
329 tcp->mask.dst_port = mask->hdr.dst_port;
330 tcp->mask.src_port = mask->hdr.src_port;
331 /* Remove unwanted bits from values. */
332 tcp->val.src_port &= tcp->mask.src_port;
333 tcp->val.dst_port &= tcp->mask.dst_port;
338 * Check support for a given item.
341 * Item specification.
343 * Bit-masks covering supported fields to compare with spec, last and mask in
346 * Bit-Mask size in bytes.
349 * 0 on success, negative value otherwise.
352 mlx4_flow_item_validate(const struct rte_flow_item *item,
353 const uint8_t *mask, unsigned int size)
357 if (!item->spec && (item->mask || item->last))
359 if (item->spec && !item->mask) {
361 const uint8_t *spec = item->spec;
363 for (i = 0; i < size; ++i)
364 if ((spec[i] | mask[i]) != mask[i])
367 if (item->last && !item->mask) {
369 const uint8_t *spec = item->last;
371 for (i = 0; i < size; ++i)
372 if ((spec[i] | mask[i]) != mask[i])
375 if (item->spec && item->last) {
378 const uint8_t *apply = mask;
383 for (i = 0; i < size; ++i) {
384 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
385 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
387 ret = memcmp(spec, last, size);
393 mlx4_flow_validate_eth(const struct rte_flow_item *item,
394 const uint8_t *mask, unsigned int size)
397 const struct rte_flow_item_eth *mask = item->mask;
399 if (mask->dst.addr_bytes[0] != 0xff ||
400 mask->dst.addr_bytes[1] != 0xff ||
401 mask->dst.addr_bytes[2] != 0xff ||
402 mask->dst.addr_bytes[3] != 0xff ||
403 mask->dst.addr_bytes[4] != 0xff ||
404 mask->dst.addr_bytes[5] != 0xff)
407 return mlx4_flow_item_validate(item, mask, size);
411 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
412 const uint8_t *mask, unsigned int size)
415 const struct rte_flow_item_vlan *mask = item->mask;
417 if (mask->tci != 0 &&
418 ntohs(mask->tci) != 0x0fff)
421 return mlx4_flow_item_validate(item, mask, size);
425 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
426 const uint8_t *mask, unsigned int size)
429 const struct rte_flow_item_ipv4 *mask = item->mask;
431 if (mask->hdr.src_addr != 0 &&
432 mask->hdr.src_addr != 0xffffffff)
434 if (mask->hdr.dst_addr != 0 &&
435 mask->hdr.dst_addr != 0xffffffff)
438 return mlx4_flow_item_validate(item, mask, size);
442 mlx4_flow_validate_udp(const struct rte_flow_item *item,
443 const uint8_t *mask, unsigned int size)
446 const struct rte_flow_item_udp *mask = item->mask;
448 if (mask->hdr.src_port != 0 &&
449 mask->hdr.src_port != 0xffff)
451 if (mask->hdr.dst_port != 0 &&
452 mask->hdr.dst_port != 0xffff)
455 return mlx4_flow_item_validate(item, mask, size);
459 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
460 const uint8_t *mask, unsigned int size)
463 const struct rte_flow_item_tcp *mask = item->mask;
465 if (mask->hdr.src_port != 0 &&
466 mask->hdr.src_port != 0xffff)
468 if (mask->hdr.dst_port != 0 &&
469 mask->hdr.dst_port != 0xffff)
472 return mlx4_flow_item_validate(item, mask, size);
475 /** Graph of supported items and associated actions. */
476 static const struct mlx4_flow_proc_item mlx4_flow_proc_item_list[] = {
477 [RTE_FLOW_ITEM_TYPE_END] = {
478 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_ETH),
480 [RTE_FLOW_ITEM_TYPE_ETH] = {
481 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_VLAN,
482 RTE_FLOW_ITEM_TYPE_IPV4),
483 .mask = &(const struct rte_flow_item_eth){
484 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
485 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
487 .default_mask = &rte_flow_item_eth_mask,
488 .mask_sz = sizeof(struct rte_flow_item_eth),
489 .validate = mlx4_flow_validate_eth,
490 .convert = mlx4_flow_create_eth,
491 .dst_sz = sizeof(struct ibv_flow_spec_eth),
493 [RTE_FLOW_ITEM_TYPE_VLAN] = {
494 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_IPV4),
495 .mask = &(const struct rte_flow_item_vlan){
496 /* Only TCI VID matching is supported. */
497 .tci = RTE_BE16(0x0fff),
499 .mask_sz = sizeof(struct rte_flow_item_vlan),
500 .validate = mlx4_flow_validate_vlan,
501 .convert = mlx4_flow_create_vlan,
504 [RTE_FLOW_ITEM_TYPE_IPV4] = {
505 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_UDP,
506 RTE_FLOW_ITEM_TYPE_TCP),
507 .mask = &(const struct rte_flow_item_ipv4){
509 .src_addr = RTE_BE32(0xffffffff),
510 .dst_addr = RTE_BE32(0xffffffff),
513 .default_mask = &rte_flow_item_ipv4_mask,
514 .mask_sz = sizeof(struct rte_flow_item_ipv4),
515 .validate = mlx4_flow_validate_ipv4,
516 .convert = mlx4_flow_create_ipv4,
517 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
519 [RTE_FLOW_ITEM_TYPE_UDP] = {
520 .mask = &(const struct rte_flow_item_udp){
522 .src_port = RTE_BE16(0xffff),
523 .dst_port = RTE_BE16(0xffff),
526 .default_mask = &rte_flow_item_udp_mask,
527 .mask_sz = sizeof(struct rte_flow_item_udp),
528 .validate = mlx4_flow_validate_udp,
529 .convert = mlx4_flow_create_udp,
530 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
532 [RTE_FLOW_ITEM_TYPE_TCP] = {
533 .mask = &(const struct rte_flow_item_tcp){
535 .src_port = RTE_BE16(0xffff),
536 .dst_port = RTE_BE16(0xffff),
539 .default_mask = &rte_flow_item_tcp_mask,
540 .mask_sz = sizeof(struct rte_flow_item_tcp),
541 .validate = mlx4_flow_validate_tcp,
542 .convert = mlx4_flow_create_tcp,
543 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
548 * Make sure a flow rule is supported and initialize associated structure.
551 * Pointer to private structure.
553 * Flow rule attributes.
555 * Pattern specification (list terminated by the END pattern item).
557 * Associated actions (list terminated by the END action).
559 * Perform verbose error reporting if not NULL.
560 * @param[in, out] addr
561 * Buffer where the resulting flow rule handle pointer must be stored.
562 * If NULL, stop processing after validation stage.
565 * 0 on success, a negative errno value otherwise and rte_errno is set.
568 mlx4_flow_prepare(struct priv *priv,
569 const struct rte_flow_attr *attr,
570 const struct rte_flow_item pattern[],
571 const struct rte_flow_action actions[],
572 struct rte_flow_error *error,
573 struct rte_flow **addr)
575 const struct rte_flow_item *item;
576 const struct rte_flow_action *action;
577 const struct mlx4_flow_proc_item *proc;
578 struct rte_flow temp = { .ibv_attr_size = sizeof(*temp.ibv_attr) };
579 struct rte_flow *flow = &temp;
582 return rte_flow_error_set
583 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
584 NULL, "groups are not supported");
585 if (attr->priority > MLX4_FLOW_PRIORITY_LAST)
586 return rte_flow_error_set
587 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
588 NULL, "maximum priority level is "
589 MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST));
591 return rte_flow_error_set
592 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
593 NULL, "egress is not supported");
595 return rte_flow_error_set
596 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
597 NULL, "only ingress is supported");
599 proc = mlx4_flow_proc_item_list;
600 /* Go over pattern. */
601 for (item = pattern; item->type; ++item) {
602 const struct mlx4_flow_proc_item *next = NULL;
606 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
608 if (item->type == MLX4_FLOW_ITEM_TYPE_INTERNAL) {
613 * The nic can support patterns with NULL eth spec only
614 * if eth is a single item in a rule.
616 if (!item->spec && item->type == RTE_FLOW_ITEM_TYPE_ETH) {
617 const struct rte_flow_item *next = item + 1;
620 return rte_flow_error_set
622 RTE_FLOW_ERROR_TYPE_ITEM, item,
623 "the rule requires an Ethernet spec");
625 for (i = 0; proc->next_item && proc->next_item[i]; ++i) {
626 if (proc->next_item[i] == item->type) {
627 next = &mlx4_flow_proc_item_list[item->type];
632 goto exit_item_not_supported;
634 /* Perform validation once, while handle is not allocated. */
636 err = proc->validate(item, proc->mask, proc->mask_sz);
638 goto exit_item_not_supported;
639 } else if (proc->convert) {
640 err = proc->convert(item,
641 (proc->default_mask ?
646 goto exit_item_not_supported;
648 flow->ibv_attr_size += proc->dst_sz;
650 /* Go over actions list. */
651 for (action = actions; action->type; ++action) {
652 switch (action->type) {
653 const struct rte_flow_action_queue *queue;
655 case RTE_FLOW_ACTION_TYPE_VOID:
657 case RTE_FLOW_ACTION_TYPE_DROP:
660 case RTE_FLOW_ACTION_TYPE_QUEUE:
661 queue = action->conf;
662 if (queue->index >= priv->dev->data->nb_rx_queues)
663 goto exit_action_not_supported;
665 flow->queue_id = queue->index;
668 goto exit_action_not_supported;
671 if (!flow->queue && !flow->drop)
672 return rte_flow_error_set
673 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
674 NULL, "no valid action");
675 /* Validation ends here. */
679 /* Allocate proper handle based on collected data. */
680 const struct mlx4_malloc_vec vec[] = {
682 .align = alignof(struct rte_flow),
683 .size = sizeof(*flow),
684 .addr = (void **)&flow,
687 .align = alignof(struct ibv_flow_attr),
688 .size = temp.ibv_attr_size,
689 .addr = (void **)&temp.ibv_attr,
693 if (!mlx4_zmallocv(__func__, vec, RTE_DIM(vec)))
694 return rte_flow_error_set
696 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
697 "flow rule handle allocation failure");
698 /* Most fields will be updated by second pass. */
699 *flow = (struct rte_flow){
700 .ibv_attr = temp.ibv_attr,
701 .ibv_attr_size = sizeof(*flow->ibv_attr),
703 *flow->ibv_attr = (struct ibv_flow_attr){
704 .type = IBV_FLOW_ATTR_NORMAL,
705 .size = sizeof(*flow->ibv_attr),
706 .priority = attr->priority,
713 exit_item_not_supported:
714 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
715 item, "item not supported");
716 exit_action_not_supported:
717 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
718 action, "action not supported");
722 * Validate a flow supported by the NIC.
724 * @see rte_flow_validate()
728 mlx4_flow_validate(struct rte_eth_dev *dev,
729 const struct rte_flow_attr *attr,
730 const struct rte_flow_item pattern[],
731 const struct rte_flow_action actions[],
732 struct rte_flow_error *error)
734 struct priv *priv = dev->data->dev_private;
736 return mlx4_flow_prepare(priv, attr, pattern, actions, error, NULL);
740 * Get a drop flow rule resources instance.
743 * Pointer to private structure.
746 * Pointer to drop flow resources on success, NULL otherwise and rte_errno
749 static struct mlx4_drop *
750 mlx4_drop_get(struct priv *priv)
752 struct mlx4_drop *drop = priv->drop;
755 assert(drop->refcnt);
756 assert(drop->priv == priv);
760 drop = rte_malloc(__func__, sizeof(*drop), 0);
763 *drop = (struct mlx4_drop){
767 drop->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
770 drop->qp = ibv_create_qp(priv->pd,
771 &(struct ibv_qp_init_attr){
774 .qp_type = IBV_QPT_RAW_PACKET,
782 claim_zero(ibv_destroy_qp(drop->qp));
784 claim_zero(ibv_destroy_cq(drop->cq));
792 * Give back a drop flow rule resources instance.
795 * Pointer to drop flow rule resources.
798 mlx4_drop_put(struct mlx4_drop *drop)
800 assert(drop->refcnt);
803 drop->priv->drop = NULL;
804 claim_zero(ibv_destroy_qp(drop->qp));
805 claim_zero(ibv_destroy_cq(drop->cq));
810 * Toggle a configured flow rule.
813 * Pointer to private structure.
815 * Flow rule handle to toggle.
817 * Whether associated Verbs flow must be created or removed.
819 * Perform verbose error reporting if not NULL.
822 * 0 on success, a negative errno value otherwise and rte_errno is set.
825 mlx4_flow_toggle(struct priv *priv,
826 struct rte_flow *flow,
828 struct rte_flow_error *error)
830 struct ibv_qp *qp = NULL;
837 claim_zero(ibv_destroy_flow(flow->ibv_flow));
838 flow->ibv_flow = NULL;
840 mlx4_drop_put(priv->drop);
843 assert(flow->ibv_attr);
844 if (!flow->internal &&
846 flow->ibv_attr->priority == MLX4_FLOW_PRIORITY_LAST) {
847 if (flow->ibv_flow) {
848 claim_zero(ibv_destroy_flow(flow->ibv_flow));
849 flow->ibv_flow = NULL;
851 mlx4_drop_put(priv->drop);
854 msg = ("priority level "
855 MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST)
856 " is reserved when not in isolated mode");
860 struct rxq *rxq = NULL;
862 if (flow->queue_id < priv->dev->data->nb_rx_queues)
863 rxq = priv->dev->data->rx_queues[flow->queue_id];
864 if (flow->ibv_flow) {
865 if (!rxq ^ !flow->drop)
867 /* Verbs flow needs updating. */
868 claim_zero(ibv_destroy_flow(flow->ibv_flow));
869 flow->ibv_flow = NULL;
871 mlx4_drop_put(priv->drop);
875 /* A missing target queue drops traffic implicitly. */
882 msg = "resources for drop flow rule cannot be created";
890 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
894 mlx4_drop_put(priv->drop);
896 msg = "flow rule rejected by device";
898 return rte_flow_error_set
899 (error, err, RTE_FLOW_ERROR_TYPE_HANDLE, flow, msg);
905 * @see rte_flow_create()
908 static struct rte_flow *
909 mlx4_flow_create(struct rte_eth_dev *dev,
910 const struct rte_flow_attr *attr,
911 const struct rte_flow_item pattern[],
912 const struct rte_flow_action actions[],
913 struct rte_flow_error *error)
915 struct priv *priv = dev->data->dev_private;
916 struct rte_flow *flow;
919 err = mlx4_flow_prepare(priv, attr, pattern, actions, error, &flow);
922 err = mlx4_flow_toggle(priv, flow, priv->started, error);
924 struct rte_flow *curr = LIST_FIRST(&priv->flows);
926 /* New rules are inserted after internal ones. */
927 if (!curr || !curr->internal) {
928 LIST_INSERT_HEAD(&priv->flows, flow, next);
930 while (LIST_NEXT(curr, next) &&
931 LIST_NEXT(curr, next)->internal)
932 curr = LIST_NEXT(curr, next);
933 LIST_INSERT_AFTER(curr, flow, next);
937 rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
944 * Configure isolated mode.
946 * @see rte_flow_isolate()
950 mlx4_flow_isolate(struct rte_eth_dev *dev,
952 struct rte_flow_error *error)
954 struct priv *priv = dev->data->dev_private;
956 if (!!enable == !!priv->isolated)
958 priv->isolated = !!enable;
959 if (mlx4_flow_sync(priv, error)) {
960 priv->isolated = !enable;
967 * Destroy a flow rule.
969 * @see rte_flow_destroy()
973 mlx4_flow_destroy(struct rte_eth_dev *dev,
974 struct rte_flow *flow,
975 struct rte_flow_error *error)
977 struct priv *priv = dev->data->dev_private;
978 int err = mlx4_flow_toggle(priv, flow, 0, error);
982 LIST_REMOVE(flow, next);
988 * Destroy user-configured flow rules.
990 * This function skips internal flows rules.
992 * @see rte_flow_flush()
996 mlx4_flow_flush(struct rte_eth_dev *dev,
997 struct rte_flow_error *error)
999 struct priv *priv = dev->data->dev_private;
1000 struct rte_flow *flow = LIST_FIRST(&priv->flows);
1003 struct rte_flow *next = LIST_NEXT(flow, next);
1005 if (!flow->internal)
1006 mlx4_flow_destroy(dev, flow, error);
1013 * Generate internal flow rules.
1016 * Pointer to private structure.
1018 * Perform verbose error reporting if not NULL.
1021 * 0 on success, a negative errno value otherwise and rte_errno is set.
1024 mlx4_flow_internal(struct priv *priv, struct rte_flow_error *error)
1026 struct rte_flow_attr attr = {
1027 .priority = MLX4_FLOW_PRIORITY_LAST,
1030 struct rte_flow_item pattern[] = {
1032 .type = MLX4_FLOW_ITEM_TYPE_INTERNAL,
1035 .type = RTE_FLOW_ITEM_TYPE_ETH,
1036 .spec = &(struct rte_flow_item_eth){
1039 .mask = &(struct rte_flow_item_eth){
1040 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1044 .type = RTE_FLOW_ITEM_TYPE_END,
1047 struct rte_flow_action actions[] = {
1049 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
1050 .conf = &(struct rte_flow_action_queue){
1055 .type = RTE_FLOW_ACTION_TYPE_END,
1059 if (!mlx4_flow_create(priv->dev, &attr, pattern, actions, error))
1065 * Synchronize flow rules.
1067 * This function synchronizes flow rules with the state of the device by
1068 * taking into account isolated mode and whether target queues are
1072 * Pointer to private structure.
1074 * Perform verbose error reporting if not NULL.
1077 * 0 on success, a negative errno value otherwise and rte_errno is set.
1080 mlx4_flow_sync(struct priv *priv, struct rte_flow_error *error)
1082 struct rte_flow *flow;
1085 /* Internal flow rules are guaranteed to come first in the list. */
1086 if (priv->isolated) {
1088 * Get rid of them in isolated mode, stop at the first
1089 * non-internal rule found.
1091 for (flow = LIST_FIRST(&priv->flows);
1092 flow && flow->internal;
1093 flow = LIST_FIRST(&priv->flows))
1094 claim_zero(mlx4_flow_destroy(priv->dev, flow, error));
1095 } else if (!LIST_FIRST(&priv->flows) ||
1096 !LIST_FIRST(&priv->flows)->internal) {
1098 * If the first rule is not internal outside isolated mode,
1099 * they must be added back.
1101 ret = mlx4_flow_internal(priv, error);
1105 /* Toggle the remaining flow rules . */
1106 for (flow = LIST_FIRST(&priv->flows);
1108 flow = LIST_NEXT(flow, next)) {
1109 ret = mlx4_flow_toggle(priv, flow, priv->started, error);
1114 assert(!priv->drop);
1119 * Clean up all flow rules.
1121 * Unlike mlx4_flow_flush(), this function takes care of all remaining flow
1122 * rules regardless of whether they are internal or user-configured.
1125 * Pointer to private structure.
1128 mlx4_flow_clean(struct priv *priv)
1130 struct rte_flow *flow;
1132 while ((flow = LIST_FIRST(&priv->flows)))
1133 mlx4_flow_destroy(priv->dev, flow, NULL);
1136 static const struct rte_flow_ops mlx4_flow_ops = {
1137 .validate = mlx4_flow_validate,
1138 .create = mlx4_flow_create,
1139 .destroy = mlx4_flow_destroy,
1140 .flush = mlx4_flow_flush,
1141 .isolate = mlx4_flow_isolate,
1145 * Manage filter operations.
1148 * Pointer to Ethernet device structure.
1149 * @param filter_type
1152 * Operation to perform.
1154 * Pointer to operation-specific structure.
1157 * 0 on success, negative errno value otherwise and rte_errno is set.
1160 mlx4_filter_ctrl(struct rte_eth_dev *dev,
1161 enum rte_filter_type filter_type,
1162 enum rte_filter_op filter_op,
1165 switch (filter_type) {
1166 case RTE_ETH_FILTER_GENERIC:
1167 if (filter_op != RTE_ETH_FILTER_GET)
1169 *(const void **)arg = &mlx4_flow_ops;
1172 ERROR("%p: filter type (%d) not supported",
1173 (void *)dev, filter_type);
1176 rte_errno = ENOTSUP;