4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Flow API operations for mlx4 driver.
39 #include <arpa/inet.h>
46 #include <sys/queue.h>
48 /* Verbs headers do not support -pedantic. */
50 #pragma GCC diagnostic ignored "-Wpedantic"
52 #include <infiniband/verbs.h>
54 #pragma GCC diagnostic error "-Wpedantic"
57 #include <rte_byteorder.h>
58 #include <rte_errno.h>
59 #include <rte_eth_ctrl.h>
60 #include <rte_ethdev.h>
62 #include <rte_flow_driver.h>
63 #include <rte_malloc.h>
67 #include "mlx4_flow.h"
68 #include "mlx4_rxtx.h"
69 #include "mlx4_utils.h"
71 /** Static initializer for a list of subsequent item types. */
72 #define NEXT_ITEM(...) \
73 (const enum rte_flow_item_type []){ \
74 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
77 /** Processor structure associated with a flow item. */
78 struct mlx4_flow_proc_item {
79 /** Bit-masks corresponding to the possibilities for the item. */
82 * Default bit-masks to use when item->mask is not provided. When
83 * \default_mask is also NULL, the full supported bit-mask (\mask) is
86 const void *default_mask;
87 /** Bit-masks size in bytes. */
88 const unsigned int mask_sz;
90 * Check support for a given item.
95 * Bit-masks covering supported fields to compare with spec,
99 * Bit-Mask size in bytes.
102 * 0 on success, negative value otherwise.
104 int (*validate)(const struct rte_flow_item *item,
105 const uint8_t *mask, unsigned int size);
107 * Conversion function from rte_flow to NIC specific flow.
110 * rte_flow item to convert.
111 * @param default_mask
112 * Default bit-masks to use when item->mask is not provided.
114 * Flow rule handle to update.
117 * 0 on success, negative value otherwise.
119 int (*convert)(const struct rte_flow_item *item,
120 const void *default_mask,
121 struct rte_flow *flow);
122 /** Size in bytes of the destination structure. */
123 const unsigned int dst_sz;
124 /** List of possible subsequent items. */
125 const enum rte_flow_item_type *const next_item;
128 struct rte_flow_drop {
129 struct ibv_qp *qp; /**< Verbs queue pair. */
130 struct ibv_cq *cq; /**< Verbs completion queue. */
134 * Convert Ethernet item to Verbs specification.
137 * Item specification.
138 * @param default_mask[in]
139 * Default bit-masks to use when item->mask is not provided.
140 * @param flow[in, out]
141 * Flow rule handle to update.
144 mlx4_flow_create_eth(const struct rte_flow_item *item,
145 const void *default_mask,
146 struct rte_flow *flow)
148 const struct rte_flow_item_eth *spec = item->spec;
149 const struct rte_flow_item_eth *mask = item->mask;
150 struct ibv_flow_spec_eth *eth;
151 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
154 ++flow->ibv_attr->num_of_specs;
155 flow->ibv_attr->priority = 2;
156 eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
157 *eth = (struct ibv_flow_spec_eth) {
158 .type = IBV_FLOW_SPEC_ETH,
162 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
167 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
168 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
169 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
170 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
171 /* Remove unwanted bits from values. */
172 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
173 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
174 eth->val.src_mac[i] &= eth->mask.src_mac[i];
180 * Convert VLAN item to Verbs specification.
183 * Item specification.
184 * @param default_mask[in]
185 * Default bit-masks to use when item->mask is not provided.
186 * @param flow[in, out]
187 * Flow rule handle to update.
190 mlx4_flow_create_vlan(const struct rte_flow_item *item,
191 const void *default_mask,
192 struct rte_flow *flow)
194 const struct rte_flow_item_vlan *spec = item->spec;
195 const struct rte_flow_item_vlan *mask = item->mask;
196 struct ibv_flow_spec_eth *eth;
197 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
199 eth = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size -
205 eth->val.vlan_tag = spec->tci;
206 eth->mask.vlan_tag = mask->tci;
207 eth->val.vlan_tag &= eth->mask.vlan_tag;
212 * Convert IPv4 item to Verbs specification.
215 * Item specification.
216 * @param default_mask[in]
217 * Default bit-masks to use when item->mask is not provided.
218 * @param flow[in, out]
219 * Flow rule handle to update.
222 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
223 const void *default_mask,
224 struct rte_flow *flow)
226 const struct rte_flow_item_ipv4 *spec = item->spec;
227 const struct rte_flow_item_ipv4 *mask = item->mask;
228 struct ibv_flow_spec_ipv4 *ipv4;
229 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
231 ++flow->ibv_attr->num_of_specs;
232 flow->ibv_attr->priority = 1;
233 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
234 *ipv4 = (struct ibv_flow_spec_ipv4) {
235 .type = IBV_FLOW_SPEC_IPV4,
240 ipv4->val = (struct ibv_flow_ipv4_filter) {
241 .src_ip = spec->hdr.src_addr,
242 .dst_ip = spec->hdr.dst_addr,
246 ipv4->mask = (struct ibv_flow_ipv4_filter) {
247 .src_ip = mask->hdr.src_addr,
248 .dst_ip = mask->hdr.dst_addr,
250 /* Remove unwanted bits from values. */
251 ipv4->val.src_ip &= ipv4->mask.src_ip;
252 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
257 * Convert UDP item to Verbs specification.
260 * Item specification.
261 * @param default_mask[in]
262 * Default bit-masks to use when item->mask is not provided.
263 * @param flow[in, out]
264 * Flow rule handle to update.
267 mlx4_flow_create_udp(const struct rte_flow_item *item,
268 const void *default_mask,
269 struct rte_flow *flow)
271 const struct rte_flow_item_udp *spec = item->spec;
272 const struct rte_flow_item_udp *mask = item->mask;
273 struct ibv_flow_spec_tcp_udp *udp;
274 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
276 ++flow->ibv_attr->num_of_specs;
277 flow->ibv_attr->priority = 0;
278 udp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
279 *udp = (struct ibv_flow_spec_tcp_udp) {
280 .type = IBV_FLOW_SPEC_UDP,
285 udp->val.dst_port = spec->hdr.dst_port;
286 udp->val.src_port = spec->hdr.src_port;
289 udp->mask.dst_port = mask->hdr.dst_port;
290 udp->mask.src_port = mask->hdr.src_port;
291 /* Remove unwanted bits from values. */
292 udp->val.src_port &= udp->mask.src_port;
293 udp->val.dst_port &= udp->mask.dst_port;
298 * Convert TCP item to Verbs specification.
301 * Item specification.
302 * @param default_mask[in]
303 * Default bit-masks to use when item->mask is not provided.
304 * @param flow[in, out]
305 * Flow rule handle to update.
308 mlx4_flow_create_tcp(const struct rte_flow_item *item,
309 const void *default_mask,
310 struct rte_flow *flow)
312 const struct rte_flow_item_tcp *spec = item->spec;
313 const struct rte_flow_item_tcp *mask = item->mask;
314 struct ibv_flow_spec_tcp_udp *tcp;
315 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
317 ++flow->ibv_attr->num_of_specs;
318 flow->ibv_attr->priority = 0;
319 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->ibv_attr_size);
320 *tcp = (struct ibv_flow_spec_tcp_udp) {
321 .type = IBV_FLOW_SPEC_TCP,
326 tcp->val.dst_port = spec->hdr.dst_port;
327 tcp->val.src_port = spec->hdr.src_port;
330 tcp->mask.dst_port = mask->hdr.dst_port;
331 tcp->mask.src_port = mask->hdr.src_port;
332 /* Remove unwanted bits from values. */
333 tcp->val.src_port &= tcp->mask.src_port;
334 tcp->val.dst_port &= tcp->mask.dst_port;
339 * Check support for a given item.
342 * Item specification.
344 * Bit-masks covering supported fields to compare with spec, last and mask in
347 * Bit-Mask size in bytes.
350 * 0 on success, negative value otherwise.
353 mlx4_flow_item_validate(const struct rte_flow_item *item,
354 const uint8_t *mask, unsigned int size)
358 if (!item->spec && (item->mask || item->last))
360 if (item->spec && !item->mask) {
362 const uint8_t *spec = item->spec;
364 for (i = 0; i < size; ++i)
365 if ((spec[i] | mask[i]) != mask[i])
368 if (item->last && !item->mask) {
370 const uint8_t *spec = item->last;
372 for (i = 0; i < size; ++i)
373 if ((spec[i] | mask[i]) != mask[i])
376 if (item->spec && item->last) {
379 const uint8_t *apply = mask;
384 for (i = 0; i < size; ++i) {
385 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
386 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
388 ret = memcmp(spec, last, size);
394 mlx4_flow_validate_eth(const struct rte_flow_item *item,
395 const uint8_t *mask, unsigned int size)
398 const struct rte_flow_item_eth *mask = item->mask;
400 if (mask->dst.addr_bytes[0] != 0xff ||
401 mask->dst.addr_bytes[1] != 0xff ||
402 mask->dst.addr_bytes[2] != 0xff ||
403 mask->dst.addr_bytes[3] != 0xff ||
404 mask->dst.addr_bytes[4] != 0xff ||
405 mask->dst.addr_bytes[5] != 0xff)
408 return mlx4_flow_item_validate(item, mask, size);
412 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
413 const uint8_t *mask, unsigned int size)
416 const struct rte_flow_item_vlan *mask = item->mask;
418 if (mask->tci != 0 &&
419 ntohs(mask->tci) != 0x0fff)
422 return mlx4_flow_item_validate(item, mask, size);
426 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
427 const uint8_t *mask, unsigned int size)
430 const struct rte_flow_item_ipv4 *mask = item->mask;
432 if (mask->hdr.src_addr != 0 &&
433 mask->hdr.src_addr != 0xffffffff)
435 if (mask->hdr.dst_addr != 0 &&
436 mask->hdr.dst_addr != 0xffffffff)
439 return mlx4_flow_item_validate(item, mask, size);
443 mlx4_flow_validate_udp(const struct rte_flow_item *item,
444 const uint8_t *mask, unsigned int size)
447 const struct rte_flow_item_udp *mask = item->mask;
449 if (mask->hdr.src_port != 0 &&
450 mask->hdr.src_port != 0xffff)
452 if (mask->hdr.dst_port != 0 &&
453 mask->hdr.dst_port != 0xffff)
456 return mlx4_flow_item_validate(item, mask, size);
460 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
461 const uint8_t *mask, unsigned int size)
464 const struct rte_flow_item_tcp *mask = item->mask;
466 if (mask->hdr.src_port != 0 &&
467 mask->hdr.src_port != 0xffff)
469 if (mask->hdr.dst_port != 0 &&
470 mask->hdr.dst_port != 0xffff)
473 return mlx4_flow_item_validate(item, mask, size);
476 /** Graph of supported items and associated actions. */
477 static const struct mlx4_flow_proc_item mlx4_flow_proc_item_list[] = {
478 [RTE_FLOW_ITEM_TYPE_END] = {
479 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_ETH),
481 [RTE_FLOW_ITEM_TYPE_ETH] = {
482 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_VLAN,
483 RTE_FLOW_ITEM_TYPE_IPV4),
484 .mask = &(const struct rte_flow_item_eth){
485 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
486 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
488 .default_mask = &rte_flow_item_eth_mask,
489 .mask_sz = sizeof(struct rte_flow_item_eth),
490 .validate = mlx4_flow_validate_eth,
491 .convert = mlx4_flow_create_eth,
492 .dst_sz = sizeof(struct ibv_flow_spec_eth),
494 [RTE_FLOW_ITEM_TYPE_VLAN] = {
495 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_IPV4),
496 .mask = &(const struct rte_flow_item_vlan){
497 /* Only TCI VID matching is supported. */
498 .tci = RTE_BE16(0x0fff),
500 .mask_sz = sizeof(struct rte_flow_item_vlan),
501 .validate = mlx4_flow_validate_vlan,
502 .convert = mlx4_flow_create_vlan,
505 [RTE_FLOW_ITEM_TYPE_IPV4] = {
506 .next_item = NEXT_ITEM(RTE_FLOW_ITEM_TYPE_UDP,
507 RTE_FLOW_ITEM_TYPE_TCP),
508 .mask = &(const struct rte_flow_item_ipv4){
510 .src_addr = RTE_BE32(0xffffffff),
511 .dst_addr = RTE_BE32(0xffffffff),
514 .default_mask = &rte_flow_item_ipv4_mask,
515 .mask_sz = sizeof(struct rte_flow_item_ipv4),
516 .validate = mlx4_flow_validate_ipv4,
517 .convert = mlx4_flow_create_ipv4,
518 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
520 [RTE_FLOW_ITEM_TYPE_UDP] = {
521 .mask = &(const struct rte_flow_item_udp){
523 .src_port = RTE_BE16(0xffff),
524 .dst_port = RTE_BE16(0xffff),
527 .default_mask = &rte_flow_item_udp_mask,
528 .mask_sz = sizeof(struct rte_flow_item_udp),
529 .validate = mlx4_flow_validate_udp,
530 .convert = mlx4_flow_create_udp,
531 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
533 [RTE_FLOW_ITEM_TYPE_TCP] = {
534 .mask = &(const struct rte_flow_item_tcp){
536 .src_port = RTE_BE16(0xffff),
537 .dst_port = RTE_BE16(0xffff),
540 .default_mask = &rte_flow_item_tcp_mask,
541 .mask_sz = sizeof(struct rte_flow_item_tcp),
542 .validate = mlx4_flow_validate_tcp,
543 .convert = mlx4_flow_create_tcp,
544 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
549 * Make sure a flow rule is supported and initialize associated structure.
552 * Pointer to private structure.
554 * Flow rule attributes.
556 * Pattern specification (list terminated by the END pattern item).
558 * Associated actions (list terminated by the END action).
560 * Perform verbose error reporting if not NULL.
561 * @param[in, out] addr
562 * Buffer where the resulting flow rule handle pointer must be stored.
563 * If NULL, stop processing after validation stage.
566 * 0 on success, a negative errno value otherwise and rte_errno is set.
569 mlx4_flow_prepare(struct priv *priv,
570 const struct rte_flow_attr *attr,
571 const struct rte_flow_item pattern[],
572 const struct rte_flow_action actions[],
573 struct rte_flow_error *error,
574 struct rte_flow **addr)
576 const struct rte_flow_item *item;
577 const struct rte_flow_action *action;
578 const struct mlx4_flow_proc_item *proc;
579 struct rte_flow temp = { .ibv_attr_size = sizeof(*temp.ibv_attr) };
580 struct rte_flow *flow = &temp;
581 uint32_t priority_override = 0;
584 return rte_flow_error_set
585 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
586 NULL, "groups are not supported");
588 priority_override = attr->priority;
589 else if (attr->priority)
590 return rte_flow_error_set
591 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
593 "priorities are not supported outside isolated mode");
594 if (attr->priority > MLX4_FLOW_PRIORITY_LAST)
595 return rte_flow_error_set
596 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
597 NULL, "maximum priority level is "
598 MLX4_STR_EXPAND(MLX4_FLOW_PRIORITY_LAST));
600 return rte_flow_error_set
601 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
602 NULL, "egress is not supported");
604 return rte_flow_error_set
605 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
606 NULL, "only ingress is supported");
608 proc = mlx4_flow_proc_item_list;
609 /* Go over pattern. */
610 for (item = pattern; item->type; ++item) {
611 const struct mlx4_flow_proc_item *next = NULL;
615 if (item->type == RTE_FLOW_ITEM_TYPE_VOID)
618 * The nic can support patterns with NULL eth spec only
619 * if eth is a single item in a rule.
621 if (!item->spec && item->type == RTE_FLOW_ITEM_TYPE_ETH) {
622 const struct rte_flow_item *next = item + 1;
625 return rte_flow_error_set
627 RTE_FLOW_ERROR_TYPE_ITEM, item,
628 "the rule requires an Ethernet spec");
630 for (i = 0; proc->next_item && proc->next_item[i]; ++i) {
631 if (proc->next_item[i] == item->type) {
632 next = &mlx4_flow_proc_item_list[item->type];
637 goto exit_item_not_supported;
639 /* Perform validation once, while handle is not allocated. */
641 err = proc->validate(item, proc->mask, proc->mask_sz);
643 goto exit_item_not_supported;
644 } else if (proc->convert) {
645 err = proc->convert(item,
646 (proc->default_mask ?
651 goto exit_item_not_supported;
653 flow->ibv_attr_size += proc->dst_sz;
655 /* Use specified priority level when in isolated mode. */
656 if (priv->isolated && flow != &temp)
657 flow->ibv_attr->priority = priority_override;
658 /* Go over actions list. */
659 for (action = actions; action->type; ++action) {
660 switch (action->type) {
661 const struct rte_flow_action_queue *queue;
663 case RTE_FLOW_ACTION_TYPE_VOID:
665 case RTE_FLOW_ACTION_TYPE_DROP:
668 case RTE_FLOW_ACTION_TYPE_QUEUE:
669 queue = action->conf;
670 if (queue->index >= priv->dev->data->nb_rx_queues)
671 goto exit_action_not_supported;
673 flow->queue_id = queue->index;
676 goto exit_action_not_supported;
679 if (!flow->queue && !flow->drop)
680 return rte_flow_error_set
681 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
682 NULL, "no valid action");
683 /* Validation ends here. */
687 /* Allocate proper handle based on collected data. */
688 const struct mlx4_malloc_vec vec[] = {
690 .align = alignof(struct rte_flow),
691 .size = sizeof(*flow),
692 .addr = (void **)&flow,
695 .align = alignof(struct ibv_flow_attr),
696 .size = temp.ibv_attr_size,
697 .addr = (void **)&temp.ibv_attr,
701 if (!mlx4_zmallocv(__func__, vec, RTE_DIM(vec)))
702 return rte_flow_error_set
704 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
705 "flow rule handle allocation failure");
706 /* Most fields will be updated by second pass. */
707 *flow = (struct rte_flow){
708 .ibv_attr = temp.ibv_attr,
709 .ibv_attr_size = sizeof(*flow->ibv_attr),
711 *flow->ibv_attr = (struct ibv_flow_attr){
712 .type = IBV_FLOW_ATTR_NORMAL,
713 .size = sizeof(*flow->ibv_attr),
720 exit_item_not_supported:
721 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
722 item, "item not supported");
723 exit_action_not_supported:
724 return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
725 action, "action not supported");
729 * Validate a flow supported by the NIC.
731 * @see rte_flow_validate()
735 mlx4_flow_validate(struct rte_eth_dev *dev,
736 const struct rte_flow_attr *attr,
737 const struct rte_flow_item pattern[],
738 const struct rte_flow_action actions[],
739 struct rte_flow_error *error)
741 struct priv *priv = dev->data->dev_private;
743 return mlx4_flow_prepare(priv, attr, pattern, actions, error, NULL);
747 * Destroy a drop queue.
750 * Pointer to private structure.
753 mlx4_flow_destroy_drop_queue(struct priv *priv)
755 if (priv->flow_drop_queue) {
756 struct rte_flow_drop *fdq = priv->flow_drop_queue;
758 priv->flow_drop_queue = NULL;
759 claim_zero(ibv_destroy_qp(fdq->qp));
760 claim_zero(ibv_destroy_cq(fdq->cq));
766 * Create a single drop queue for all drop flows.
769 * Pointer to private structure.
772 * 0 on success, negative value otherwise.
775 mlx4_flow_create_drop_queue(struct priv *priv)
779 struct rte_flow_drop *fdq;
781 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
783 ERROR("Cannot allocate memory for drop struct");
786 cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
788 ERROR("Cannot create drop CQ");
791 qp = ibv_create_qp(priv->pd,
792 &(struct ibv_qp_init_attr){
799 .qp_type = IBV_QPT_RAW_PACKET,
802 ERROR("Cannot create drop QP");
805 *fdq = (struct rte_flow_drop){
809 priv->flow_drop_queue = fdq;
812 claim_zero(ibv_destroy_cq(cq));
820 * Toggle a configured flow rule.
823 * Pointer to private structure.
825 * Flow rule handle to toggle.
827 * Whether associated Verbs flow must be created or removed.
829 * Perform verbose error reporting if not NULL.
832 * 0 on success, a negative errno value otherwise and rte_errno is set.
835 mlx4_flow_toggle(struct priv *priv,
836 struct rte_flow *flow,
838 struct rte_flow_error *error)
840 struct ibv_qp *qp = NULL;
847 claim_zero(ibv_destroy_flow(flow->ibv_flow));
848 flow->ibv_flow = NULL;
853 assert(flow->queue ^ flow->drop);
857 assert(flow->queue_id < priv->dev->data->nb_rx_queues);
858 rxq = priv->dev->data->rx_queues[flow->queue_id];
861 msg = "target queue must be configured first";
867 assert(priv->flow_drop_queue);
868 qp = priv->flow_drop_queue->qp;
871 assert(flow->ibv_attr);
872 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
876 msg = "flow rule rejected by device";
878 return rte_flow_error_set
879 (error, err, RTE_FLOW_ERROR_TYPE_HANDLE, flow, msg);
885 * @see rte_flow_create()
888 static struct rte_flow *
889 mlx4_flow_create(struct rte_eth_dev *dev,
890 const struct rte_flow_attr *attr,
891 const struct rte_flow_item pattern[],
892 const struct rte_flow_action actions[],
893 struct rte_flow_error *error)
895 struct priv *priv = dev->data->dev_private;
896 struct rte_flow *flow;
899 err = mlx4_flow_prepare(priv, attr, pattern, actions, error, &flow);
902 err = mlx4_flow_toggle(priv, flow, priv->started, error);
904 LIST_INSERT_HEAD(&priv->flows, flow, next);
907 rte_flow_error_set(error, -err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
914 * Configure isolated mode.
916 * @see rte_flow_isolate()
920 mlx4_flow_isolate(struct rte_eth_dev *dev,
922 struct rte_flow_error *error)
924 struct priv *priv = dev->data->dev_private;
926 if (!!enable == !!priv->isolated)
928 priv->isolated = !!enable;
930 mlx4_mac_addr_del(priv);
931 } else if (mlx4_mac_addr_add(priv) < 0) {
933 return rte_flow_error_set(error, rte_errno,
934 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
935 NULL, "cannot leave isolated mode");
941 * Destroy a flow rule.
943 * @see rte_flow_destroy()
947 mlx4_flow_destroy(struct rte_eth_dev *dev,
948 struct rte_flow *flow,
949 struct rte_flow_error *error)
951 struct priv *priv = dev->data->dev_private;
952 int err = mlx4_flow_toggle(priv, flow, 0, error);
956 LIST_REMOVE(flow, next);
962 * Destroy all flow rules.
964 * @see rte_flow_flush()
968 mlx4_flow_flush(struct rte_eth_dev *dev,
969 struct rte_flow_error *error)
971 struct priv *priv = dev->data->dev_private;
973 while (!LIST_EMPTY(&priv->flows)) {
974 struct rte_flow *flow;
976 flow = LIST_FIRST(&priv->flows);
977 mlx4_flow_destroy(dev, flow, error);
983 * Disable flow rules.
986 * Pointer to private structure.
989 mlx4_flow_stop(struct priv *priv)
991 struct rte_flow *flow;
993 for (flow = LIST_FIRST(&priv->flows);
995 flow = LIST_NEXT(flow, next)) {
996 claim_zero(mlx4_flow_toggle(priv, flow, 0, NULL));
998 mlx4_flow_destroy_drop_queue(priv);
1002 * Enable flow rules.
1005 * Pointer to private structure.
1008 * 0 on success, a negative errno value otherwise and rte_errno is set.
1011 mlx4_flow_start(struct priv *priv)
1014 struct rte_flow *flow;
1016 ret = mlx4_flow_create_drop_queue(priv);
1019 for (flow = LIST_FIRST(&priv->flows);
1021 flow = LIST_NEXT(flow, next)) {
1022 ret = mlx4_flow_toggle(priv, flow, 1, NULL);
1023 if (unlikely(ret)) {
1024 mlx4_flow_stop(priv);
1031 static const struct rte_flow_ops mlx4_flow_ops = {
1032 .validate = mlx4_flow_validate,
1033 .create = mlx4_flow_create,
1034 .destroy = mlx4_flow_destroy,
1035 .flush = mlx4_flow_flush,
1036 .isolate = mlx4_flow_isolate,
1040 * Manage filter operations.
1043 * Pointer to Ethernet device structure.
1044 * @param filter_type
1047 * Operation to perform.
1049 * Pointer to operation-specific structure.
1052 * 0 on success, negative errno value otherwise and rte_errno is set.
1055 mlx4_filter_ctrl(struct rte_eth_dev *dev,
1056 enum rte_filter_type filter_type,
1057 enum rte_filter_op filter_op,
1060 switch (filter_type) {
1061 case RTE_ETH_FILTER_GENERIC:
1062 if (filter_op != RTE_ETH_FILTER_GET)
1064 *(const void **)arg = &mlx4_flow_ops;
1067 ERROR("%p: filter type (%d) not supported",
1068 (void *)dev, filter_type);
1071 rte_errno = ENOTSUP;