4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <arpa/inet.h>
40 #include <sys/queue.h>
42 /* Verbs headers do not support -pedantic. */
44 #pragma GCC diagnostic ignored "-Wpedantic"
46 #include <infiniband/verbs.h>
48 #pragma GCC diagnostic error "-Wpedantic"
51 #include <rte_errno.h>
52 #include <rte_eth_ctrl.h>
53 #include <rte_ethdev.h>
55 #include <rte_flow_driver.h>
56 #include <rte_malloc.h>
60 #include "mlx4_flow.h"
61 #include "mlx4_rxtx.h"
62 #include "mlx4_utils.h"
64 /** Static initializer for items. */
66 (const enum rte_flow_item_type []){ \
67 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
70 /** Structure to generate a simple graph of layers supported by the NIC. */
71 struct mlx4_flow_items {
72 /** List of possible actions for these items. */
73 const enum rte_flow_action_type *const actions;
74 /** Bit-masks corresponding to the possibilities for the item. */
77 * Default bit-masks to use when item->mask is not provided. When
78 * \default_mask is also NULL, the full supported bit-mask (\mask) is
81 const void *default_mask;
82 /** Bit-masks size in bytes. */
83 const unsigned int mask_sz;
85 * Check support for a given item.
90 * Bit-masks covering supported fields to compare with spec,
94 * Bit-Mask size in bytes.
97 * 0 on success, negative value otherwise.
99 int (*validate)(const struct rte_flow_item *item,
100 const uint8_t *mask, unsigned int size);
102 * Conversion function from rte_flow to NIC specific flow.
105 * rte_flow item to convert.
106 * @param default_mask
107 * Default bit-masks to use when item->mask is not provided.
109 * Internal structure to store the conversion.
112 * 0 on success, negative value otherwise.
114 int (*convert)(const struct rte_flow_item *item,
115 const void *default_mask,
117 /** Size in bytes of the destination structure. */
118 const unsigned int dst_sz;
119 /** List of possible following items. */
120 const enum rte_flow_item_type *const items;
123 struct rte_flow_drop {
124 struct ibv_qp *qp; /**< Verbs queue pair. */
125 struct ibv_cq *cq; /**< Verbs completion queue. */
128 /** Valid action for this PMD. */
129 static const enum rte_flow_action_type valid_actions[] = {
130 RTE_FLOW_ACTION_TYPE_DROP,
131 RTE_FLOW_ACTION_TYPE_QUEUE,
132 RTE_FLOW_ACTION_TYPE_END,
136 * Convert Ethernet item to Verbs specification.
139 * Item specification.
140 * @param default_mask[in]
141 * Default bit-masks to use when item->mask is not provided.
142 * @param data[in, out]
146 mlx4_flow_create_eth(const struct rte_flow_item *item,
147 const void *default_mask,
150 const struct rte_flow_item_eth *spec = item->spec;
151 const struct rte_flow_item_eth *mask = item->mask;
152 struct mlx4_flow *flow = (struct mlx4_flow *)data;
153 struct ibv_flow_spec_eth *eth;
154 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
157 ++flow->ibv_attr->num_of_specs;
158 flow->ibv_attr->priority = 2;
159 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
160 *eth = (struct ibv_flow_spec_eth) {
161 .type = IBV_FLOW_SPEC_ETH,
165 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
170 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
171 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
172 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
173 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
174 /* Remove unwanted bits from values. */
175 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
176 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
177 eth->val.src_mac[i] &= eth->mask.src_mac[i];
183 * Convert VLAN item to Verbs specification.
186 * Item specification.
187 * @param default_mask[in]
188 * Default bit-masks to use when item->mask is not provided.
189 * @param data[in, out]
193 mlx4_flow_create_vlan(const struct rte_flow_item *item,
194 const void *default_mask,
197 const struct rte_flow_item_vlan *spec = item->spec;
198 const struct rte_flow_item_vlan *mask = item->mask;
199 struct mlx4_flow *flow = (struct mlx4_flow *)data;
200 struct ibv_flow_spec_eth *eth;
201 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
203 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
208 eth->val.vlan_tag = spec->tci;
209 eth->mask.vlan_tag = mask->tci;
210 eth->val.vlan_tag &= eth->mask.vlan_tag;
215 * Convert IPv4 item to Verbs specification.
218 * Item specification.
219 * @param default_mask[in]
220 * Default bit-masks to use when item->mask is not provided.
221 * @param data[in, out]
225 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
226 const void *default_mask,
229 const struct rte_flow_item_ipv4 *spec = item->spec;
230 const struct rte_flow_item_ipv4 *mask = item->mask;
231 struct mlx4_flow *flow = (struct mlx4_flow *)data;
232 struct ibv_flow_spec_ipv4 *ipv4;
233 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
235 ++flow->ibv_attr->num_of_specs;
236 flow->ibv_attr->priority = 1;
237 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
238 *ipv4 = (struct ibv_flow_spec_ipv4) {
239 .type = IBV_FLOW_SPEC_IPV4,
244 ipv4->val = (struct ibv_flow_ipv4_filter) {
245 .src_ip = spec->hdr.src_addr,
246 .dst_ip = spec->hdr.dst_addr,
250 ipv4->mask = (struct ibv_flow_ipv4_filter) {
251 .src_ip = mask->hdr.src_addr,
252 .dst_ip = mask->hdr.dst_addr,
254 /* Remove unwanted bits from values. */
255 ipv4->val.src_ip &= ipv4->mask.src_ip;
256 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
261 * Convert UDP item to Verbs specification.
264 * Item specification.
265 * @param default_mask[in]
266 * Default bit-masks to use when item->mask is not provided.
267 * @param data[in, out]
271 mlx4_flow_create_udp(const struct rte_flow_item *item,
272 const void *default_mask,
275 const struct rte_flow_item_udp *spec = item->spec;
276 const struct rte_flow_item_udp *mask = item->mask;
277 struct mlx4_flow *flow = (struct mlx4_flow *)data;
278 struct ibv_flow_spec_tcp_udp *udp;
279 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
281 ++flow->ibv_attr->num_of_specs;
282 flow->ibv_attr->priority = 0;
283 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
284 *udp = (struct ibv_flow_spec_tcp_udp) {
285 .type = IBV_FLOW_SPEC_UDP,
290 udp->val.dst_port = spec->hdr.dst_port;
291 udp->val.src_port = spec->hdr.src_port;
294 udp->mask.dst_port = mask->hdr.dst_port;
295 udp->mask.src_port = mask->hdr.src_port;
296 /* Remove unwanted bits from values. */
297 udp->val.src_port &= udp->mask.src_port;
298 udp->val.dst_port &= udp->mask.dst_port;
303 * Convert TCP item to Verbs specification.
306 * Item specification.
307 * @param default_mask[in]
308 * Default bit-masks to use when item->mask is not provided.
309 * @param data[in, out]
313 mlx4_flow_create_tcp(const struct rte_flow_item *item,
314 const void *default_mask,
317 const struct rte_flow_item_tcp *spec = item->spec;
318 const struct rte_flow_item_tcp *mask = item->mask;
319 struct mlx4_flow *flow = (struct mlx4_flow *)data;
320 struct ibv_flow_spec_tcp_udp *tcp;
321 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
323 ++flow->ibv_attr->num_of_specs;
324 flow->ibv_attr->priority = 0;
325 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
326 *tcp = (struct ibv_flow_spec_tcp_udp) {
327 .type = IBV_FLOW_SPEC_TCP,
332 tcp->val.dst_port = spec->hdr.dst_port;
333 tcp->val.src_port = spec->hdr.src_port;
336 tcp->mask.dst_port = mask->hdr.dst_port;
337 tcp->mask.src_port = mask->hdr.src_port;
338 /* Remove unwanted bits from values. */
339 tcp->val.src_port &= tcp->mask.src_port;
340 tcp->val.dst_port &= tcp->mask.dst_port;
345 * Check support for a given item.
348 * Item specification.
350 * Bit-masks covering supported fields to compare with spec, last and mask in
353 * Bit-Mask size in bytes.
356 * 0 on success, negative value otherwise.
359 mlx4_flow_item_validate(const struct rte_flow_item *item,
360 const uint8_t *mask, unsigned int size)
364 if (!item->spec && (item->mask || item->last))
366 if (item->spec && !item->mask) {
368 const uint8_t *spec = item->spec;
370 for (i = 0; i < size; ++i)
371 if ((spec[i] | mask[i]) != mask[i])
374 if (item->last && !item->mask) {
376 const uint8_t *spec = item->last;
378 for (i = 0; i < size; ++i)
379 if ((spec[i] | mask[i]) != mask[i])
382 if (item->spec && item->last) {
385 const uint8_t *apply = mask;
390 for (i = 0; i < size; ++i) {
391 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
392 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
394 ret = memcmp(spec, last, size);
400 mlx4_flow_validate_eth(const struct rte_flow_item *item,
401 const uint8_t *mask, unsigned int size)
404 const struct rte_flow_item_eth *mask = item->mask;
406 if (mask->dst.addr_bytes[0] != 0xff ||
407 mask->dst.addr_bytes[1] != 0xff ||
408 mask->dst.addr_bytes[2] != 0xff ||
409 mask->dst.addr_bytes[3] != 0xff ||
410 mask->dst.addr_bytes[4] != 0xff ||
411 mask->dst.addr_bytes[5] != 0xff)
414 return mlx4_flow_item_validate(item, mask, size);
418 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
419 const uint8_t *mask, unsigned int size)
422 const struct rte_flow_item_vlan *mask = item->mask;
424 if (mask->tci != 0 &&
425 ntohs(mask->tci) != 0x0fff)
428 return mlx4_flow_item_validate(item, mask, size);
432 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
433 const uint8_t *mask, unsigned int size)
436 const struct rte_flow_item_ipv4 *mask = item->mask;
438 if (mask->hdr.src_addr != 0 &&
439 mask->hdr.src_addr != 0xffffffff)
441 if (mask->hdr.dst_addr != 0 &&
442 mask->hdr.dst_addr != 0xffffffff)
445 return mlx4_flow_item_validate(item, mask, size);
449 mlx4_flow_validate_udp(const struct rte_flow_item *item,
450 const uint8_t *mask, unsigned int size)
453 const struct rte_flow_item_udp *mask = item->mask;
455 if (mask->hdr.src_port != 0 &&
456 mask->hdr.src_port != 0xffff)
458 if (mask->hdr.dst_port != 0 &&
459 mask->hdr.dst_port != 0xffff)
462 return mlx4_flow_item_validate(item, mask, size);
466 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
467 const uint8_t *mask, unsigned int size)
470 const struct rte_flow_item_tcp *mask = item->mask;
472 if (mask->hdr.src_port != 0 &&
473 mask->hdr.src_port != 0xffff)
475 if (mask->hdr.dst_port != 0 &&
476 mask->hdr.dst_port != 0xffff)
479 return mlx4_flow_item_validate(item, mask, size);
482 /** Graph of supported items and associated actions. */
483 static const struct mlx4_flow_items mlx4_flow_items[] = {
484 [RTE_FLOW_ITEM_TYPE_END] = {
485 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
487 [RTE_FLOW_ITEM_TYPE_ETH] = {
488 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
489 RTE_FLOW_ITEM_TYPE_IPV4),
490 .actions = valid_actions,
491 .mask = &(const struct rte_flow_item_eth){
492 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
493 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
495 .default_mask = &rte_flow_item_eth_mask,
496 .mask_sz = sizeof(struct rte_flow_item_eth),
497 .validate = mlx4_flow_validate_eth,
498 .convert = mlx4_flow_create_eth,
499 .dst_sz = sizeof(struct ibv_flow_spec_eth),
501 [RTE_FLOW_ITEM_TYPE_VLAN] = {
502 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
503 .actions = valid_actions,
504 .mask = &(const struct rte_flow_item_vlan){
505 /* rte_flow_item_vlan_mask is invalid for mlx4. */
506 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
512 .mask_sz = sizeof(struct rte_flow_item_vlan),
513 .validate = mlx4_flow_validate_vlan,
514 .convert = mlx4_flow_create_vlan,
517 [RTE_FLOW_ITEM_TYPE_IPV4] = {
518 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
519 RTE_FLOW_ITEM_TYPE_TCP),
520 .actions = valid_actions,
521 .mask = &(const struct rte_flow_item_ipv4){
527 .default_mask = &rte_flow_item_ipv4_mask,
528 .mask_sz = sizeof(struct rte_flow_item_ipv4),
529 .validate = mlx4_flow_validate_ipv4,
530 .convert = mlx4_flow_create_ipv4,
531 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
533 [RTE_FLOW_ITEM_TYPE_UDP] = {
534 .actions = valid_actions,
535 .mask = &(const struct rte_flow_item_udp){
541 .default_mask = &rte_flow_item_udp_mask,
542 .mask_sz = sizeof(struct rte_flow_item_udp),
543 .validate = mlx4_flow_validate_udp,
544 .convert = mlx4_flow_create_udp,
545 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
547 [RTE_FLOW_ITEM_TYPE_TCP] = {
548 .actions = valid_actions,
549 .mask = &(const struct rte_flow_item_tcp){
555 .default_mask = &rte_flow_item_tcp_mask,
556 .mask_sz = sizeof(struct rte_flow_item_tcp),
557 .validate = mlx4_flow_validate_tcp,
558 .convert = mlx4_flow_create_tcp,
559 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
564 * Make sure a flow rule is supported and initialize associated structure.
567 * Pointer to private structure.
569 * Flow rule attributes.
571 * Pattern specification (list terminated by the END pattern item).
573 * Associated actions (list terminated by the END action).
575 * Perform verbose error reporting if not NULL.
576 * @param[in, out] flow
577 * Flow structure to update.
580 * 0 on success, a negative errno value otherwise and rte_errno is set.
583 mlx4_flow_prepare(struct priv *priv,
584 const struct rte_flow_attr *attr,
585 const struct rte_flow_item items[],
586 const struct rte_flow_action actions[],
587 struct rte_flow_error *error,
588 struct mlx4_flow *flow)
590 const struct mlx4_flow_items *cur_item = mlx4_flow_items;
591 struct mlx4_flow_action action = {
598 rte_flow_error_set(error, ENOTSUP,
599 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
601 "groups are not supported");
604 if (attr->priority) {
605 rte_flow_error_set(error, ENOTSUP,
606 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
608 "priorities are not supported");
612 rte_flow_error_set(error, ENOTSUP,
613 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
615 "egress is not supported");
618 if (!attr->ingress) {
619 rte_flow_error_set(error, ENOTSUP,
620 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
622 "only ingress is supported");
625 /* Go over items list. */
626 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
627 const struct mlx4_flow_items *token = NULL;
631 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
634 * The nic can support patterns with NULL eth spec only
635 * if eth is a single item in a rule.
638 items->type == RTE_FLOW_ITEM_TYPE_ETH) {
639 const struct rte_flow_item *next = items + 1;
641 if (next->type != RTE_FLOW_ITEM_TYPE_END) {
642 rte_flow_error_set(error, ENOTSUP,
643 RTE_FLOW_ERROR_TYPE_ITEM,
646 " an Ethernet spec");
652 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
654 if (cur_item->items[i] == items->type) {
655 token = &mlx4_flow_items[items->type];
660 goto exit_item_not_supported;
662 err = cur_item->validate(items,
663 (const uint8_t *)cur_item->mask,
666 goto exit_item_not_supported;
667 if (flow->ibv_attr && cur_item->convert) {
668 err = cur_item->convert(items,
669 (cur_item->default_mask ?
670 cur_item->default_mask :
674 goto exit_item_not_supported;
676 flow->offset += cur_item->dst_sz;
678 /* Go over actions list */
679 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
680 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
682 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
684 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
685 const struct rte_flow_action_queue *queue =
686 (const struct rte_flow_action_queue *)
689 if (!queue || (queue->index > (priv->rxqs_n - 1)))
690 goto exit_action_not_supported;
693 goto exit_action_not_supported;
696 if (!action.queue && !action.drop) {
697 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
698 NULL, "no valid action");
702 exit_item_not_supported:
703 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
704 items, "item not supported");
706 exit_action_not_supported:
707 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
708 actions, "action not supported");
713 * Validate a flow supported by the NIC.
715 * @see rte_flow_validate()
719 mlx4_flow_validate(struct rte_eth_dev *dev,
720 const struct rte_flow_attr *attr,
721 const struct rte_flow_item items[],
722 const struct rte_flow_action actions[],
723 struct rte_flow_error *error)
725 struct priv *priv = dev->data->dev_private;
726 struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
728 return mlx4_flow_prepare(priv, attr, items, actions, error, &flow);
732 * Destroy a drop queue.
735 * Pointer to private structure.
738 mlx4_flow_destroy_drop_queue(struct priv *priv)
740 if (priv->flow_drop_queue) {
741 struct rte_flow_drop *fdq = priv->flow_drop_queue;
743 priv->flow_drop_queue = NULL;
744 claim_zero(ibv_destroy_qp(fdq->qp));
745 claim_zero(ibv_destroy_cq(fdq->cq));
751 * Create a single drop queue for all drop flows.
754 * Pointer to private structure.
757 * 0 on success, negative value otherwise.
760 mlx4_flow_create_drop_queue(struct priv *priv)
764 struct rte_flow_drop *fdq;
766 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
768 ERROR("Cannot allocate memory for drop struct");
771 cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0);
773 ERROR("Cannot create drop CQ");
776 qp = ibv_create_qp(priv->pd,
777 &(struct ibv_qp_init_attr){
784 .qp_type = IBV_QPT_RAW_PACKET,
787 ERROR("Cannot create drop QP");
790 *fdq = (struct rte_flow_drop){
794 priv->flow_drop_queue = fdq;
797 claim_zero(ibv_destroy_cq(cq));
805 * Complete flow rule creation.
808 * Pointer to private structure.
810 * Verbs flow attributes.
812 * Target action structure.
814 * Perform verbose error reporting if not NULL.
817 * A flow if the rule could be created.
819 static struct rte_flow *
820 mlx4_flow_create_action_queue(struct priv *priv,
821 struct ibv_flow_attr *ibv_attr,
822 struct mlx4_flow_action *action,
823 struct rte_flow_error *error)
826 struct rte_flow *rte_flow;
830 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
832 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
833 NULL, "cannot allocate flow memory");
837 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
839 struct rxq *rxq = (*priv->rxqs)[action->queue_id];
844 rte_flow->ibv_attr = ibv_attr;
847 rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
848 if (!rte_flow->ibv_flow) {
849 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
850 NULL, "flow rule creation failure");
862 * @see rte_flow_create()
865 static struct rte_flow *
866 mlx4_flow_create(struct rte_eth_dev *dev,
867 const struct rte_flow_attr *attr,
868 const struct rte_flow_item items[],
869 const struct rte_flow_action actions[],
870 struct rte_flow_error *error)
872 struct priv *priv = dev->data->dev_private;
873 struct rte_flow *rte_flow;
874 struct mlx4_flow_action action;
875 struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
878 err = mlx4_flow_prepare(priv, attr, items, actions, error, &flow);
881 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
882 if (!flow.ibv_attr) {
883 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
884 NULL, "cannot allocate ibv_attr memory");
887 flow.offset = sizeof(struct ibv_flow_attr);
888 *flow.ibv_attr = (struct ibv_flow_attr){
890 .type = IBV_FLOW_ATTR_NORMAL,
891 .size = sizeof(struct ibv_flow_attr),
892 .priority = attr->priority,
897 claim_zero(mlx4_flow_prepare(priv, attr, items, actions,
899 action = (struct mlx4_flow_action){
903 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
904 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
906 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
909 ((const struct rte_flow_action_queue *)
910 actions->conf)->index;
911 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
914 rte_flow_error_set(error, ENOTSUP,
915 RTE_FLOW_ERROR_TYPE_ACTION,
916 actions, "unsupported action");
920 rte_flow = mlx4_flow_create_action_queue(priv, flow.ibv_attr,
923 LIST_INSERT_HEAD(&priv->flows, rte_flow, next);
924 DEBUG("Flow created %p", (void *)rte_flow);
928 rte_free(flow.ibv_attr);
933 * @see rte_flow_isolate()
935 * Must be done before calling dev_configure().
938 * Pointer to the ethernet device structure.
940 * Nonzero to enter isolated mode, attempt to leave it otherwise.
942 * Perform verbose error reporting if not NULL. PMDs initialize this
943 * structure in case of error only.
946 * 0 on success, a negative value on error.
949 mlx4_flow_isolate(struct rte_eth_dev *dev,
951 struct rte_flow_error *error)
953 struct priv *priv = dev->data->dev_private;
956 rte_flow_error_set(error, ENOTSUP,
957 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
958 NULL, "isolated mode must be set"
959 " before configuring the device");
962 priv->isolated = !!enable;
969 * @see rte_flow_destroy()
973 mlx4_flow_destroy(struct rte_eth_dev *dev,
974 struct rte_flow *flow,
975 struct rte_flow_error *error)
979 LIST_REMOVE(flow, next);
981 claim_zero(ibv_destroy_flow(flow->ibv_flow));
982 rte_free(flow->ibv_attr);
983 DEBUG("Flow destroyed %p", (void *)flow);
991 * @see rte_flow_flush()
995 mlx4_flow_flush(struct rte_eth_dev *dev,
996 struct rte_flow_error *error)
998 struct priv *priv = dev->data->dev_private;
1000 while (!LIST_EMPTY(&priv->flows)) {
1001 struct rte_flow *flow;
1003 flow = LIST_FIRST(&priv->flows);
1004 mlx4_flow_destroy(dev, flow, error);
1012 * Called by dev_stop() to remove all flows.
1015 * Pointer to private structure.
1018 mlx4_flow_stop(struct priv *priv)
1020 struct rte_flow *flow;
1022 for (flow = LIST_FIRST(&priv->flows);
1024 flow = LIST_NEXT(flow, next)) {
1025 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1026 flow->ibv_flow = NULL;
1027 DEBUG("Flow %p removed", (void *)flow);
1029 mlx4_flow_destroy_drop_queue(priv);
1036 * Pointer to private structure.
1039 * 0 on success, a errno value otherwise and rte_errno is set.
1042 mlx4_flow_start(struct priv *priv)
1046 struct rte_flow *flow;
1048 ret = mlx4_flow_create_drop_queue(priv);
1051 for (flow = LIST_FIRST(&priv->flows);
1053 flow = LIST_NEXT(flow, next)) {
1054 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1055 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1056 if (!flow->ibv_flow) {
1057 DEBUG("Flow %p cannot be applied", (void *)flow);
1061 DEBUG("Flow %p applied", (void *)flow);
1066 static const struct rte_flow_ops mlx4_flow_ops = {
1067 .validate = mlx4_flow_validate,
1068 .create = mlx4_flow_create,
1069 .destroy = mlx4_flow_destroy,
1070 .flush = mlx4_flow_flush,
1071 .isolate = mlx4_flow_isolate,
1075 * Manage filter operations.
1078 * Pointer to Ethernet device structure.
1079 * @param filter_type
1082 * Operation to perform.
1084 * Pointer to operation-specific structure.
1087 * 0 on success, negative errno value otherwise and rte_errno is set.
1090 mlx4_filter_ctrl(struct rte_eth_dev *dev,
1091 enum rte_filter_type filter_type,
1092 enum rte_filter_op filter_op,
1095 switch (filter_type) {
1096 case RTE_ETH_FILTER_GENERIC:
1097 if (filter_op != RTE_ETH_FILTER_GET)
1099 *(const void **)arg = &mlx4_flow_ops;
1102 ERROR("%p: filter type (%d) not supported",
1103 (void *)dev, filter_type);
1106 rte_errno = ENOTSUP;