4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 #include <rte_flow_driver.h>
38 #include <rte_malloc.h>
40 /* Generated configuration header. */
41 #include "mlx4_autoconf.h"
45 #include "mlx4_flow.h"
47 /** Static initializer for items. */
49 (const enum rte_flow_item_type []){ \
50 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
53 /** Structure to generate a simple graph of layers supported by the NIC. */
54 struct mlx4_flow_items {
55 /** List of possible actions for these items. */
56 const enum rte_flow_action_type *const actions;
57 /** Bit-masks corresponding to the possibilities for the item. */
60 * Default bit-masks to use when item->mask is not provided. When
61 * \default_mask is also NULL, the full supported bit-mask (\mask) is
64 const void *default_mask;
65 /** Bit-masks size in bytes. */
66 const unsigned int mask_sz;
68 * Check support for a given item.
73 * Bit-masks covering supported fields to compare with spec,
77 * Bit-Mask size in bytes.
80 * 0 on success, negative value otherwise.
82 int (*validate)(const struct rte_flow_item *item,
83 const uint8_t *mask, unsigned int size);
85 * Conversion function from rte_flow to NIC specific flow.
88 * rte_flow item to convert.
90 * Default bit-masks to use when item->mask is not provided.
92 * Internal structure to store the conversion.
95 * 0 on success, negative value otherwise.
97 int (*convert)(const struct rte_flow_item *item,
98 const void *default_mask,
100 /** Size in bytes of the destination structure. */
101 const unsigned int dst_sz;
102 /** List of possible following items. */
103 const enum rte_flow_item_type *const items;
106 struct rte_flow_drop {
107 struct ibv_qp *qp; /**< Verbs queue pair. */
108 struct ibv_cq *cq; /**< Verbs completion queue. */
111 /** Valid action for this PMD. */
112 static const enum rte_flow_action_type valid_actions[] = {
113 RTE_FLOW_ACTION_TYPE_DROP,
114 RTE_FLOW_ACTION_TYPE_QUEUE,
115 RTE_FLOW_ACTION_TYPE_RSS,
116 RTE_FLOW_ACTION_TYPE_END,
120 * Convert Ethernet item to Verbs specification.
123 * Item specification.
124 * @param default_mask[in]
125 * Default bit-masks to use when item->mask is not provided.
126 * @param data[in, out]
130 mlx4_flow_create_eth(const struct rte_flow_item *item,
131 const void *default_mask,
134 const struct rte_flow_item_eth *spec = item->spec;
135 const struct rte_flow_item_eth *mask = item->mask;
136 struct mlx4_flow *flow = (struct mlx4_flow *)data;
137 struct ibv_flow_spec_eth *eth;
138 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
141 ++flow->ibv_attr->num_of_specs;
142 flow->ibv_attr->priority = 2;
143 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
144 *eth = (struct ibv_flow_spec_eth) {
145 .type = IBV_FLOW_SPEC_ETH,
149 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
154 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
155 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
156 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
157 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
158 /* Remove unwanted bits from values. */
159 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
160 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
161 eth->val.src_mac[i] &= eth->mask.src_mac[i];
167 * Convert VLAN item to Verbs specification.
170 * Item specification.
171 * @param default_mask[in]
172 * Default bit-masks to use when item->mask is not provided.
173 * @param data[in, out]
177 mlx4_flow_create_vlan(const struct rte_flow_item *item,
178 const void *default_mask,
181 const struct rte_flow_item_vlan *spec = item->spec;
182 const struct rte_flow_item_vlan *mask = item->mask;
183 struct mlx4_flow *flow = (struct mlx4_flow *)data;
184 struct ibv_flow_spec_eth *eth;
185 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
187 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
192 eth->val.vlan_tag = spec->tci;
193 eth->mask.vlan_tag = mask->tci;
194 eth->val.vlan_tag &= eth->mask.vlan_tag;
199 * Convert IPv4 item to Verbs specification.
202 * Item specification.
203 * @param default_mask[in]
204 * Default bit-masks to use when item->mask is not provided.
205 * @param data[in, out]
209 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
210 const void *default_mask,
213 const struct rte_flow_item_ipv4 *spec = item->spec;
214 const struct rte_flow_item_ipv4 *mask = item->mask;
215 struct mlx4_flow *flow = (struct mlx4_flow *)data;
216 struct ibv_flow_spec_ipv4 *ipv4;
217 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
219 ++flow->ibv_attr->num_of_specs;
220 flow->ibv_attr->priority = 1;
221 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
222 *ipv4 = (struct ibv_flow_spec_ipv4) {
223 .type = IBV_FLOW_SPEC_IPV4,
228 ipv4->val = (struct ibv_flow_ipv4_filter) {
229 .src_ip = spec->hdr.src_addr,
230 .dst_ip = spec->hdr.dst_addr,
234 ipv4->mask = (struct ibv_flow_ipv4_filter) {
235 .src_ip = mask->hdr.src_addr,
236 .dst_ip = mask->hdr.dst_addr,
238 /* Remove unwanted bits from values. */
239 ipv4->val.src_ip &= ipv4->mask.src_ip;
240 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
245 * Convert UDP item to Verbs specification.
248 * Item specification.
249 * @param default_mask[in]
250 * Default bit-masks to use when item->mask is not provided.
251 * @param data[in, out]
255 mlx4_flow_create_udp(const struct rte_flow_item *item,
256 const void *default_mask,
259 const struct rte_flow_item_udp *spec = item->spec;
260 const struct rte_flow_item_udp *mask = item->mask;
261 struct mlx4_flow *flow = (struct mlx4_flow *)data;
262 struct ibv_flow_spec_tcp_udp *udp;
263 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
265 ++flow->ibv_attr->num_of_specs;
266 flow->ibv_attr->priority = 0;
267 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
268 *udp = (struct ibv_flow_spec_tcp_udp) {
269 .type = IBV_FLOW_SPEC_UDP,
274 udp->val.dst_port = spec->hdr.dst_port;
275 udp->val.src_port = spec->hdr.src_port;
278 udp->mask.dst_port = mask->hdr.dst_port;
279 udp->mask.src_port = mask->hdr.src_port;
280 /* Remove unwanted bits from values. */
281 udp->val.src_port &= udp->mask.src_port;
282 udp->val.dst_port &= udp->mask.dst_port;
287 * Convert TCP item to Verbs specification.
290 * Item specification.
291 * @param default_mask[in]
292 * Default bit-masks to use when item->mask is not provided.
293 * @param data[in, out]
297 mlx4_flow_create_tcp(const struct rte_flow_item *item,
298 const void *default_mask,
301 const struct rte_flow_item_tcp *spec = item->spec;
302 const struct rte_flow_item_tcp *mask = item->mask;
303 struct mlx4_flow *flow = (struct mlx4_flow *)data;
304 struct ibv_flow_spec_tcp_udp *tcp;
305 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
307 ++flow->ibv_attr->num_of_specs;
308 flow->ibv_attr->priority = 0;
309 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
310 *tcp = (struct ibv_flow_spec_tcp_udp) {
311 .type = IBV_FLOW_SPEC_TCP,
316 tcp->val.dst_port = spec->hdr.dst_port;
317 tcp->val.src_port = spec->hdr.src_port;
320 tcp->mask.dst_port = mask->hdr.dst_port;
321 tcp->mask.src_port = mask->hdr.src_port;
322 /* Remove unwanted bits from values. */
323 tcp->val.src_port &= tcp->mask.src_port;
324 tcp->val.dst_port &= tcp->mask.dst_port;
329 * Check support for a given item.
332 * Item specification.
334 * Bit-masks covering supported fields to compare with spec, last and mask in
337 * Bit-Mask size in bytes.
340 * 0 on success, negative value otherwise.
343 mlx4_flow_item_validate(const struct rte_flow_item *item,
344 const uint8_t *mask, unsigned int size)
348 if (!item->spec && (item->mask || item->last))
350 if (item->spec && !item->mask) {
352 const uint8_t *spec = item->spec;
354 for (i = 0; i < size; ++i)
355 if ((spec[i] | mask[i]) != mask[i])
358 if (item->last && !item->mask) {
360 const uint8_t *spec = item->last;
362 for (i = 0; i < size; ++i)
363 if ((spec[i] | mask[i]) != mask[i])
366 if (item->spec && item->last) {
369 const uint8_t *apply = mask;
374 for (i = 0; i < size; ++i) {
375 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
376 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
378 ret = memcmp(spec, last, size);
384 mlx4_flow_validate_eth(const struct rte_flow_item *item,
385 const uint8_t *mask, unsigned int size)
388 const struct rte_flow_item_eth *mask = item->mask;
390 if (mask->dst.addr_bytes[0] != 0xff ||
391 mask->dst.addr_bytes[1] != 0xff ||
392 mask->dst.addr_bytes[2] != 0xff ||
393 mask->dst.addr_bytes[3] != 0xff ||
394 mask->dst.addr_bytes[4] != 0xff ||
395 mask->dst.addr_bytes[5] != 0xff)
398 return mlx4_flow_item_validate(item, mask, size);
402 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
403 const uint8_t *mask, unsigned int size)
406 const struct rte_flow_item_vlan *mask = item->mask;
408 if (mask->tci != 0 &&
409 ntohs(mask->tci) != 0x0fff)
412 return mlx4_flow_item_validate(item, mask, size);
416 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
417 const uint8_t *mask, unsigned int size)
420 const struct rte_flow_item_ipv4 *mask = item->mask;
422 if (mask->hdr.src_addr != 0 &&
423 mask->hdr.src_addr != 0xffffffff)
425 if (mask->hdr.dst_addr != 0 &&
426 mask->hdr.dst_addr != 0xffffffff)
429 return mlx4_flow_item_validate(item, mask, size);
433 mlx4_flow_validate_udp(const struct rte_flow_item *item,
434 const uint8_t *mask, unsigned int size)
437 const struct rte_flow_item_udp *mask = item->mask;
439 if (mask->hdr.src_port != 0 &&
440 mask->hdr.src_port != 0xffff)
442 if (mask->hdr.dst_port != 0 &&
443 mask->hdr.dst_port != 0xffff)
446 return mlx4_flow_item_validate(item, mask, size);
450 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
451 const uint8_t *mask, unsigned int size)
454 const struct rte_flow_item_tcp *mask = item->mask;
456 if (mask->hdr.src_port != 0 &&
457 mask->hdr.src_port != 0xffff)
459 if (mask->hdr.dst_port != 0 &&
460 mask->hdr.dst_port != 0xffff)
463 return mlx4_flow_item_validate(item, mask, size);
466 /** Graph of supported items and associated actions. */
467 static const struct mlx4_flow_items mlx4_flow_items[] = {
468 [RTE_FLOW_ITEM_TYPE_END] = {
469 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
471 [RTE_FLOW_ITEM_TYPE_ETH] = {
472 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
473 RTE_FLOW_ITEM_TYPE_IPV4),
474 .actions = valid_actions,
475 .mask = &(const struct rte_flow_item_eth){
476 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
477 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
479 .default_mask = &rte_flow_item_eth_mask,
480 .mask_sz = sizeof(struct rte_flow_item_eth),
481 .validate = mlx4_flow_validate_eth,
482 .convert = mlx4_flow_create_eth,
483 .dst_sz = sizeof(struct ibv_flow_spec_eth),
485 [RTE_FLOW_ITEM_TYPE_VLAN] = {
486 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
487 .actions = valid_actions,
488 .mask = &(const struct rte_flow_item_vlan){
489 /* rte_flow_item_vlan_mask is invalid for mlx4. */
490 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
496 .mask_sz = sizeof(struct rte_flow_item_vlan),
497 .validate = mlx4_flow_validate_vlan,
498 .convert = mlx4_flow_create_vlan,
501 [RTE_FLOW_ITEM_TYPE_IPV4] = {
502 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
503 RTE_FLOW_ITEM_TYPE_TCP),
504 .actions = valid_actions,
505 .mask = &(const struct rte_flow_item_ipv4){
511 .default_mask = &rte_flow_item_ipv4_mask,
512 .mask_sz = sizeof(struct rte_flow_item_ipv4),
513 .validate = mlx4_flow_validate_ipv4,
514 .convert = mlx4_flow_create_ipv4,
515 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
517 [RTE_FLOW_ITEM_TYPE_UDP] = {
518 .actions = valid_actions,
519 .mask = &(const struct rte_flow_item_udp){
525 .default_mask = &rte_flow_item_udp_mask,
526 .mask_sz = sizeof(struct rte_flow_item_udp),
527 .validate = mlx4_flow_validate_udp,
528 .convert = mlx4_flow_create_udp,
529 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
531 [RTE_FLOW_ITEM_TYPE_TCP] = {
532 .actions = valid_actions,
533 .mask = &(const struct rte_flow_item_tcp){
539 .default_mask = &rte_flow_item_tcp_mask,
540 .mask_sz = sizeof(struct rte_flow_item_tcp),
541 .validate = mlx4_flow_validate_tcp,
542 .convert = mlx4_flow_create_tcp,
543 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
548 * Validate a flow supported by the NIC.
551 * Pointer to private structure.
553 * Flow rule attributes.
555 * Pattern specification (list terminated by the END pattern item).
557 * Associated actions (list terminated by the END action).
559 * Perform verbose error reporting if not NULL.
560 * @param[in, out] flow
561 * Flow structure to update.
564 * 0 on success, a negative errno value otherwise and rte_errno is set.
567 priv_flow_validate(struct priv *priv,
568 const struct rte_flow_attr *attr,
569 const struct rte_flow_item items[],
570 const struct rte_flow_action actions[],
571 struct rte_flow_error *error,
572 struct mlx4_flow *flow)
574 const struct mlx4_flow_items *cur_item = mlx4_flow_items;
575 struct mlx4_flow_action action = {
582 rte_flow_error_set(error, ENOTSUP,
583 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
585 "groups are not supported");
588 if (attr->priority) {
589 rte_flow_error_set(error, ENOTSUP,
590 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
592 "priorities are not supported");
596 rte_flow_error_set(error, ENOTSUP,
597 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
599 "egress is not supported");
602 if (!attr->ingress) {
603 rte_flow_error_set(error, ENOTSUP,
604 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
606 "only ingress is supported");
609 /* Go over items list. */
610 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
611 const struct mlx4_flow_items *token = NULL;
615 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
618 * The nic can support patterns with NULL eth spec only
619 * if eth is a single item in a rule.
622 items->type == RTE_FLOW_ITEM_TYPE_ETH) {
623 const struct rte_flow_item *next = items + 1;
625 if (next->type != RTE_FLOW_ITEM_TYPE_END) {
626 rte_flow_error_set(error, ENOTSUP,
627 RTE_FLOW_ERROR_TYPE_ITEM,
630 " an Ethernet spec");
636 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
638 if (cur_item->items[i] == items->type) {
639 token = &mlx4_flow_items[items->type];
644 goto exit_item_not_supported;
646 err = cur_item->validate(items,
647 (const uint8_t *)cur_item->mask,
650 goto exit_item_not_supported;
651 if (flow->ibv_attr && cur_item->convert) {
652 err = cur_item->convert(items,
653 (cur_item->default_mask ?
654 cur_item->default_mask :
658 goto exit_item_not_supported;
660 flow->offset += cur_item->dst_sz;
662 /* Go over actions list */
663 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
664 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
666 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
668 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
669 const struct rte_flow_action_queue *queue =
670 (const struct rte_flow_action_queue *)
673 if (!queue || (queue->index > (priv->rxqs_n - 1)))
674 goto exit_action_not_supported;
677 action.queues[0] = queue->index;
678 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
681 const struct rte_flow_action_rss *rss =
682 (const struct rte_flow_action_rss *)
686 rte_flow_error_set(error, ENOTSUP,
687 RTE_FLOW_ERROR_TYPE_ACTION,
689 "RSS cannot be used with "
690 "the current configuration");
693 if (!priv->isolated) {
694 rte_flow_error_set(error, ENOTSUP,
695 RTE_FLOW_ERROR_TYPE_ACTION,
697 "RSS cannot be used without "
701 if (!rte_is_power_of_2(rss->num)) {
702 rte_flow_error_set(error, ENOTSUP,
703 RTE_FLOW_ERROR_TYPE_ACTION,
705 "the number of queues "
706 "should be power of two");
709 if (priv->max_rss_tbl_sz < rss->num) {
710 rte_flow_error_set(error, ENOTSUP,
711 RTE_FLOW_ERROR_TYPE_ACTION,
713 "the number of queues "
717 /* checking indexes array */
719 for (i = 0; i < rss->num; ++i) {
721 if (rss->queue[i] >= priv->rxqs_n)
724 * Prevent the user from specifying
725 * the same queue twice in the RSS array.
727 for (j = i + 1; j < rss->num && !ierr; ++j)
728 if (rss->queue[j] == rss->queue[i])
734 RTE_FLOW_ERROR_TYPE_HANDLE,
736 "RSS action only supports "
737 "unique queue indices "
743 action.queues_n = rss->num;
744 for (i = 0; i < rss->num; ++i)
745 action.queues[i] = rss->queue[i];
747 goto exit_action_not_supported;
750 if (!action.queue && !action.drop) {
751 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
752 NULL, "no valid action");
756 exit_item_not_supported:
757 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
758 items, "item not supported");
760 exit_action_not_supported:
761 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
762 actions, "action not supported");
767 * Validate a flow supported by the NIC.
769 * @see rte_flow_validate()
773 mlx4_flow_validate(struct rte_eth_dev *dev,
774 const struct rte_flow_attr *attr,
775 const struct rte_flow_item items[],
776 const struct rte_flow_action actions[],
777 struct rte_flow_error *error)
779 struct priv *priv = dev->data->dev_private;
781 struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
784 ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
790 * Destroy a drop queue.
793 * Pointer to private structure.
796 mlx4_flow_destroy_drop_queue(struct priv *priv)
798 if (priv->flow_drop_queue) {
799 struct rte_flow_drop *fdq = priv->flow_drop_queue;
801 priv->flow_drop_queue = NULL;
802 claim_zero(ibv_destroy_qp(fdq->qp));
803 claim_zero(ibv_destroy_cq(fdq->cq));
809 * Create a single drop queue for all drop flows.
812 * Pointer to private structure.
815 * 0 on success, negative value otherwise.
818 mlx4_flow_create_drop_queue(struct priv *priv)
822 struct rte_flow_drop *fdq;
824 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
826 ERROR("Cannot allocate memory for drop struct");
829 cq = ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
830 &(struct ibv_exp_cq_init_attr){
834 ERROR("Cannot create drop CQ");
837 qp = ibv_exp_create_qp(priv->ctx,
838 &(struct ibv_exp_qp_init_attr){
845 .qp_type = IBV_QPT_RAW_PACKET,
847 IBV_EXP_QP_INIT_ATTR_PD |
848 IBV_EXP_QP_INIT_ATTR_PORT,
850 .port_num = priv->port,
853 ERROR("Cannot create drop QP");
856 *fdq = (struct rte_flow_drop){
860 priv->flow_drop_queue = fdq;
863 claim_zero(ibv_destroy_cq(cq));
871 * Get RSS parent rxq structure for given queues.
873 * Creates a new or returns an existed one.
876 * Pointer to private structure.
878 * queues indices array, NULL in default RSS case.
880 * the size of queues array.
883 * Pointer to a parent rxq structure, NULL on failure.
886 priv_parent_get(struct priv *priv,
889 struct rte_flow_error *error)
894 for (parent = LIST_FIRST(&priv->parents);
896 parent = LIST_NEXT(parent, next)) {
897 unsigned int same = 0;
898 unsigned int overlap = 0;
901 * Find out whether an appropriate parent queue already exists
902 * and can be reused, otherwise make sure there are no overlaps.
904 for (i = 0; i < children_n; ++i) {
907 for (j = 0; j < parent->rss.queues_n; ++j) {
908 if (parent->rss.queues[j] != queues[i])
915 if (same == children_n &&
916 children_n == parent->rss.queues_n)
921 /* Exclude the cases when some QPs were created without RSS */
922 for (i = 0; i < children_n; ++i) {
923 struct rxq *rxq = (*priv->rxqs)[queues[i]];
927 parent = priv_parent_create(priv, queues, children_n);
929 rte_flow_error_set(error,
930 ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
931 NULL, "flow rule creation failure");
937 rte_flow_error_set(error,
939 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
941 "sharing a queue between several"
942 " RSS groups is not supported");
947 * Complete flow rule creation.
950 * Pointer to private structure.
952 * Verbs flow attributes.
954 * Target action structure.
956 * Perform verbose error reporting if not NULL.
959 * A flow if the rule could be created.
961 static struct rte_flow *
962 priv_flow_create_action_queue(struct priv *priv,
963 struct ibv_flow_attr *ibv_attr,
964 struct mlx4_flow_action *action,
965 struct rte_flow_error *error)
968 struct rte_flow *rte_flow;
969 struct rxq *rxq_parent = NULL;
973 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
975 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
976 NULL, "cannot allocate flow memory");
980 qp = priv->flow_drop_queue->qp;
984 struct rxq *rxq = NULL;
986 if (action->queues_n > 1) {
987 rxq_parent = priv_parent_get(priv, action->queues,
988 action->queues_n, error);
992 for (i = 0; i < action->queues_n; ++i) {
993 rxq = (*priv->rxqs)[action->queues[i]];
995 * In case of isolated mode we postpone
996 * ibv receive queue creation till the first
997 * rte_flow rule will be applied on that queue.
1000 assert(priv->isolated);
1001 ret = rxq_create_qp(rxq, rxq->elts_n,
1007 RTE_FLOW_ERROR_TYPE_HANDLE,
1009 "flow rule creation failure");
1014 qp = action->queues_n > 1 ? rxq_parent->qp : rxq->qp;
1017 rte_flow->ibv_attr = ibv_attr;
1018 rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
1019 if (!rte_flow->ibv_flow) {
1020 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1021 NULL, "flow rule creation failure");
1028 rxq_parent_cleanup(rxq_parent);
1037 * Pointer to private structure.
1039 * Flow rule attributes.
1041 * Pattern specification (list terminated by the END pattern item).
1042 * @param[in] actions
1043 * Associated actions (list terminated by the END action).
1045 * Perform verbose error reporting if not NULL.
1048 * A flow on success, NULL otherwise.
1050 static struct rte_flow *
1051 priv_flow_create(struct priv *priv,
1052 const struct rte_flow_attr *attr,
1053 const struct rte_flow_item items[],
1054 const struct rte_flow_action actions[],
1055 struct rte_flow_error *error)
1057 struct rte_flow *rte_flow;
1058 struct mlx4_flow_action action;
1059 struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
1062 err = priv_flow_validate(priv, attr, items, actions, error, &flow);
1065 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
1066 if (!flow.ibv_attr) {
1067 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1068 NULL, "cannot allocate ibv_attr memory");
1071 flow.offset = sizeof(struct ibv_flow_attr);
1072 *flow.ibv_attr = (struct ibv_flow_attr){
1074 .type = IBV_FLOW_ATTR_NORMAL,
1075 .size = sizeof(struct ibv_flow_attr),
1076 .priority = attr->priority,
1081 claim_zero(priv_flow_validate(priv, attr, items, actions,
1083 action = (struct mlx4_flow_action){
1087 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1088 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1090 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1092 action.queues_n = 1;
1094 ((const struct rte_flow_action_queue *)
1095 actions->conf)->index;
1096 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1098 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
1100 const struct rte_flow_action_rss *rss =
1101 (const struct rte_flow_action_rss *)
1105 action.queues_n = rss->num;
1106 for (i = 0; i < rss->num; ++i)
1107 action.queues[i] = rss->queue[i];
1109 rte_flow_error_set(error, ENOTSUP,
1110 RTE_FLOW_ERROR_TYPE_ACTION,
1111 actions, "unsupported action");
1115 rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
1120 rte_free(flow.ibv_attr);
1127 * @see rte_flow_create()
1131 mlx4_flow_create(struct rte_eth_dev *dev,
1132 const struct rte_flow_attr *attr,
1133 const struct rte_flow_item items[],
1134 const struct rte_flow_action actions[],
1135 struct rte_flow_error *error)
1137 struct priv *priv = dev->data->dev_private;
1138 struct rte_flow *flow;
1141 flow = priv_flow_create(priv, attr, items, actions, error);
1143 LIST_INSERT_HEAD(&priv->flows, flow, next);
1144 DEBUG("Flow created %p", (void *)flow);
1151 * @see rte_flow_isolate()
1153 * Must be done before calling dev_configure().
1156 * Pointer to the ethernet device structure.
1158 * Nonzero to enter isolated mode, attempt to leave it otherwise.
1160 * Perform verbose error reporting if not NULL. PMDs initialize this
1161 * structure in case of error only.
1164 * 0 on success, a negative value on error.
1167 mlx4_flow_isolate(struct rte_eth_dev *dev,
1169 struct rte_flow_error *error)
1171 struct priv *priv = dev->data->dev_private;
1175 rte_flow_error_set(error, ENOTSUP,
1176 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1177 NULL, "isolated mode must be set"
1178 " before configuring the device");
1182 priv->isolated = !!enable;
1191 * Pointer to private structure.
1196 priv_flow_destroy(struct priv *priv, struct rte_flow *flow)
1199 LIST_REMOVE(flow, next);
1201 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1202 rte_free(flow->ibv_attr);
1203 DEBUG("Flow destroyed %p", (void *)flow);
1210 * @see rte_flow_destroy()
1214 mlx4_flow_destroy(struct rte_eth_dev *dev,
1215 struct rte_flow *flow,
1216 struct rte_flow_error *error)
1218 struct priv *priv = dev->data->dev_private;
1222 priv_flow_destroy(priv, flow);
1228 * Destroy all flows.
1231 * Pointer to private structure.
1234 priv_flow_flush(struct priv *priv)
1236 while (!LIST_EMPTY(&priv->flows)) {
1237 struct rte_flow *flow;
1239 flow = LIST_FIRST(&priv->flows);
1240 priv_flow_destroy(priv, flow);
1245 * Destroy all flows.
1247 * @see rte_flow_flush()
1251 mlx4_flow_flush(struct rte_eth_dev *dev,
1252 struct rte_flow_error *error)
1254 struct priv *priv = dev->data->dev_private;
1258 priv_flow_flush(priv);
1266 * Called by dev_stop() to remove all flows.
1269 * Pointer to private structure.
1272 mlx4_priv_flow_stop(struct priv *priv)
1274 struct rte_flow *flow;
1276 for (flow = LIST_FIRST(&priv->flows);
1278 flow = LIST_NEXT(flow, next)) {
1279 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1280 flow->ibv_flow = NULL;
1281 DEBUG("Flow %p removed", (void *)flow);
1283 mlx4_flow_destroy_drop_queue(priv);
1290 * Pointer to private structure.
1293 * 0 on success, a errno value otherwise and rte_errno is set.
1296 mlx4_priv_flow_start(struct priv *priv)
1300 struct rte_flow *flow;
1302 ret = mlx4_flow_create_drop_queue(priv);
1305 for (flow = LIST_FIRST(&priv->flows);
1307 flow = LIST_NEXT(flow, next)) {
1308 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1309 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1310 if (!flow->ibv_flow) {
1311 DEBUG("Flow %p cannot be applied", (void *)flow);
1315 DEBUG("Flow %p applied", (void *)flow);