4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 #include <rte_flow_driver.h>
38 #include <rte_malloc.h>
42 #include "mlx4_flow.h"
44 /** Static initializer for items. */
46 (const enum rte_flow_item_type []){ \
47 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
50 /** Structure to generate a simple graph of layers supported by the NIC. */
51 struct mlx4_flow_items {
52 /** List of possible actions for these items. */
53 const enum rte_flow_action_type *const actions;
54 /** Bit-masks corresponding to the possibilities for the item. */
57 * Default bit-masks to use when item->mask is not provided. When
58 * \default_mask is also NULL, the full supported bit-mask (\mask) is
61 const void *default_mask;
62 /** Bit-masks size in bytes. */
63 const unsigned int mask_sz;
65 * Check support for a given item.
70 * Bit-masks covering supported fields to compare with spec,
74 * Bit-Mask size in bytes.
77 * 0 on success, negative value otherwise.
79 int (*validate)(const struct rte_flow_item *item,
80 const uint8_t *mask, unsigned int size);
82 * Conversion function from rte_flow to NIC specific flow.
85 * rte_flow item to convert.
87 * Default bit-masks to use when item->mask is not provided.
89 * Internal structure to store the conversion.
92 * 0 on success, negative value otherwise.
94 int (*convert)(const struct rte_flow_item *item,
95 const void *default_mask,
97 /** Size in bytes of the destination structure. */
98 const unsigned int dst_sz;
99 /** List of possible following items. */
100 const enum rte_flow_item_type *const items;
103 struct rte_flow_drop {
104 struct ibv_qp *qp; /**< Verbs queue pair. */
105 struct ibv_cq *cq; /**< Verbs completion queue. */
108 /** Valid action for this PMD. */
109 static const enum rte_flow_action_type valid_actions[] = {
110 RTE_FLOW_ACTION_TYPE_DROP,
111 RTE_FLOW_ACTION_TYPE_QUEUE,
112 RTE_FLOW_ACTION_TYPE_RSS,
113 RTE_FLOW_ACTION_TYPE_END,
117 * Convert Ethernet item to Verbs specification.
120 * Item specification.
121 * @param default_mask[in]
122 * Default bit-masks to use when item->mask is not provided.
123 * @param data[in, out]
127 mlx4_flow_create_eth(const struct rte_flow_item *item,
128 const void *default_mask,
131 const struct rte_flow_item_eth *spec = item->spec;
132 const struct rte_flow_item_eth *mask = item->mask;
133 struct mlx4_flow *flow = (struct mlx4_flow *)data;
134 struct ibv_flow_spec_eth *eth;
135 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
138 ++flow->ibv_attr->num_of_specs;
139 flow->ibv_attr->priority = 2;
140 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
141 *eth = (struct ibv_flow_spec_eth) {
142 .type = IBV_FLOW_SPEC_ETH,
146 flow->ibv_attr->type = IBV_FLOW_ATTR_ALL_DEFAULT;
151 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
152 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
153 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
154 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
155 /* Remove unwanted bits from values. */
156 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
157 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
158 eth->val.src_mac[i] &= eth->mask.src_mac[i];
164 * Convert VLAN item to Verbs specification.
167 * Item specification.
168 * @param default_mask[in]
169 * Default bit-masks to use when item->mask is not provided.
170 * @param data[in, out]
174 mlx4_flow_create_vlan(const struct rte_flow_item *item,
175 const void *default_mask,
178 const struct rte_flow_item_vlan *spec = item->spec;
179 const struct rte_flow_item_vlan *mask = item->mask;
180 struct mlx4_flow *flow = (struct mlx4_flow *)data;
181 struct ibv_flow_spec_eth *eth;
182 const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth);
184 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
189 eth->val.vlan_tag = spec->tci;
190 eth->mask.vlan_tag = mask->tci;
191 eth->val.vlan_tag &= eth->mask.vlan_tag;
196 * Convert IPv4 item to Verbs specification.
199 * Item specification.
200 * @param default_mask[in]
201 * Default bit-masks to use when item->mask is not provided.
202 * @param data[in, out]
206 mlx4_flow_create_ipv4(const struct rte_flow_item *item,
207 const void *default_mask,
210 const struct rte_flow_item_ipv4 *spec = item->spec;
211 const struct rte_flow_item_ipv4 *mask = item->mask;
212 struct mlx4_flow *flow = (struct mlx4_flow *)data;
213 struct ibv_flow_spec_ipv4 *ipv4;
214 unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4);
216 ++flow->ibv_attr->num_of_specs;
217 flow->ibv_attr->priority = 1;
218 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
219 *ipv4 = (struct ibv_flow_spec_ipv4) {
220 .type = IBV_FLOW_SPEC_IPV4,
225 ipv4->val = (struct ibv_flow_ipv4_filter) {
226 .src_ip = spec->hdr.src_addr,
227 .dst_ip = spec->hdr.dst_addr,
231 ipv4->mask = (struct ibv_flow_ipv4_filter) {
232 .src_ip = mask->hdr.src_addr,
233 .dst_ip = mask->hdr.dst_addr,
235 /* Remove unwanted bits from values. */
236 ipv4->val.src_ip &= ipv4->mask.src_ip;
237 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
242 * Convert UDP item to Verbs specification.
245 * Item specification.
246 * @param default_mask[in]
247 * Default bit-masks to use when item->mask is not provided.
248 * @param data[in, out]
252 mlx4_flow_create_udp(const struct rte_flow_item *item,
253 const void *default_mask,
256 const struct rte_flow_item_udp *spec = item->spec;
257 const struct rte_flow_item_udp *mask = item->mask;
258 struct mlx4_flow *flow = (struct mlx4_flow *)data;
259 struct ibv_flow_spec_tcp_udp *udp;
260 unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
262 ++flow->ibv_attr->num_of_specs;
263 flow->ibv_attr->priority = 0;
264 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
265 *udp = (struct ibv_flow_spec_tcp_udp) {
266 .type = IBV_FLOW_SPEC_UDP,
271 udp->val.dst_port = spec->hdr.dst_port;
272 udp->val.src_port = spec->hdr.src_port;
275 udp->mask.dst_port = mask->hdr.dst_port;
276 udp->mask.src_port = mask->hdr.src_port;
277 /* Remove unwanted bits from values. */
278 udp->val.src_port &= udp->mask.src_port;
279 udp->val.dst_port &= udp->mask.dst_port;
284 * Convert TCP item to Verbs specification.
287 * Item specification.
288 * @param default_mask[in]
289 * Default bit-masks to use when item->mask is not provided.
290 * @param data[in, out]
294 mlx4_flow_create_tcp(const struct rte_flow_item *item,
295 const void *default_mask,
298 const struct rte_flow_item_tcp *spec = item->spec;
299 const struct rte_flow_item_tcp *mask = item->mask;
300 struct mlx4_flow *flow = (struct mlx4_flow *)data;
301 struct ibv_flow_spec_tcp_udp *tcp;
302 unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
304 ++flow->ibv_attr->num_of_specs;
305 flow->ibv_attr->priority = 0;
306 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
307 *tcp = (struct ibv_flow_spec_tcp_udp) {
308 .type = IBV_FLOW_SPEC_TCP,
313 tcp->val.dst_port = spec->hdr.dst_port;
314 tcp->val.src_port = spec->hdr.src_port;
317 tcp->mask.dst_port = mask->hdr.dst_port;
318 tcp->mask.src_port = mask->hdr.src_port;
319 /* Remove unwanted bits from values. */
320 tcp->val.src_port &= tcp->mask.src_port;
321 tcp->val.dst_port &= tcp->mask.dst_port;
326 * Check support for a given item.
329 * Item specification.
331 * Bit-masks covering supported fields to compare with spec, last and mask in
334 * Bit-Mask size in bytes.
337 * 0 on success, negative value otherwise.
340 mlx4_flow_item_validate(const struct rte_flow_item *item,
341 const uint8_t *mask, unsigned int size)
345 if (!item->spec && (item->mask || item->last))
347 if (item->spec && !item->mask) {
349 const uint8_t *spec = item->spec;
351 for (i = 0; i < size; ++i)
352 if ((spec[i] | mask[i]) != mask[i])
355 if (item->last && !item->mask) {
357 const uint8_t *spec = item->last;
359 for (i = 0; i < size; ++i)
360 if ((spec[i] | mask[i]) != mask[i])
363 if (item->spec && item->last) {
366 const uint8_t *apply = mask;
371 for (i = 0; i < size; ++i) {
372 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
373 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
375 ret = memcmp(spec, last, size);
381 mlx4_flow_validate_eth(const struct rte_flow_item *item,
382 const uint8_t *mask, unsigned int size)
385 const struct rte_flow_item_eth *mask = item->mask;
387 if (mask->dst.addr_bytes[0] != 0xff ||
388 mask->dst.addr_bytes[1] != 0xff ||
389 mask->dst.addr_bytes[2] != 0xff ||
390 mask->dst.addr_bytes[3] != 0xff ||
391 mask->dst.addr_bytes[4] != 0xff ||
392 mask->dst.addr_bytes[5] != 0xff)
395 return mlx4_flow_item_validate(item, mask, size);
399 mlx4_flow_validate_vlan(const struct rte_flow_item *item,
400 const uint8_t *mask, unsigned int size)
403 const struct rte_flow_item_vlan *mask = item->mask;
405 if (mask->tci != 0 &&
406 ntohs(mask->tci) != 0x0fff)
409 return mlx4_flow_item_validate(item, mask, size);
413 mlx4_flow_validate_ipv4(const struct rte_flow_item *item,
414 const uint8_t *mask, unsigned int size)
417 const struct rte_flow_item_ipv4 *mask = item->mask;
419 if (mask->hdr.src_addr != 0 &&
420 mask->hdr.src_addr != 0xffffffff)
422 if (mask->hdr.dst_addr != 0 &&
423 mask->hdr.dst_addr != 0xffffffff)
426 return mlx4_flow_item_validate(item, mask, size);
430 mlx4_flow_validate_udp(const struct rte_flow_item *item,
431 const uint8_t *mask, unsigned int size)
434 const struct rte_flow_item_udp *mask = item->mask;
436 if (mask->hdr.src_port != 0 &&
437 mask->hdr.src_port != 0xffff)
439 if (mask->hdr.dst_port != 0 &&
440 mask->hdr.dst_port != 0xffff)
443 return mlx4_flow_item_validate(item, mask, size);
447 mlx4_flow_validate_tcp(const struct rte_flow_item *item,
448 const uint8_t *mask, unsigned int size)
451 const struct rte_flow_item_tcp *mask = item->mask;
453 if (mask->hdr.src_port != 0 &&
454 mask->hdr.src_port != 0xffff)
456 if (mask->hdr.dst_port != 0 &&
457 mask->hdr.dst_port != 0xffff)
460 return mlx4_flow_item_validate(item, mask, size);
463 /** Graph of supported items and associated actions. */
464 static const struct mlx4_flow_items mlx4_flow_items[] = {
465 [RTE_FLOW_ITEM_TYPE_END] = {
466 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
468 [RTE_FLOW_ITEM_TYPE_ETH] = {
469 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
470 RTE_FLOW_ITEM_TYPE_IPV4),
471 .actions = valid_actions,
472 .mask = &(const struct rte_flow_item_eth){
473 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
474 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
476 .default_mask = &rte_flow_item_eth_mask,
477 .mask_sz = sizeof(struct rte_flow_item_eth),
478 .validate = mlx4_flow_validate_eth,
479 .convert = mlx4_flow_create_eth,
480 .dst_sz = sizeof(struct ibv_flow_spec_eth),
482 [RTE_FLOW_ITEM_TYPE_VLAN] = {
483 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4),
484 .actions = valid_actions,
485 .mask = &(const struct rte_flow_item_vlan){
486 /* rte_flow_item_vlan_mask is invalid for mlx4. */
487 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
493 .mask_sz = sizeof(struct rte_flow_item_vlan),
494 .validate = mlx4_flow_validate_vlan,
495 .convert = mlx4_flow_create_vlan,
498 [RTE_FLOW_ITEM_TYPE_IPV4] = {
499 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
500 RTE_FLOW_ITEM_TYPE_TCP),
501 .actions = valid_actions,
502 .mask = &(const struct rte_flow_item_ipv4){
508 .default_mask = &rte_flow_item_ipv4_mask,
509 .mask_sz = sizeof(struct rte_flow_item_ipv4),
510 .validate = mlx4_flow_validate_ipv4,
511 .convert = mlx4_flow_create_ipv4,
512 .dst_sz = sizeof(struct ibv_flow_spec_ipv4),
514 [RTE_FLOW_ITEM_TYPE_UDP] = {
515 .actions = valid_actions,
516 .mask = &(const struct rte_flow_item_udp){
522 .default_mask = &rte_flow_item_udp_mask,
523 .mask_sz = sizeof(struct rte_flow_item_udp),
524 .validate = mlx4_flow_validate_udp,
525 .convert = mlx4_flow_create_udp,
526 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
528 [RTE_FLOW_ITEM_TYPE_TCP] = {
529 .actions = valid_actions,
530 .mask = &(const struct rte_flow_item_tcp){
536 .default_mask = &rte_flow_item_tcp_mask,
537 .mask_sz = sizeof(struct rte_flow_item_tcp),
538 .validate = mlx4_flow_validate_tcp,
539 .convert = mlx4_flow_create_tcp,
540 .dst_sz = sizeof(struct ibv_flow_spec_tcp_udp),
545 * Validate a flow supported by the NIC.
548 * Pointer to private structure.
550 * Flow rule attributes.
552 * Pattern specification (list terminated by the END pattern item).
554 * Associated actions (list terminated by the END action).
556 * Perform verbose error reporting if not NULL.
557 * @param[in, out] flow
558 * Flow structure to update.
561 * 0 on success, a negative errno value otherwise and rte_errno is set.
564 priv_flow_validate(struct priv *priv,
565 const struct rte_flow_attr *attr,
566 const struct rte_flow_item items[],
567 const struct rte_flow_action actions[],
568 struct rte_flow_error *error,
569 struct mlx4_flow *flow)
571 const struct mlx4_flow_items *cur_item = mlx4_flow_items;
572 struct mlx4_flow_action action = {
579 rte_flow_error_set(error, ENOTSUP,
580 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
582 "groups are not supported");
585 if (attr->priority) {
586 rte_flow_error_set(error, ENOTSUP,
587 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
589 "priorities are not supported");
593 rte_flow_error_set(error, ENOTSUP,
594 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
596 "egress is not supported");
599 if (!attr->ingress) {
600 rte_flow_error_set(error, ENOTSUP,
601 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
603 "only ingress is supported");
606 /* Go over items list. */
607 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
608 const struct mlx4_flow_items *token = NULL;
612 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
615 * The nic can support patterns with NULL eth spec only
616 * if eth is a single item in a rule.
619 items->type == RTE_FLOW_ITEM_TYPE_ETH) {
620 const struct rte_flow_item *next = items + 1;
622 if (next->type != RTE_FLOW_ITEM_TYPE_END) {
623 rte_flow_error_set(error, ENOTSUP,
624 RTE_FLOW_ERROR_TYPE_ITEM,
627 " an Ethernet spec");
633 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
635 if (cur_item->items[i] == items->type) {
636 token = &mlx4_flow_items[items->type];
641 goto exit_item_not_supported;
643 err = cur_item->validate(items,
644 (const uint8_t *)cur_item->mask,
647 goto exit_item_not_supported;
648 if (flow->ibv_attr && cur_item->convert) {
649 err = cur_item->convert(items,
650 (cur_item->default_mask ?
651 cur_item->default_mask :
655 goto exit_item_not_supported;
657 flow->offset += cur_item->dst_sz;
659 /* Go over actions list */
660 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
661 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
663 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
665 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
666 const struct rte_flow_action_queue *queue =
667 (const struct rte_flow_action_queue *)
670 if (!queue || (queue->index > (priv->rxqs_n - 1)))
671 goto exit_action_not_supported;
674 action.queues[0] = queue->index;
675 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
678 const struct rte_flow_action_rss *rss =
679 (const struct rte_flow_action_rss *)
683 rte_flow_error_set(error, ENOTSUP,
684 RTE_FLOW_ERROR_TYPE_ACTION,
686 "RSS cannot be used with "
687 "the current configuration");
690 if (!priv->isolated) {
691 rte_flow_error_set(error, ENOTSUP,
692 RTE_FLOW_ERROR_TYPE_ACTION,
694 "RSS cannot be used without "
698 if (!rte_is_power_of_2(rss->num)) {
699 rte_flow_error_set(error, ENOTSUP,
700 RTE_FLOW_ERROR_TYPE_ACTION,
702 "the number of queues "
703 "should be power of two");
706 if (priv->max_rss_tbl_sz < rss->num) {
707 rte_flow_error_set(error, ENOTSUP,
708 RTE_FLOW_ERROR_TYPE_ACTION,
710 "the number of queues "
714 /* checking indexes array */
716 for (i = 0; i < rss->num; ++i) {
718 if (rss->queue[i] >= priv->rxqs_n)
721 * Prevent the user from specifying
722 * the same queue twice in the RSS array.
724 for (j = i + 1; j < rss->num && !ierr; ++j)
725 if (rss->queue[j] == rss->queue[i])
731 RTE_FLOW_ERROR_TYPE_HANDLE,
733 "RSS action only supports "
734 "unique queue indices "
740 action.queues_n = rss->num;
741 for (i = 0; i < rss->num; ++i)
742 action.queues[i] = rss->queue[i];
744 goto exit_action_not_supported;
747 if (!action.queue && !action.drop) {
748 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
749 NULL, "no valid action");
753 exit_item_not_supported:
754 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
755 items, "item not supported");
757 exit_action_not_supported:
758 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
759 actions, "action not supported");
764 * Validate a flow supported by the NIC.
766 * @see rte_flow_validate()
770 mlx4_flow_validate(struct rte_eth_dev *dev,
771 const struct rte_flow_attr *attr,
772 const struct rte_flow_item items[],
773 const struct rte_flow_action actions[],
774 struct rte_flow_error *error)
776 struct priv *priv = dev->data->dev_private;
778 struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr) };
781 ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
787 * Destroy a drop queue.
790 * Pointer to private structure.
793 mlx4_flow_destroy_drop_queue(struct priv *priv)
795 if (priv->flow_drop_queue) {
796 struct rte_flow_drop *fdq = priv->flow_drop_queue;
798 priv->flow_drop_queue = NULL;
799 claim_zero(ibv_destroy_qp(fdq->qp));
800 claim_zero(ibv_destroy_cq(fdq->cq));
806 * Create a single drop queue for all drop flows.
809 * Pointer to private structure.
812 * 0 on success, negative value otherwise.
815 mlx4_flow_create_drop_queue(struct priv *priv)
819 struct rte_flow_drop *fdq;
821 fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
823 ERROR("Cannot allocate memory for drop struct");
826 cq = ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
827 &(struct ibv_exp_cq_init_attr){
831 ERROR("Cannot create drop CQ");
834 qp = ibv_exp_create_qp(priv->ctx,
835 &(struct ibv_exp_qp_init_attr){
842 .qp_type = IBV_QPT_RAW_PACKET,
844 IBV_EXP_QP_INIT_ATTR_PD |
845 IBV_EXP_QP_INIT_ATTR_PORT,
847 .port_num = priv->port,
850 ERROR("Cannot create drop QP");
853 *fdq = (struct rte_flow_drop){
857 priv->flow_drop_queue = fdq;
860 claim_zero(ibv_destroy_cq(cq));
868 * Get RSS parent rxq structure for given queues.
870 * Creates a new or returns an existed one.
873 * Pointer to private structure.
875 * queues indices array, NULL in default RSS case.
877 * the size of queues array.
880 * Pointer to a parent rxq structure, NULL on failure.
883 priv_parent_get(struct priv *priv,
886 struct rte_flow_error *error)
891 for (parent = LIST_FIRST(&priv->parents);
893 parent = LIST_NEXT(parent, next)) {
894 unsigned int same = 0;
895 unsigned int overlap = 0;
898 * Find out whether an appropriate parent queue already exists
899 * and can be reused, otherwise make sure there are no overlaps.
901 for (i = 0; i < children_n; ++i) {
904 for (j = 0; j < parent->rss.queues_n; ++j) {
905 if (parent->rss.queues[j] != queues[i])
912 if (same == children_n &&
913 children_n == parent->rss.queues_n)
918 /* Exclude the cases when some QPs were created without RSS */
919 for (i = 0; i < children_n; ++i) {
920 struct rxq *rxq = (*priv->rxqs)[queues[i]];
924 parent = priv_parent_create(priv, queues, children_n);
926 rte_flow_error_set(error,
927 ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
928 NULL, "flow rule creation failure");
934 rte_flow_error_set(error,
936 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
938 "sharing a queue between several"
939 " RSS groups is not supported");
944 * Complete flow rule creation.
947 * Pointer to private structure.
949 * Verbs flow attributes.
951 * Target action structure.
953 * Perform verbose error reporting if not NULL.
956 * A flow if the rule could be created.
958 static struct rte_flow *
959 priv_flow_create_action_queue(struct priv *priv,
960 struct ibv_flow_attr *ibv_attr,
961 struct mlx4_flow_action *action,
962 struct rte_flow_error *error)
965 struct rte_flow *rte_flow;
966 struct rxq *rxq_parent = NULL;
970 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
972 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
973 NULL, "cannot allocate flow memory");
977 qp = priv->flow_drop_queue ? priv->flow_drop_queue->qp : NULL;
981 struct rxq *rxq = NULL;
983 if (action->queues_n > 1) {
984 rxq_parent = priv_parent_get(priv, action->queues,
985 action->queues_n, error);
989 for (i = 0; i < action->queues_n; ++i) {
990 rxq = (*priv->rxqs)[action->queues[i]];
992 * In case of isolated mode we postpone
993 * ibv receive queue creation till the first
994 * rte_flow rule will be applied on that queue.
997 assert(priv->isolated);
998 ret = rxq_create_qp(rxq, rxq->elts_n,
1004 RTE_FLOW_ERROR_TYPE_HANDLE,
1006 "flow rule creation failure");
1011 qp = action->queues_n > 1 ? rxq_parent->qp : rxq->qp;
1014 rte_flow->ibv_attr = ibv_attr;
1017 rte_flow->ibv_flow = ibv_create_flow(qp, rte_flow->ibv_attr);
1018 if (!rte_flow->ibv_flow) {
1019 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1020 NULL, "flow rule creation failure");
1027 rxq_parent_cleanup(rxq_parent);
1036 * Pointer to private structure.
1038 * Flow rule attributes.
1040 * Pattern specification (list terminated by the END pattern item).
1041 * @param[in] actions
1042 * Associated actions (list terminated by the END action).
1044 * Perform verbose error reporting if not NULL.
1047 * A flow on success, NULL otherwise.
1049 static struct rte_flow *
1050 priv_flow_create(struct priv *priv,
1051 const struct rte_flow_attr *attr,
1052 const struct rte_flow_item items[],
1053 const struct rte_flow_action actions[],
1054 struct rte_flow_error *error)
1056 struct rte_flow *rte_flow;
1057 struct mlx4_flow_action action;
1058 struct mlx4_flow flow = { .offset = sizeof(struct ibv_flow_attr), };
1061 err = priv_flow_validate(priv, attr, items, actions, error, &flow);
1064 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
1065 if (!flow.ibv_attr) {
1066 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1067 NULL, "cannot allocate ibv_attr memory");
1070 flow.offset = sizeof(struct ibv_flow_attr);
1071 *flow.ibv_attr = (struct ibv_flow_attr){
1073 .type = IBV_FLOW_ATTR_NORMAL,
1074 .size = sizeof(struct ibv_flow_attr),
1075 .priority = attr->priority,
1080 claim_zero(priv_flow_validate(priv, attr, items, actions,
1082 action = (struct mlx4_flow_action){
1086 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1087 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1089 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1091 action.queues_n = 1;
1093 ((const struct rte_flow_action_queue *)
1094 actions->conf)->index;
1095 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1097 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
1099 const struct rte_flow_action_rss *rss =
1100 (const struct rte_flow_action_rss *)
1104 action.queues_n = rss->num;
1105 for (i = 0; i < rss->num; ++i)
1106 action.queues[i] = rss->queue[i];
1108 rte_flow_error_set(error, ENOTSUP,
1109 RTE_FLOW_ERROR_TYPE_ACTION,
1110 actions, "unsupported action");
1114 rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
1119 rte_free(flow.ibv_attr);
1126 * @see rte_flow_create()
1130 mlx4_flow_create(struct rte_eth_dev *dev,
1131 const struct rte_flow_attr *attr,
1132 const struct rte_flow_item items[],
1133 const struct rte_flow_action actions[],
1134 struct rte_flow_error *error)
1136 struct priv *priv = dev->data->dev_private;
1137 struct rte_flow *flow;
1140 flow = priv_flow_create(priv, attr, items, actions, error);
1142 LIST_INSERT_HEAD(&priv->flows, flow, next);
1143 DEBUG("Flow created %p", (void *)flow);
1150 * @see rte_flow_isolate()
1152 * Must be done before calling dev_configure().
1155 * Pointer to the ethernet device structure.
1157 * Nonzero to enter isolated mode, attempt to leave it otherwise.
1159 * Perform verbose error reporting if not NULL. PMDs initialize this
1160 * structure in case of error only.
1163 * 0 on success, a negative value on error.
1166 mlx4_flow_isolate(struct rte_eth_dev *dev,
1168 struct rte_flow_error *error)
1170 struct priv *priv = dev->data->dev_private;
1174 rte_flow_error_set(error, ENOTSUP,
1175 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1176 NULL, "isolated mode must be set"
1177 " before configuring the device");
1181 priv->isolated = !!enable;
1190 * Pointer to private structure.
1195 priv_flow_destroy(struct priv *priv, struct rte_flow *flow)
1198 LIST_REMOVE(flow, next);
1200 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1201 rte_free(flow->ibv_attr);
1202 DEBUG("Flow destroyed %p", (void *)flow);
1209 * @see rte_flow_destroy()
1213 mlx4_flow_destroy(struct rte_eth_dev *dev,
1214 struct rte_flow *flow,
1215 struct rte_flow_error *error)
1217 struct priv *priv = dev->data->dev_private;
1221 priv_flow_destroy(priv, flow);
1227 * Destroy all flows.
1230 * Pointer to private structure.
1233 priv_flow_flush(struct priv *priv)
1235 while (!LIST_EMPTY(&priv->flows)) {
1236 struct rte_flow *flow;
1238 flow = LIST_FIRST(&priv->flows);
1239 priv_flow_destroy(priv, flow);
1244 * Destroy all flows.
1246 * @see rte_flow_flush()
1250 mlx4_flow_flush(struct rte_eth_dev *dev,
1251 struct rte_flow_error *error)
1253 struct priv *priv = dev->data->dev_private;
1257 priv_flow_flush(priv);
1265 * Called by dev_stop() to remove all flows.
1268 * Pointer to private structure.
1271 mlx4_priv_flow_stop(struct priv *priv)
1273 struct rte_flow *flow;
1275 for (flow = LIST_FIRST(&priv->flows);
1277 flow = LIST_NEXT(flow, next)) {
1278 claim_zero(ibv_destroy_flow(flow->ibv_flow));
1279 flow->ibv_flow = NULL;
1280 DEBUG("Flow %p removed", (void *)flow);
1282 mlx4_flow_destroy_drop_queue(priv);
1289 * Pointer to private structure.
1292 * 0 on success, a errno value otherwise and rte_errno is set.
1295 mlx4_priv_flow_start(struct priv *priv)
1299 struct rte_flow *flow;
1301 ret = mlx4_flow_create_drop_queue(priv);
1304 for (flow = LIST_FIRST(&priv->flows);
1306 flow = LIST_NEXT(flow, next)) {
1307 qp = flow->qp ? flow->qp : priv->flow_drop_queue->qp;
1308 flow->ibv_flow = ibv_create_flow(qp, flow->ibv_attr);
1309 if (!flow->ibv_flow) {
1310 DEBUG("Flow %p cannot be applied", (void *)flow);
1314 DEBUG("Flow %p applied", (void *)flow);