4 * Copyright 2016 6WIND S.A.
5 * Copyright 2016 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
38 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
40 #pragma GCC diagnostic ignored "-Wpedantic"
42 #include <infiniband/verbs.h>
44 #pragma GCC diagnostic error "-Wpedantic"
47 #include <rte_ethdev.h>
49 #include <rte_flow_driver.h>
50 #include <rte_malloc.h>
55 mlx5_flow_create_eth(const struct rte_flow_item *item,
56 const void *default_mask,
60 mlx5_flow_create_vlan(const struct rte_flow_item *item,
61 const void *default_mask,
65 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
66 const void *default_mask,
70 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
71 const void *default_mask,
75 mlx5_flow_create_udp(const struct rte_flow_item *item,
76 const void *default_mask,
80 mlx5_flow_create_tcp(const struct rte_flow_item *item,
81 const void *default_mask,
85 LIST_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
86 struct ibv_exp_flow_attr *ibv_attr; /**< Pointer to Verbs attributes. */
87 struct ibv_exp_rwq_ind_table *ind_table; /**< Indirection table. */
88 struct ibv_qp *qp; /**< Verbs queue pair. */
89 struct ibv_exp_flow *ibv_flow; /**< Verbs flow. */
90 struct ibv_exp_wq *wq; /**< Verbs work queue. */
91 struct ibv_cq *cq; /**< Verbs completion queue. */
92 struct rxq *rxq; /**< Pointer to the queue, NULL if drop queue. */
95 /** Static initializer for items. */
97 (const enum rte_flow_item_type []){ \
98 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
101 /** Structure to generate a simple graph of layers supported by the NIC. */
102 struct mlx5_flow_items {
103 /** List of possible actions for these items. */
104 const enum rte_flow_action_type *const actions;
105 /** Bit-masks corresponding to the possibilities for the item. */
107 /** Bit-masks size in bytes. */
108 const unsigned int mask_sz;
110 * Conversion function from rte_flow to NIC specific flow.
113 * rte_flow item to convert.
114 * @param default_mask
115 * Default bit-masks to use when item->mask is not provided.
117 * Internal structure to store the conversion.
120 * 0 on success, negative value otherwise.
122 int (*convert)(const struct rte_flow_item *item,
123 const void *default_mask,
125 /** Size in bytes of the destination structure. */
126 const unsigned int dst_sz;
127 /** List of possible following items. */
128 const enum rte_flow_item_type *const items;
131 /** Valid action for this PMD. */
132 static const enum rte_flow_action_type valid_actions[] = {
133 RTE_FLOW_ACTION_TYPE_DROP,
134 RTE_FLOW_ACTION_TYPE_QUEUE,
135 RTE_FLOW_ACTION_TYPE_END,
138 /** Graph of supported items and associated actions. */
139 static const struct mlx5_flow_items mlx5_flow_items[] = {
140 [RTE_FLOW_ITEM_TYPE_END] = {
141 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
143 [RTE_FLOW_ITEM_TYPE_ETH] = {
144 .items = ITEMS(RTE_FLOW_ITEM_TYPE_VLAN,
145 RTE_FLOW_ITEM_TYPE_IPV4,
146 RTE_FLOW_ITEM_TYPE_IPV6),
147 .actions = valid_actions,
148 .mask = &(const struct rte_flow_item_eth){
149 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
150 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
152 .mask_sz = sizeof(struct rte_flow_item_eth),
153 .convert = mlx5_flow_create_eth,
154 .dst_sz = sizeof(struct ibv_exp_flow_spec_eth),
156 [RTE_FLOW_ITEM_TYPE_VLAN] = {
157 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
158 RTE_FLOW_ITEM_TYPE_IPV6),
159 .actions = valid_actions,
160 .mask = &(const struct rte_flow_item_vlan){
163 .mask_sz = sizeof(struct rte_flow_item_vlan),
164 .convert = mlx5_flow_create_vlan,
167 [RTE_FLOW_ITEM_TYPE_IPV4] = {
168 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
169 RTE_FLOW_ITEM_TYPE_TCP),
170 .actions = valid_actions,
171 .mask = &(const struct rte_flow_item_ipv4){
177 .mask_sz = sizeof(struct rte_flow_item_ipv4),
178 .convert = mlx5_flow_create_ipv4,
179 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv4),
181 [RTE_FLOW_ITEM_TYPE_IPV6] = {
182 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
183 RTE_FLOW_ITEM_TYPE_TCP),
184 .actions = valid_actions,
185 .mask = &(const struct rte_flow_item_ipv6){
188 0xff, 0xff, 0xff, 0xff,
189 0xff, 0xff, 0xff, 0xff,
190 0xff, 0xff, 0xff, 0xff,
191 0xff, 0xff, 0xff, 0xff,
194 0xff, 0xff, 0xff, 0xff,
195 0xff, 0xff, 0xff, 0xff,
196 0xff, 0xff, 0xff, 0xff,
197 0xff, 0xff, 0xff, 0xff,
201 .mask_sz = sizeof(struct rte_flow_item_ipv6),
202 .convert = mlx5_flow_create_ipv6,
203 .dst_sz = sizeof(struct ibv_exp_flow_spec_ipv6),
205 [RTE_FLOW_ITEM_TYPE_UDP] = {
206 .actions = valid_actions,
207 .mask = &(const struct rte_flow_item_udp){
213 .mask_sz = sizeof(struct rte_flow_item_udp),
214 .convert = mlx5_flow_create_udp,
215 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
217 [RTE_FLOW_ITEM_TYPE_TCP] = {
218 .actions = valid_actions,
219 .mask = &(const struct rte_flow_item_tcp){
225 .mask_sz = sizeof(struct rte_flow_item_tcp),
226 .convert = mlx5_flow_create_tcp,
227 .dst_sz = sizeof(struct ibv_exp_flow_spec_tcp_udp),
231 /** Structure to pass to the conversion function. */
233 struct ibv_exp_flow_attr *ibv_attr; /**< Verbs attribute. */
234 unsigned int offset; /**< Offset in bytes in the ibv_attr buffer. */
237 struct mlx5_flow_action {
238 uint32_t queue:1; /**< Target is a receive queue. */
239 uint32_t drop:1; /**< Target is a drop queue. */
240 uint32_t queue_id; /**< Identifier of the queue. */
244 * Check support for a given item.
247 * Item specification.
249 * Bit-masks covering supported fields to compare with spec, last and mask in
252 * Bit-Mask size in bytes.
258 mlx5_flow_item_validate(const struct rte_flow_item *item,
259 const uint8_t *mask, unsigned int size)
263 if (!item->spec && (item->mask || item->last))
265 if (item->spec && !item->mask) {
267 const uint8_t *spec = item->spec;
269 for (i = 0; i < size; ++i)
270 if ((spec[i] | mask[i]) != mask[i])
273 if (item->last && !item->mask) {
275 const uint8_t *spec = item->last;
277 for (i = 0; i < size; ++i)
278 if ((spec[i] | mask[i]) != mask[i])
283 const uint8_t *spec = item->mask;
285 for (i = 0; i < size; ++i)
286 if ((spec[i] | mask[i]) != mask[i])
289 if (item->spec && item->last) {
292 const uint8_t *apply = mask;
297 for (i = 0; i < size; ++i) {
298 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
299 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
301 ret = memcmp(spec, last, size);
307 * Validate a flow supported by the NIC.
310 * Pointer to private structure.
312 * Flow rule attributes.
314 * Pattern specification (list terminated by the END pattern item).
316 * Associated actions (list terminated by the END action).
318 * Perform verbose error reporting if not NULL.
319 * @param[in, out] flow
320 * Flow structure to update.
323 * 0 on success, a negative errno value otherwise and rte_errno is set.
326 priv_flow_validate(struct priv *priv,
327 const struct rte_flow_attr *attr,
328 const struct rte_flow_item items[],
329 const struct rte_flow_action actions[],
330 struct rte_flow_error *error,
331 struct mlx5_flow *flow)
333 const struct mlx5_flow_items *cur_item = mlx5_flow_items;
334 struct mlx5_flow_action action = {
341 rte_flow_error_set(error, ENOTSUP,
342 RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
344 "groups are not supported");
347 if (attr->priority) {
348 rte_flow_error_set(error, ENOTSUP,
349 RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
351 "priorities are not supported");
355 rte_flow_error_set(error, ENOTSUP,
356 RTE_FLOW_ERROR_TYPE_ATTR_EGRESS,
358 "egress is not supported");
361 if (!attr->ingress) {
362 rte_flow_error_set(error, ENOTSUP,
363 RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
365 "only ingress is supported");
368 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
369 const struct mlx5_flow_items *token = NULL;
373 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
375 /* Handle special situation for VLAN. */
376 if (items->type == RTE_FLOW_ITEM_TYPE_VLAN) {
377 if (((const struct rte_flow_item_vlan *)items)->tci >
379 rte_flow_error_set(error, ENOTSUP,
380 RTE_FLOW_ERROR_TYPE_ITEM,
382 "wrong VLAN id value");
388 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
390 if (cur_item->items[i] == items->type) {
391 token = &mlx5_flow_items[items->type];
396 goto exit_item_not_supported;
398 err = mlx5_flow_item_validate(items,
399 (const uint8_t *)cur_item->mask,
400 sizeof(cur_item->mask_sz));
402 goto exit_item_not_supported;
403 if (flow->ibv_attr && cur_item->convert) {
404 err = cur_item->convert(items, cur_item->mask, flow);
406 goto exit_item_not_supported;
408 flow->offset += cur_item->dst_sz;
410 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
411 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
413 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
415 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
416 const struct rte_flow_action_queue *queue =
417 (const struct rte_flow_action_queue *)
420 if (!queue || (queue->index > (priv->rxqs_n - 1)))
421 goto exit_action_not_supported;
424 goto exit_action_not_supported;
427 if (!action.queue && !action.drop) {
428 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
429 NULL, "no valid action");
433 exit_item_not_supported:
434 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
435 items, "item not supported");
437 exit_action_not_supported:
438 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
439 actions, "action not supported");
444 * Validate a flow supported by the NIC.
446 * @see rte_flow_validate()
450 mlx5_flow_validate(struct rte_eth_dev *dev,
451 const struct rte_flow_attr *attr,
452 const struct rte_flow_item items[],
453 const struct rte_flow_action actions[],
454 struct rte_flow_error *error)
456 struct priv *priv = dev->data->dev_private;
458 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr) };
461 ret = priv_flow_validate(priv, attr, items, actions, error, &flow);
467 * Convert Ethernet item to Verbs specification.
470 * Item specification.
471 * @param default_mask[in]
472 * Default bit-masks to use when item->mask is not provided.
473 * @param data[in, out]
477 mlx5_flow_create_eth(const struct rte_flow_item *item,
478 const void *default_mask,
481 const struct rte_flow_item_eth *spec = item->spec;
482 const struct rte_flow_item_eth *mask = item->mask;
483 struct mlx5_flow *flow = (struct mlx5_flow *)data;
484 struct ibv_exp_flow_spec_eth *eth;
485 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
488 ++flow->ibv_attr->num_of_specs;
489 flow->ibv_attr->priority = 2;
490 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
491 *eth = (struct ibv_exp_flow_spec_eth) {
492 .type = IBV_EXP_FLOW_SPEC_ETH,
499 memcpy(eth->val.dst_mac, spec->dst.addr_bytes, ETHER_ADDR_LEN);
500 memcpy(eth->val.src_mac, spec->src.addr_bytes, ETHER_ADDR_LEN);
501 memcpy(eth->mask.dst_mac, mask->dst.addr_bytes, ETHER_ADDR_LEN);
502 memcpy(eth->mask.src_mac, mask->src.addr_bytes, ETHER_ADDR_LEN);
503 /* Remove unwanted bits from values. */
504 for (i = 0; i < ETHER_ADDR_LEN; ++i) {
505 eth->val.dst_mac[i] &= eth->mask.dst_mac[i];
506 eth->val.src_mac[i] &= eth->mask.src_mac[i];
512 * Convert VLAN item to Verbs specification.
515 * Item specification.
516 * @param default_mask[in]
517 * Default bit-masks to use when item->mask is not provided.
518 * @param data[in, out]
522 mlx5_flow_create_vlan(const struct rte_flow_item *item,
523 const void *default_mask,
526 const struct rte_flow_item_vlan *spec = item->spec;
527 const struct rte_flow_item_vlan *mask = item->mask;
528 struct mlx5_flow *flow = (struct mlx5_flow *)data;
529 struct ibv_exp_flow_spec_eth *eth;
530 const unsigned int eth_size = sizeof(struct ibv_exp_flow_spec_eth);
532 eth = (void *)((uintptr_t)flow->ibv_attr + flow->offset - eth_size);
537 eth->val.vlan_tag = spec->tci;
538 eth->mask.vlan_tag = mask->tci;
539 eth->val.vlan_tag &= eth->mask.vlan_tag;
544 * Convert IPv4 item to Verbs specification.
547 * Item specification.
548 * @param default_mask[in]
549 * Default bit-masks to use when item->mask is not provided.
550 * @param data[in, out]
554 mlx5_flow_create_ipv4(const struct rte_flow_item *item,
555 const void *default_mask,
558 const struct rte_flow_item_ipv4 *spec = item->spec;
559 const struct rte_flow_item_ipv4 *mask = item->mask;
560 struct mlx5_flow *flow = (struct mlx5_flow *)data;
561 struct ibv_exp_flow_spec_ipv4 *ipv4;
562 unsigned int ipv4_size = sizeof(struct ibv_exp_flow_spec_ipv4);
564 ++flow->ibv_attr->num_of_specs;
565 flow->ibv_attr->priority = 1;
566 ipv4 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
567 *ipv4 = (struct ibv_exp_flow_spec_ipv4) {
568 .type = IBV_EXP_FLOW_SPEC_IPV4,
575 ipv4->val = (struct ibv_exp_flow_ipv4_filter){
576 .src_ip = spec->hdr.src_addr,
577 .dst_ip = spec->hdr.dst_addr,
579 ipv4->mask = (struct ibv_exp_flow_ipv4_filter){
580 .src_ip = mask->hdr.src_addr,
581 .dst_ip = mask->hdr.dst_addr,
583 /* Remove unwanted bits from values. */
584 ipv4->val.src_ip &= ipv4->mask.src_ip;
585 ipv4->val.dst_ip &= ipv4->mask.dst_ip;
590 * Convert IPv6 item to Verbs specification.
593 * Item specification.
594 * @param default_mask[in]
595 * Default bit-masks to use when item->mask is not provided.
596 * @param data[in, out]
600 mlx5_flow_create_ipv6(const struct rte_flow_item *item,
601 const void *default_mask,
604 const struct rte_flow_item_ipv6 *spec = item->spec;
605 const struct rte_flow_item_ipv6 *mask = item->mask;
606 struct mlx5_flow *flow = (struct mlx5_flow *)data;
607 struct ibv_exp_flow_spec_ipv6 *ipv6;
608 unsigned int ipv6_size = sizeof(struct ibv_exp_flow_spec_ipv6);
611 ++flow->ibv_attr->num_of_specs;
612 flow->ibv_attr->priority = 1;
613 ipv6 = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
614 *ipv6 = (struct ibv_exp_flow_spec_ipv6) {
615 .type = IBV_EXP_FLOW_SPEC_IPV6,
622 memcpy(ipv6->val.src_ip, spec->hdr.src_addr,
623 RTE_DIM(ipv6->val.src_ip));
624 memcpy(ipv6->val.dst_ip, spec->hdr.dst_addr,
625 RTE_DIM(ipv6->val.dst_ip));
626 memcpy(ipv6->mask.src_ip, mask->hdr.src_addr,
627 RTE_DIM(ipv6->mask.src_ip));
628 memcpy(ipv6->mask.dst_ip, mask->hdr.dst_addr,
629 RTE_DIM(ipv6->mask.dst_ip));
630 /* Remove unwanted bits from values. */
631 for (i = 0; i < RTE_DIM(ipv6->val.src_ip); ++i) {
632 ipv6->val.src_ip[i] &= ipv6->mask.src_ip[i];
633 ipv6->val.dst_ip[i] &= ipv6->mask.dst_ip[i];
639 * Convert UDP item to Verbs specification.
642 * Item specification.
643 * @param default_mask[in]
644 * Default bit-masks to use when item->mask is not provided.
645 * @param data[in, out]
649 mlx5_flow_create_udp(const struct rte_flow_item *item,
650 const void *default_mask,
653 const struct rte_flow_item_udp *spec = item->spec;
654 const struct rte_flow_item_udp *mask = item->mask;
655 struct mlx5_flow *flow = (struct mlx5_flow *)data;
656 struct ibv_exp_flow_spec_tcp_udp *udp;
657 unsigned int udp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
659 ++flow->ibv_attr->num_of_specs;
660 flow->ibv_attr->priority = 0;
661 udp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
662 *udp = (struct ibv_exp_flow_spec_tcp_udp) {
663 .type = IBV_EXP_FLOW_SPEC_UDP,
670 udp->val.dst_port = spec->hdr.dst_port;
671 udp->val.src_port = spec->hdr.src_port;
672 udp->mask.dst_port = mask->hdr.dst_port;
673 udp->mask.src_port = mask->hdr.src_port;
674 /* Remove unwanted bits from values. */
675 udp->val.src_port &= udp->mask.src_port;
676 udp->val.dst_port &= udp->mask.dst_port;
681 * Convert TCP item to Verbs specification.
684 * Item specification.
685 * @param default_mask[in]
686 * Default bit-masks to use when item->mask is not provided.
687 * @param data[in, out]
691 mlx5_flow_create_tcp(const struct rte_flow_item *item,
692 const void *default_mask,
695 const struct rte_flow_item_tcp *spec = item->spec;
696 const struct rte_flow_item_tcp *mask = item->mask;
697 struct mlx5_flow *flow = (struct mlx5_flow *)data;
698 struct ibv_exp_flow_spec_tcp_udp *tcp;
699 unsigned int tcp_size = sizeof(struct ibv_exp_flow_spec_tcp_udp);
701 ++flow->ibv_attr->num_of_specs;
702 flow->ibv_attr->priority = 0;
703 tcp = (void *)((uintptr_t)flow->ibv_attr + flow->offset);
704 *tcp = (struct ibv_exp_flow_spec_tcp_udp) {
705 .type = IBV_EXP_FLOW_SPEC_TCP,
712 tcp->val.dst_port = spec->hdr.dst_port;
713 tcp->val.src_port = spec->hdr.src_port;
714 tcp->mask.dst_port = mask->hdr.dst_port;
715 tcp->mask.src_port = mask->hdr.src_port;
716 /* Remove unwanted bits from values. */
717 tcp->val.src_port &= tcp->mask.src_port;
718 tcp->val.dst_port &= tcp->mask.dst_port;
723 * Complete flow rule creation.
726 * Pointer to private structure.
728 * Verbs flow attributes.
730 * Target action structure.
732 * Perform verbose error reporting if not NULL.
735 * A flow if the rule could be created.
737 static struct rte_flow *
738 priv_flow_create_action_queue(struct priv *priv,
739 struct ibv_exp_flow_attr *ibv_attr,
740 struct mlx5_flow_action *action,
741 struct rte_flow_error *error)
743 struct rxq_ctrl *rxq;
744 struct rte_flow *rte_flow;
748 rte_flow = rte_calloc(__func__, 1, sizeof(*rte_flow), 0);
750 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
751 NULL, "cannot allocate flow memory");
756 ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0,
757 &(struct ibv_exp_cq_init_attr){
761 rte_flow_error_set(error, ENOMEM,
762 RTE_FLOW_ERROR_TYPE_HANDLE,
763 NULL, "cannot allocate CQ");
766 rte_flow->wq = ibv_exp_create_wq(priv->ctx,
767 &(struct ibv_exp_wq_init_attr){
768 .wq_type = IBV_EXP_WQT_RQ,
775 rxq = container_of((*priv->rxqs)[action->queue_id],
776 struct rxq_ctrl, rxq);
777 rte_flow->rxq = &rxq->rxq;
778 rte_flow->wq = rxq->wq;
780 rte_flow->ibv_attr = ibv_attr;
781 rte_flow->ind_table = ibv_exp_create_rwq_ind_table(
783 &(struct ibv_exp_rwq_ind_table_init_attr){
785 .log_ind_tbl_size = 0,
786 .ind_tbl = &rte_flow->wq,
789 if (!rte_flow->ind_table) {
790 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
791 NULL, "cannot allocate indirection table");
794 rte_flow->qp = ibv_exp_create_qp(
796 &(struct ibv_exp_qp_init_attr){
797 .qp_type = IBV_QPT_RAW_PACKET,
799 IBV_EXP_QP_INIT_ATTR_PD |
800 IBV_EXP_QP_INIT_ATTR_PORT |
801 IBV_EXP_QP_INIT_ATTR_RX_HASH,
803 .rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
805 IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
806 .rx_hash_key_len = rss_hash_default_key_len,
807 .rx_hash_key = rss_hash_default_key,
808 .rx_hash_fields_mask = 0,
809 .rwq_ind_tbl = rte_flow->ind_table,
811 .port_num = priv->port,
814 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
815 NULL, "cannot allocate QP");
818 rte_flow->ibv_flow = ibv_exp_create_flow(rte_flow->qp,
820 if (!rte_flow->ibv_flow) {
821 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
822 NULL, "flow rule creation failure");
829 ibv_destroy_qp(rte_flow->qp);
830 if (rte_flow->ind_table)
831 ibv_exp_destroy_rwq_ind_table(rte_flow->ind_table);
832 if (!rte_flow->rxq && rte_flow->wq)
833 ibv_exp_destroy_wq(rte_flow->wq);
834 if (!rte_flow->rxq && rte_flow->cq)
835 ibv_destroy_cq(rte_flow->cq);
836 rte_free(rte_flow->ibv_attr);
845 * Pointer to private structure.
847 * Flow rule attributes.
849 * Pattern specification (list terminated by the END pattern item).
851 * Associated actions (list terminated by the END action).
853 * Perform verbose error reporting if not NULL.
856 * A flow on success, NULL otherwise.
858 static struct rte_flow *
859 priv_flow_create(struct priv *priv,
860 const struct rte_flow_attr *attr,
861 const struct rte_flow_item items[],
862 const struct rte_flow_action actions[],
863 struct rte_flow_error *error)
865 struct rte_flow *rte_flow;
866 struct mlx5_flow_action action;
867 struct mlx5_flow flow = { .offset = sizeof(struct ibv_exp_flow_attr), };
870 err = priv_flow_validate(priv, attr, items, actions, error, &flow);
873 flow.ibv_attr = rte_malloc(__func__, flow.offset, 0);
874 flow.offset = sizeof(struct ibv_exp_flow_attr);
875 if (!flow.ibv_attr) {
876 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
877 NULL, "cannot allocate ibv_attr memory");
880 *flow.ibv_attr = (struct ibv_exp_flow_attr){
881 .type = IBV_EXP_FLOW_ATTR_NORMAL,
882 .size = sizeof(struct ibv_exp_flow_attr),
883 .priority = attr->priority,
889 claim_zero(priv_flow_validate(priv, attr, items, actions,
891 action = (struct mlx5_flow_action){
895 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
896 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
898 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
901 ((const struct rte_flow_action_queue *)
902 actions->conf)->index;
903 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
906 rte_flow_error_set(error, ENOTSUP,
907 RTE_FLOW_ERROR_TYPE_ACTION,
908 actions, "unsupported action");
912 rte_flow = priv_flow_create_action_queue(priv, flow.ibv_attr,
916 rte_free(flow.ibv_attr);
923 * @see rte_flow_create()
927 mlx5_flow_create(struct rte_eth_dev *dev,
928 const struct rte_flow_attr *attr,
929 const struct rte_flow_item items[],
930 const struct rte_flow_action actions[],
931 struct rte_flow_error *error)
933 struct priv *priv = dev->data->dev_private;
934 struct rte_flow *flow;
937 flow = priv_flow_create(priv, attr, items, actions, error);
939 LIST_INSERT_HEAD(&priv->flows, flow, next);
940 DEBUG("Flow created %p", (void *)flow);
950 * Pointer to private structure.
955 priv_flow_destroy(struct priv *priv,
956 struct rte_flow *flow)
959 LIST_REMOVE(flow, next);
961 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
963 claim_zero(ibv_destroy_qp(flow->qp));
965 claim_zero(ibv_exp_destroy_rwq_ind_table(flow->ind_table));
966 if (!flow->rxq && flow->wq)
967 claim_zero(ibv_exp_destroy_wq(flow->wq));
968 if (!flow->rxq && flow->cq)
969 claim_zero(ibv_destroy_cq(flow->cq));
970 rte_free(flow->ibv_attr);
971 DEBUG("Flow destroyed %p", (void *)flow);
978 * @see rte_flow_destroy()
982 mlx5_flow_destroy(struct rte_eth_dev *dev,
983 struct rte_flow *flow,
984 struct rte_flow_error *error)
986 struct priv *priv = dev->data->dev_private;
990 priv_flow_destroy(priv, flow);
999 * Pointer to private structure.
1002 priv_flow_flush(struct priv *priv)
1004 while (!LIST_EMPTY(&priv->flows)) {
1005 struct rte_flow *flow;
1007 flow = LIST_FIRST(&priv->flows);
1008 priv_flow_destroy(priv, flow);
1013 * Destroy all flows.
1015 * @see rte_flow_flush()
1019 mlx5_flow_flush(struct rte_eth_dev *dev,
1020 struct rte_flow_error *error)
1022 struct priv *priv = dev->data->dev_private;
1026 priv_flow_flush(priv);
1034 * Called by dev_stop() to remove all flows.
1037 * Pointer to private structure.
1040 priv_flow_stop(struct priv *priv)
1042 struct rte_flow *flow;
1044 for (flow = LIST_FIRST(&priv->flows);
1046 flow = LIST_NEXT(flow, next)) {
1047 claim_zero(ibv_exp_destroy_flow(flow->ibv_flow));
1048 flow->ibv_flow = NULL;
1049 DEBUG("Flow %p removed", (void *)flow);
1057 * Pointer to private structure.
1060 * 0 on success, a errno value otherwise and rte_errno is set.
1063 priv_flow_start(struct priv *priv)
1065 struct rte_flow *flow;
1067 for (flow = LIST_FIRST(&priv->flows);
1069 flow = LIST_NEXT(flow, next)) {
1070 flow->ibv_flow = ibv_exp_create_flow(flow->qp,
1072 if (!flow->ibv_flow) {
1073 DEBUG("Flow %p cannot be applied", (void *)flow);
1077 DEBUG("Flow %p applied", (void *)flow);