1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2016 6WIND S.A.
3 * Copyright 2016 Mellanox Technologies, Ltd
11 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
13 #pragma GCC diagnostic ignored "-Wpedantic"
15 #include <infiniband/verbs.h>
17 #pragma GCC diagnostic error "-Wpedantic"
20 #include <rte_common.h>
21 #include <rte_ether.h>
22 #include <rte_eth_ctrl.h>
23 #include <rte_ethdev_driver.h>
25 #include <rte_flow_driver.h>
26 #include <rte_malloc.h>
30 #include "mlx5_defs.h"
32 #include "mlx5_glue.h"
34 /* Dev ops structure defined in mlx5.c */
35 extern const struct eth_dev_ops mlx5_dev_ops;
36 extern const struct eth_dev_ops mlx5_dev_ops_isolate;
39 TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
42 static const struct rte_flow_ops mlx5_flow_ops = {
43 .isolate = mlx5_flow_isolate,
46 /* Convert FDIR request to Generic flow. */
48 struct rte_flow_attr attr;
49 struct rte_flow_action actions[2];
50 struct rte_flow_item items[4];
51 struct rte_flow_item_eth l2;
52 struct rte_flow_item_eth l2_mask;
54 struct rte_flow_item_ipv4 ipv4;
55 struct rte_flow_item_ipv6 ipv6;
58 struct rte_flow_item_ipv4 ipv4;
59 struct rte_flow_item_ipv6 ipv6;
62 struct rte_flow_item_udp udp;
63 struct rte_flow_item_tcp tcp;
66 struct rte_flow_item_udp udp;
67 struct rte_flow_item_tcp tcp;
69 struct rte_flow_action_queue queue;
72 /* Verbs specification header. */
73 struct ibv_spec_header {
74 enum ibv_flow_spec_type type;
82 * Pointer to Ethernet device.
84 * Pointer to a TAILQ flow list.
86 * Flow rule attributes.
88 * Pattern specification (list terminated by the END pattern item).
90 * Associated actions (list terminated by the END action).
92 * Perform verbose error reporting if not NULL.
95 * A flow on success, NULL otherwise and rte_errno is set.
97 static struct rte_flow *
98 mlx5_flow_list_create(struct rte_eth_dev *dev __rte_unused,
99 struct mlx5_flows *list __rte_unused,
100 const struct rte_flow_attr *attr __rte_unused,
101 const struct rte_flow_item items[] __rte_unused,
102 const struct rte_flow_action actions[] __rte_unused,
103 struct rte_flow_error *error)
105 rte_flow_error_set(error, ENOTSUP,
106 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
108 "action not supported");
113 * Validate a flow supported by the NIC.
115 * @see rte_flow_validate()
119 mlx5_flow_validate(struct rte_eth_dev *dev __rte_unused,
120 const struct rte_flow_attr *attr __rte_unused,
121 const struct rte_flow_item items[] __rte_unused,
122 const struct rte_flow_action actions[] __rte_unused,
123 struct rte_flow_error *error)
125 return rte_flow_error_set(error, ENOTSUP,
126 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
128 "action not supported");
134 * @see rte_flow_create()
138 mlx5_flow_create(struct rte_eth_dev *dev __rte_unused,
139 const struct rte_flow_attr *attr __rte_unused,
140 const struct rte_flow_item items[] __rte_unused,
141 const struct rte_flow_action actions[] __rte_unused,
142 struct rte_flow_error *error)
144 rte_flow_error_set(error, ENOTSUP,
145 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
147 "action not supported");
152 * Destroy a flow in a list.
155 * Pointer to Ethernet device.
157 * Pointer to a TAILQ flow list.
162 mlx5_flow_list_destroy(struct rte_eth_dev *dev __rte_unused,
163 struct mlx5_flows *list __rte_unused,
164 struct rte_flow *flow __rte_unused)
172 * Pointer to Ethernet device.
174 * Pointer to a TAILQ flow list.
177 mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
179 while (!TAILQ_EMPTY(list)) {
180 struct rte_flow *flow;
182 flow = TAILQ_FIRST(list);
183 mlx5_flow_list_destroy(dev, list, flow);
191 * Pointer to Ethernet device.
194 * 0 on success, a negative errno value otherwise and rte_errno is set.
197 mlx5_flow_create_drop_queue(struct rte_eth_dev *dev __rte_unused)
206 * Pointer to Ethernet device.
209 mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev __rte_unused)
217 * Pointer to Ethernet device.
219 * Pointer to a TAILQ flow list.
222 mlx5_flow_stop(struct rte_eth_dev *dev __rte_unused,
223 struct mlx5_flows *list __rte_unused)
231 * Pointer to Ethernet device.
233 * Pointer to a TAILQ flow list.
236 * 0 on success, a negative errno value otherwise and rte_errno is set.
239 mlx5_flow_start(struct rte_eth_dev *dev __rte_unused,
240 struct mlx5_flows *list __rte_unused)
246 * Verify the flow list is empty
249 * Pointer to Ethernet device.
251 * @return the number of flows not released.
254 mlx5_flow_verify(struct rte_eth_dev *dev)
256 struct priv *priv = dev->data->dev_private;
257 struct rte_flow *flow;
260 TAILQ_FOREACH(flow, &priv->flows, next) {
261 DRV_LOG(DEBUG, "port %u flow %p still referenced",
262 dev->data->port_id, (void *)flow);
269 * Enable a control flow configured from the control plane.
272 * Pointer to Ethernet device.
274 * An Ethernet flow spec to apply.
276 * An Ethernet flow mask to apply.
278 * A VLAN flow spec to apply.
280 * A VLAN flow mask to apply.
283 * 0 on success, a negative errno value otherwise and rte_errno is set.
286 mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
287 struct rte_flow_item_eth *eth_spec,
288 struct rte_flow_item_eth *eth_mask,
289 struct rte_flow_item_vlan *vlan_spec,
290 struct rte_flow_item_vlan *vlan_mask)
292 struct priv *priv = dev->data->dev_private;
293 const struct rte_flow_attr attr = {
296 struct rte_flow_item items[] = {
298 .type = RTE_FLOW_ITEM_TYPE_ETH,
304 .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
305 RTE_FLOW_ITEM_TYPE_END,
311 .type = RTE_FLOW_ITEM_TYPE_END,
314 uint16_t queue[priv->reta_idx_n];
315 struct rte_flow_action_rss action_rss = {
316 .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
318 .types = priv->rss_conf.rss_hf,
319 .key_len = priv->rss_conf.rss_key_len,
320 .queue_num = priv->reta_idx_n,
321 .key = priv->rss_conf.rss_key,
324 struct rte_flow_action actions[] = {
326 .type = RTE_FLOW_ACTION_TYPE_RSS,
330 .type = RTE_FLOW_ACTION_TYPE_END,
333 struct rte_flow *flow;
334 struct rte_flow_error error;
337 if (!priv->reta_idx_n) {
341 for (i = 0; i != priv->reta_idx_n; ++i)
342 queue[i] = (*priv->reta_idx)[i];
343 flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
351 * Enable a flow control configured from the control plane.
354 * Pointer to Ethernet device.
356 * An Ethernet flow spec to apply.
358 * An Ethernet flow mask to apply.
361 * 0 on success, a negative errno value otherwise and rte_errno is set.
364 mlx5_ctrl_flow(struct rte_eth_dev *dev,
365 struct rte_flow_item_eth *eth_spec,
366 struct rte_flow_item_eth *eth_mask)
368 return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
374 * @see rte_flow_destroy()
378 mlx5_flow_destroy(struct rte_eth_dev *dev,
379 struct rte_flow *flow,
380 struct rte_flow_error *error __rte_unused)
382 struct priv *priv = dev->data->dev_private;
384 mlx5_flow_list_destroy(dev, &priv->flows, flow);
391 * @see rte_flow_flush()
395 mlx5_flow_flush(struct rte_eth_dev *dev,
396 struct rte_flow_error *error __rte_unused)
398 struct priv *priv = dev->data->dev_private;
400 mlx5_flow_list_flush(dev, &priv->flows);
407 * @see rte_flow_isolate()
411 mlx5_flow_isolate(struct rte_eth_dev *dev,
413 struct rte_flow_error *error)
415 struct priv *priv = dev->data->dev_private;
417 if (dev->data->dev_started) {
418 rte_flow_error_set(error, EBUSY,
419 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
421 "port must be stopped first");
424 priv->isolated = !!enable;
426 dev->dev_ops = &mlx5_dev_ops_isolate;
428 dev->dev_ops = &mlx5_dev_ops;
433 * Convert a flow director filter to a generic flow.
436 * Pointer to Ethernet device.
438 * Flow director filter to add.
440 * Generic flow parameters structure.
443 * 0 on success, a negative errno value otherwise and rte_errno is set.
446 mlx5_fdir_filter_convert(struct rte_eth_dev *dev,
447 const struct rte_eth_fdir_filter *fdir_filter,
448 struct mlx5_fdir *attributes)
450 struct priv *priv = dev->data->dev_private;
451 const struct rte_eth_fdir_input *input = &fdir_filter->input;
452 const struct rte_eth_fdir_masks *mask =
453 &dev->data->dev_conf.fdir_conf.mask;
455 /* Validate queue number. */
456 if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
457 DRV_LOG(ERR, "port %u invalid queue number %d",
458 dev->data->port_id, fdir_filter->action.rx_queue);
462 attributes->attr.ingress = 1;
463 attributes->items[0] = (struct rte_flow_item) {
464 .type = RTE_FLOW_ITEM_TYPE_ETH,
465 .spec = &attributes->l2,
466 .mask = &attributes->l2_mask,
468 switch (fdir_filter->action.behavior) {
469 case RTE_ETH_FDIR_ACCEPT:
470 attributes->actions[0] = (struct rte_flow_action){
471 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
472 .conf = &attributes->queue,
475 case RTE_ETH_FDIR_REJECT:
476 attributes->actions[0] = (struct rte_flow_action){
477 .type = RTE_FLOW_ACTION_TYPE_DROP,
481 DRV_LOG(ERR, "port %u invalid behavior %d",
483 fdir_filter->action.behavior);
487 attributes->queue.index = fdir_filter->action.rx_queue;
489 switch (fdir_filter->input.flow_type) {
490 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
491 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
492 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
493 attributes->l3.ipv4.hdr = (struct ipv4_hdr){
494 .src_addr = input->flow.ip4_flow.src_ip,
495 .dst_addr = input->flow.ip4_flow.dst_ip,
496 .time_to_live = input->flow.ip4_flow.ttl,
497 .type_of_service = input->flow.ip4_flow.tos,
498 .next_proto_id = input->flow.ip4_flow.proto,
500 attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){
501 .src_addr = mask->ipv4_mask.src_ip,
502 .dst_addr = mask->ipv4_mask.dst_ip,
503 .time_to_live = mask->ipv4_mask.ttl,
504 .type_of_service = mask->ipv4_mask.tos,
505 .next_proto_id = mask->ipv4_mask.proto,
507 attributes->items[1] = (struct rte_flow_item){
508 .type = RTE_FLOW_ITEM_TYPE_IPV4,
509 .spec = &attributes->l3,
510 .mask = &attributes->l3_mask,
513 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
514 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
515 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
516 attributes->l3.ipv6.hdr = (struct ipv6_hdr){
517 .hop_limits = input->flow.ipv6_flow.hop_limits,
518 .proto = input->flow.ipv6_flow.proto,
521 memcpy(attributes->l3.ipv6.hdr.src_addr,
522 input->flow.ipv6_flow.src_ip,
523 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
524 memcpy(attributes->l3.ipv6.hdr.dst_addr,
525 input->flow.ipv6_flow.dst_ip,
526 RTE_DIM(attributes->l3.ipv6.hdr.src_addr));
527 memcpy(attributes->l3_mask.ipv6.hdr.src_addr,
528 mask->ipv6_mask.src_ip,
529 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
530 memcpy(attributes->l3_mask.ipv6.hdr.dst_addr,
531 mask->ipv6_mask.dst_ip,
532 RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr));
533 attributes->items[1] = (struct rte_flow_item){
534 .type = RTE_FLOW_ITEM_TYPE_IPV6,
535 .spec = &attributes->l3,
536 .mask = &attributes->l3_mask,
540 DRV_LOG(ERR, "port %u invalid flow type%d",
541 dev->data->port_id, fdir_filter->input.flow_type);
546 switch (fdir_filter->input.flow_type) {
547 case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
548 attributes->l4.udp.hdr = (struct udp_hdr){
549 .src_port = input->flow.udp4_flow.src_port,
550 .dst_port = input->flow.udp4_flow.dst_port,
552 attributes->l4_mask.udp.hdr = (struct udp_hdr){
553 .src_port = mask->src_port_mask,
554 .dst_port = mask->dst_port_mask,
556 attributes->items[2] = (struct rte_flow_item){
557 .type = RTE_FLOW_ITEM_TYPE_UDP,
558 .spec = &attributes->l4,
559 .mask = &attributes->l4_mask,
562 case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
563 attributes->l4.tcp.hdr = (struct tcp_hdr){
564 .src_port = input->flow.tcp4_flow.src_port,
565 .dst_port = input->flow.tcp4_flow.dst_port,
567 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
568 .src_port = mask->src_port_mask,
569 .dst_port = mask->dst_port_mask,
571 attributes->items[2] = (struct rte_flow_item){
572 .type = RTE_FLOW_ITEM_TYPE_TCP,
573 .spec = &attributes->l4,
574 .mask = &attributes->l4_mask,
577 case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
578 attributes->l4.udp.hdr = (struct udp_hdr){
579 .src_port = input->flow.udp6_flow.src_port,
580 .dst_port = input->flow.udp6_flow.dst_port,
582 attributes->l4_mask.udp.hdr = (struct udp_hdr){
583 .src_port = mask->src_port_mask,
584 .dst_port = mask->dst_port_mask,
586 attributes->items[2] = (struct rte_flow_item){
587 .type = RTE_FLOW_ITEM_TYPE_UDP,
588 .spec = &attributes->l4,
589 .mask = &attributes->l4_mask,
592 case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
593 attributes->l4.tcp.hdr = (struct tcp_hdr){
594 .src_port = input->flow.tcp6_flow.src_port,
595 .dst_port = input->flow.tcp6_flow.dst_port,
597 attributes->l4_mask.tcp.hdr = (struct tcp_hdr){
598 .src_port = mask->src_port_mask,
599 .dst_port = mask->dst_port_mask,
601 attributes->items[2] = (struct rte_flow_item){
602 .type = RTE_FLOW_ITEM_TYPE_TCP,
603 .spec = &attributes->l4,
604 .mask = &attributes->l4_mask,
607 case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
608 case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
611 DRV_LOG(ERR, "port %u invalid flow type%d",
612 dev->data->port_id, fdir_filter->input.flow_type);
620 * Add new flow director filter and store it in list.
623 * Pointer to Ethernet device.
625 * Flow director filter to add.
628 * 0 on success, a negative errno value otherwise and rte_errno is set.
631 mlx5_fdir_filter_add(struct rte_eth_dev *dev,
632 const struct rte_eth_fdir_filter *fdir_filter)
634 struct priv *priv = dev->data->dev_private;
635 struct mlx5_fdir attributes = {
638 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
639 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
643 struct rte_flow_error error;
644 struct rte_flow *flow;
647 ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
650 flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
651 attributes.items, attributes.actions,
654 DRV_LOG(DEBUG, "port %u FDIR created %p", dev->data->port_id,
662 * Delete specific filter.
665 * Pointer to Ethernet device.
667 * Filter to be deleted.
670 * 0 on success, a negative errno value otherwise and rte_errno is set.
673 mlx5_fdir_filter_delete(struct rte_eth_dev *dev __rte_unused,
674 const struct rte_eth_fdir_filter *fdir_filter
682 * Update queue for specific filter.
685 * Pointer to Ethernet device.
687 * Filter to be updated.
690 * 0 on success, a negative errno value otherwise and rte_errno is set.
693 mlx5_fdir_filter_update(struct rte_eth_dev *dev,
694 const struct rte_eth_fdir_filter *fdir_filter)
698 ret = mlx5_fdir_filter_delete(dev, fdir_filter);
701 return mlx5_fdir_filter_add(dev, fdir_filter);
708 * Pointer to Ethernet device.
711 mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
713 struct priv *priv = dev->data->dev_private;
715 mlx5_flow_list_flush(dev, &priv->flows);
719 * Get flow director information.
722 * Pointer to Ethernet device.
723 * @param[out] fdir_info
724 * Resulting flow director information.
727 mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
729 struct rte_eth_fdir_masks *mask =
730 &dev->data->dev_conf.fdir_conf.mask;
732 fdir_info->mode = dev->data->dev_conf.fdir_conf.mode;
733 fdir_info->guarant_spc = 0;
734 rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
735 fdir_info->max_flexpayload = 0;
736 fdir_info->flow_types_mask[0] = 0;
737 fdir_info->flex_payload_unit = 0;
738 fdir_info->max_flex_payload_segment_num = 0;
739 fdir_info->flex_payload_limit = 0;
740 memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
744 * Deal with flow director operations.
747 * Pointer to Ethernet device.
749 * Operation to perform.
751 * Pointer to operation-specific structure.
754 * 0 on success, a negative errno value otherwise and rte_errno is set.
757 mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
760 enum rte_fdir_mode fdir_mode =
761 dev->data->dev_conf.fdir_conf.mode;
763 if (filter_op == RTE_ETH_FILTER_NOP)
765 if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
766 fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
767 DRV_LOG(ERR, "port %u flow director mode %d not supported",
768 dev->data->port_id, fdir_mode);
773 case RTE_ETH_FILTER_ADD:
774 return mlx5_fdir_filter_add(dev, arg);
775 case RTE_ETH_FILTER_UPDATE:
776 return mlx5_fdir_filter_update(dev, arg);
777 case RTE_ETH_FILTER_DELETE:
778 return mlx5_fdir_filter_delete(dev, arg);
779 case RTE_ETH_FILTER_FLUSH:
780 mlx5_fdir_filter_flush(dev);
782 case RTE_ETH_FILTER_INFO:
783 mlx5_fdir_info_get(dev, arg);
786 DRV_LOG(DEBUG, "port %u unknown operation %u",
787 dev->data->port_id, filter_op);
795 * Manage filter operations.
798 * Pointer to Ethernet device structure.
802 * Operation to perform.
804 * Pointer to operation-specific structure.
807 * 0 on success, a negative errno value otherwise and rte_errno is set.
810 mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
811 enum rte_filter_type filter_type,
812 enum rte_filter_op filter_op,
815 switch (filter_type) {
816 case RTE_ETH_FILTER_GENERIC:
817 if (filter_op != RTE_ETH_FILTER_GET) {
821 *(const void **)arg = &mlx5_flow_ops;
823 case RTE_ETH_FILTER_FDIR:
824 return mlx5_fdir_ctrl_func(dev, filter_op, arg);
826 DRV_LOG(ERR, "port %u filter type (%d) not supported",
827 dev->data->port_id, filter_type);
835 * Detect number of Verbs flow priorities supported.
838 * Pointer to Ethernet device.
841 * number of supported Verbs flow priority.
844 mlx5_get_max_verbs_prio(struct rte_eth_dev *dev __rte_unused)