+ struct mlx5_flow_data *data)
+{
+ struct priv *priv = data->dev->data->dev_private;
+ const struct rte_flow_item_ipv4 *spec = item->spec;
+ const struct rte_flow_item_ipv4 *mask = item->mask;
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext);
+ struct ibv_flow_spec_ipv4_ext ipv4 = {
+ .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT,
+ .size = ipv4_size,
+ };
+
+ if (parser->layer == HASH_RXQ_TUNNEL &&
+ parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
+ !priv->config.l3_vxlan_en)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "L3 VXLAN not enabled by device"
+ " parameter and/or not configured"
+ " in firmware");
+ parser->layer = HASH_RXQ_IPV4;
+ if (spec) {
+ if (!mask)
+ mask = default_mask;
+ ipv4.val = (struct ibv_flow_ipv4_ext_filter){
+ .src_ip = spec->hdr.src_addr,
+ .dst_ip = spec->hdr.dst_addr,
+ .proto = spec->hdr.next_proto_id,
+ .tos = spec->hdr.type_of_service,
+ };
+ ipv4.mask = (struct ibv_flow_ipv4_ext_filter){
+ .src_ip = mask->hdr.src_addr,
+ .dst_ip = mask->hdr.dst_addr,
+ .proto = mask->hdr.next_proto_id,
+ .tos = mask->hdr.type_of_service,
+ };
+ /* Remove unwanted bits from values. */
+ ipv4.val.src_ip &= ipv4.mask.src_ip;
+ ipv4.val.dst_ip &= ipv4.mask.dst_ip;
+ ipv4.val.proto &= ipv4.mask.proto;
+ ipv4.val.tos &= ipv4.mask.tos;
+ }
+ mlx5_flow_create_copy(parser, &ipv4, ipv4_size);
+ return 0;
+}
+
+/**
+ * Convert IPv6 item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_ipv6(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data)
+{
+ struct priv *priv = data->dev->data->dev_private;
+ const struct rte_flow_item_ipv6 *spec = item->spec;
+ const struct rte_flow_item_ipv6 *mask = item->mask;
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6);
+ struct ibv_flow_spec_ipv6 ipv6 = {
+ .type = parser->inner | IBV_FLOW_SPEC_IPV6,
+ .size = ipv6_size,
+ };
+
+ if (parser->layer == HASH_RXQ_TUNNEL &&
+ parser->tunnel == ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)] &&
+ !priv->config.l3_vxlan_en)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "L3 VXLAN not enabled by device"
+ " parameter and/or not configured"
+ " in firmware");
+ parser->layer = HASH_RXQ_IPV6;
+ if (spec) {
+ unsigned int i;
+ uint32_t vtc_flow_val;
+ uint32_t vtc_flow_mask;
+
+ if (!mask)
+ mask = default_mask;
+ memcpy(&ipv6.val.src_ip, spec->hdr.src_addr,
+ RTE_DIM(ipv6.val.src_ip));
+ memcpy(&ipv6.val.dst_ip, spec->hdr.dst_addr,
+ RTE_DIM(ipv6.val.dst_ip));
+ memcpy(&ipv6.mask.src_ip, mask->hdr.src_addr,
+ RTE_DIM(ipv6.mask.src_ip));
+ memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr,
+ RTE_DIM(ipv6.mask.dst_ip));
+ vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow);
+ vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow);
+ ipv6.val.flow_label =
+ rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >>
+ IPV6_HDR_FL_SHIFT);
+ ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >>
+ IPV6_HDR_TC_SHIFT;
+ ipv6.val.next_hdr = spec->hdr.proto;
+ ipv6.val.hop_limit = spec->hdr.hop_limits;
+ ipv6.mask.flow_label =
+ rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >>
+ IPV6_HDR_FL_SHIFT);
+ ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >>
+ IPV6_HDR_TC_SHIFT;
+ ipv6.mask.next_hdr = mask->hdr.proto;
+ ipv6.mask.hop_limit = mask->hdr.hop_limits;
+ /* Remove unwanted bits from values. */
+ for (i = 0; i < RTE_DIM(ipv6.val.src_ip); ++i) {
+ ipv6.val.src_ip[i] &= ipv6.mask.src_ip[i];
+ ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i];
+ }
+ ipv6.val.flow_label &= ipv6.mask.flow_label;
+ ipv6.val.traffic_class &= ipv6.mask.traffic_class;
+ ipv6.val.next_hdr &= ipv6.mask.next_hdr;
+ ipv6.val.hop_limit &= ipv6.mask.hop_limit;
+ }
+ mlx5_flow_create_copy(parser, &ipv6, ipv6_size);
+ return 0;
+}
+
+/**
+ * Convert UDP item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_udp(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data)
+{
+ const struct rte_flow_item_udp *spec = item->spec;
+ const struct rte_flow_item_udp *mask = item->mask;
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp);
+ struct ibv_flow_spec_tcp_udp udp = {
+ .type = parser->inner | IBV_FLOW_SPEC_UDP,
+ .size = udp_size,
+ };
+
+ if (parser->layer == HASH_RXQ_IPV4)
+ parser->layer = HASH_RXQ_UDPV4;
+ else
+ parser->layer = HASH_RXQ_UDPV6;
+ if (spec) {
+ if (!mask)
+ mask = default_mask;
+ udp.val.dst_port = spec->hdr.dst_port;
+ udp.val.src_port = spec->hdr.src_port;
+ udp.mask.dst_port = mask->hdr.dst_port;
+ udp.mask.src_port = mask->hdr.src_port;
+ /* Remove unwanted bits from values. */
+ udp.val.src_port &= udp.mask.src_port;
+ udp.val.dst_port &= udp.mask.dst_port;
+ }
+ mlx5_flow_create_copy(parser, &udp, udp_size);
+ return 0;
+}
+
+/**
+ * Convert TCP item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_tcp(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data)
+{
+ const struct rte_flow_item_tcp *spec = item->spec;
+ const struct rte_flow_item_tcp *mask = item->mask;
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp);
+ struct ibv_flow_spec_tcp_udp tcp = {
+ .type = parser->inner | IBV_FLOW_SPEC_TCP,
+ .size = tcp_size,
+ };
+
+ if (parser->layer == HASH_RXQ_IPV4)
+ parser->layer = HASH_RXQ_TCPV4;
+ else
+ parser->layer = HASH_RXQ_TCPV6;
+ if (spec) {
+ if (!mask)
+ mask = default_mask;
+ tcp.val.dst_port = spec->hdr.dst_port;
+ tcp.val.src_port = spec->hdr.src_port;
+ tcp.mask.dst_port = mask->hdr.dst_port;
+ tcp.mask.src_port = mask->hdr.src_port;
+ /* Remove unwanted bits from values. */
+ tcp.val.src_port &= tcp.mask.src_port;
+ tcp.val.dst_port &= tcp.mask.dst_port;
+ }
+ mlx5_flow_create_copy(parser, &tcp, tcp_size);
+ return 0;
+}
+
+/**
+ * Convert VXLAN item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_vxlan(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data)
+{
+ const struct rte_flow_item_vxlan *spec = item->spec;
+ const struct rte_flow_item_vxlan *mask = item->mask;
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+ struct ibv_flow_spec_tunnel vxlan = {
+ .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
+ .size = size,
+ };
+ union vni {
+ uint32_t vlan_id;
+ uint8_t vni[4];
+ } id;
+
+ id.vni[0] = 0;
+ parser->inner = IBV_FLOW_SPEC_INNER;
+ parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN)];
+ parser->out_layer = parser->layer;
+ parser->layer = HASH_RXQ_TUNNEL;
+ /* Default VXLAN to outer RSS. */
+ if (!parser->rss_conf.level)
+ parser->rss_conf.level = 1;
+ if (spec) {
+ if (!mask)
+ mask = default_mask;
+ memcpy(&id.vni[1], spec->vni, 3);
+ vxlan.val.tunnel_id = id.vlan_id;
+ memcpy(&id.vni[1], mask->vni, 3);
+ vxlan.mask.tunnel_id = id.vlan_id;
+ /* Remove unwanted bits from values. */
+ vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+ }
+ /*
+ * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
+ * layer is defined in the Verbs specification it is interpreted as
+ * wildcard and all packets will match this rule, if it follows a full
+ * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
+ * before will also match this rule.
+ * To avoid such situation, VNI 0 is currently refused.
+ */
+ /* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
+ if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "VxLAN vni cannot be 0");
+ mlx5_flow_create_copy(parser, &vxlan, size);
+ return 0;
+}
+
+/**
+ * Convert VXLAN-GPE item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_vxlan_gpe(const struct rte_flow_item *item,
+ const void *default_mask,
+ struct mlx5_flow_data *data)
+{
+ struct priv *priv = data->dev->data->dev_private;
+ const struct rte_flow_item_vxlan_gpe *spec = item->spec;
+ const struct rte_flow_item_vxlan_gpe *mask = item->mask;
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+ struct ibv_flow_spec_tunnel vxlan = {
+ .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
+ .size = size,
+ };
+ union vni {
+ uint32_t vlan_id;
+ uint8_t vni[4];
+ } id;
+
+ if (!priv->config.l3_vxlan_en)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "L3 VXLAN not enabled by device"
+ " parameter and/or not configured"
+ " in firmware");
+ id.vni[0] = 0;
+ parser->inner = IBV_FLOW_SPEC_INNER;
+ parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_VXLAN_GPE)];
+ parser->out_layer = parser->layer;
+ parser->layer = HASH_RXQ_TUNNEL;
+ /* Default VXLAN-GPE to outer RSS. */
+ if (!parser->rss_conf.level)
+ parser->rss_conf.level = 1;
+ if (spec) {
+ if (!mask)
+ mask = default_mask;
+ memcpy(&id.vni[1], spec->vni, 3);
+ vxlan.val.tunnel_id = id.vlan_id;
+ memcpy(&id.vni[1], mask->vni, 3);
+ vxlan.mask.tunnel_id = id.vlan_id;
+ if (spec->protocol)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "VxLAN-GPE protocol not"
+ " supported");
+ /* Remove unwanted bits from values. */
+ vxlan.val.tunnel_id &= vxlan.mask.tunnel_id;
+ }
+ /*
+ * Tunnel id 0 is equivalent as not adding a VXLAN layer, if only this
+ * layer is defined in the Verbs specification it is interpreted as
+ * wildcard and all packets will match this rule, if it follows a full
+ * stack layer (ex: eth / ipv4 / udp), all packets matching the layers
+ * before will also match this rule.
+ * To avoid such situation, VNI 0 is currently refused.
+ */
+ /* Only allow tunnel w/o tunnel id pattern after proper outer spec. */
+ if (parser->out_layer == HASH_RXQ_ETH && !vxlan.val.tunnel_id)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "VxLAN-GPE vni cannot be 0");
+ mlx5_flow_create_copy(parser, &vxlan, size);
+ return 0;
+}
+
+/**
+ * Convert GRE item to Verbs specification.
+ *
+ * @param item[in]
+ * Item specification.
+ * @param default_mask[in]
+ * Default bit-masks to use when item->mask is not provided.
+ * @param data[in, out]
+ * User structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_gre(const struct rte_flow_item *item __rte_unused,
+ const void *default_mask __rte_unused,
+ struct mlx5_flow_data *data)
+{
+ struct mlx5_flow_parse *parser = data->parser;
+ unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
+ struct ibv_flow_spec_tunnel tunnel = {
+ .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL,
+ .size = size,
+ };
+ struct ibv_flow_spec_ipv4_ext *ipv4;
+ struct ibv_flow_spec_ipv6 *ipv6;
+ unsigned int i;
+
+ parser->inner = IBV_FLOW_SPEC_INNER;
+ parser->tunnel = ptype_ext[PTYPE_IDX(RTE_PTYPE_TUNNEL_GRE)];
+ parser->out_layer = parser->layer;
+ parser->layer = HASH_RXQ_TUNNEL;
+ /* Default GRE to inner RSS. */
+ if (!parser->rss_conf.level)
+ parser->rss_conf.level = 2;
+ /* Update encapsulation IP layer protocol. */
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!parser->queue[i].ibv_attr)
+ continue;
+ if (parser->out_layer == HASH_RXQ_IPV4) {
+ ipv4 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
+ parser->queue[i].offset -
+ sizeof(struct ibv_flow_spec_ipv4_ext));
+ if (ipv4->mask.proto && ipv4->val.proto != MLX5_GRE)
+ break;
+ ipv4->val.proto = MLX5_GRE;
+ ipv4->mask.proto = 0xff;
+ } else if (parser->out_layer == HASH_RXQ_IPV6) {
+ ipv6 = (void *)((uintptr_t)parser->queue[i].ibv_attr +
+ parser->queue[i].offset -
+ sizeof(struct ibv_flow_spec_ipv6));
+ if (ipv6->mask.next_hdr &&
+ ipv6->val.next_hdr != MLX5_GRE)
+ break;
+ ipv6->val.next_hdr = MLX5_GRE;
+ ipv6->mask.next_hdr = 0xff;
+ }
+ }
+ if (i != hash_rxq_init_n)
+ return rte_flow_error_set(data->error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ITEM,
+ item,
+ "IP protocol of GRE must be 47");
+ mlx5_flow_create_copy(parser, &tunnel, size);
+ return 0;
+}
+
+/**
+ * Convert mark/flag action to Verbs specification.
+ *
+ * @param parser
+ * Internal parser structure.
+ * @param mark_id
+ * Mark identifier.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
+{
+ unsigned int size = sizeof(struct ibv_flow_spec_action_tag);
+ struct ibv_flow_spec_action_tag tag = {
+ .type = IBV_FLOW_SPEC_ACTION_TAG,
+ .size = size,
+ .tag_id = mlx5_flow_mark_set(mark_id),
+ };
+
+ assert(parser->mark);
+ mlx5_flow_create_copy(parser, &tag, size);
+ return 0;
+}
+
+/**
+ * Convert count action to Verbs specification.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param parser
+ * Pointer to MLX5 flow parser structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
+ struct mlx5_flow_parse *parser __rte_unused)
+{
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+ struct priv *priv = dev->data->dev_private;
+ unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
+ struct ibv_counter_set_init_attr init_attr = {0};
+ struct ibv_flow_spec_counter_action counter = {
+ .type = IBV_FLOW_SPEC_ACTION_COUNT,
+ .size = size,
+ .counter_set_handle = 0,
+ };
+
+ init_attr.counter_set_id = 0;
+ parser->cs = mlx5_glue->create_counter_set(priv->ctx, &init_attr);
+ if (!parser->cs) {
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ counter.counter_set_handle = parser->cs->handle;
+ mlx5_flow_create_copy(parser, &counter, size);
+#endif
+ return 0;
+}
+
+/**
+ * Complete flow rule creation with a drop queue.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param parser
+ * Internal parser structure.
+ * @param flow
+ * Pointer to the rte_flow.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
+ struct mlx5_flow_parse *parser,
+ struct rte_flow *flow,
+ struct rte_flow_error *error)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct ibv_flow_spec_action_drop *drop;
+ unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
+
+ assert(priv->pd);
+ assert(priv->ctx);
+ flow->drop = 1;
+ drop = (void *)((uintptr_t)parser->queue[HASH_RXQ_ETH].ibv_attr +
+ parser->queue[HASH_RXQ_ETH].offset);
+ *drop = (struct ibv_flow_spec_action_drop){
+ .type = IBV_FLOW_SPEC_ACTION_DROP,
+ .size = size,
+ };
+ ++parser->queue[HASH_RXQ_ETH].ibv_attr->num_of_specs;
+ parser->queue[HASH_RXQ_ETH].offset += size;
+ flow->frxq[HASH_RXQ_ETH].ibv_attr =
+ parser->queue[HASH_RXQ_ETH].ibv_attr;
+ if (parser->count)
+ flow->cs = parser->cs;
+ if (!dev->data->dev_started)
+ return 0;
+ parser->queue[HASH_RXQ_ETH].ibv_attr = NULL;
+ flow->frxq[HASH_RXQ_ETH].ibv_flow =
+ mlx5_glue->create_flow(priv->flow_drop_queue->qp,
+ flow->frxq[HASH_RXQ_ETH].ibv_attr);
+ if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
+ rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
+ NULL, "flow rule creation failure");
+ goto error;
+ }
+ return 0;
+error:
+ assert(flow);
+ if (flow->frxq[HASH_RXQ_ETH].ibv_flow) {
+ claim_zero(mlx5_glue->destroy_flow
+ (flow->frxq[HASH_RXQ_ETH].ibv_flow));
+ flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
+ }
+ if (flow->frxq[HASH_RXQ_ETH].ibv_attr) {
+ rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
+ flow->frxq[HASH_RXQ_ETH].ibv_attr = NULL;
+ }
+ if (flow->cs) {
+ claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
+ flow->cs = NULL;
+ parser->cs = NULL;
+ }
+ return -rte_errno;
+}
+
+/**
+ * Create hash Rx queues when RSS is enabled.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param parser
+ * Internal parser structure.
+ * @param flow
+ * Pointer to the rte_flow.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
+ struct mlx5_flow_parse *parser,
+ struct rte_flow *flow,
+ struct rte_flow_error *error)
+{
+ unsigned int i;
+
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!parser->queue[i].ibv_attr)
+ continue;
+ flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr;
+ parser->queue[i].ibv_attr = NULL;
+ flow->frxq[i].hash_fields = parser->queue[i].hash_fields;
+ if (!dev->data->dev_started)
+ continue;
+ flow->frxq[i].hrxq =
+ mlx5_hrxq_get(dev,
+ parser->rss_conf.key,
+ parser->rss_conf.key_len,
+ flow->frxq[i].hash_fields,
+ parser->rss_conf.queue,
+ parser->rss_conf.queue_num,
+ parser->tunnel,
+ parser->rss_conf.level);
+ if (flow->frxq[i].hrxq)
+ continue;
+ flow->frxq[i].hrxq =
+ mlx5_hrxq_new(dev,
+ parser->rss_conf.key,
+ parser->rss_conf.key_len,
+ flow->frxq[i].hash_fields,
+ parser->rss_conf.queue,
+ parser->rss_conf.queue_num,
+ parser->tunnel,
+ parser->rss_conf.level);
+ if (!flow->frxq[i].hrxq) {
+ return rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE,
+ NULL,
+ "cannot create hash rxq");
+ }
+ }
+ return 0;
+}
+
+/**
+ * RXQ update after flow rule creation.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param flow
+ * Pointer to the flow rule.
+ */
+static void
+mlx5_flow_create_update_rxqs(struct rte_eth_dev *dev, struct rte_flow *flow)
+{
+ struct priv *priv = dev->data->dev_private;
+ unsigned int i;
+ unsigned int j;
+
+ if (!dev->data->dev_started)
+ return;
+ for (i = 0; i != flow->rss_conf.queue_num; ++i) {
+ struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
+ [(*flow->queues)[i]];
+ struct mlx5_rxq_ctrl *rxq_ctrl =
+ container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
+ uint8_t tunnel = PTYPE_IDX(flow->tunnel);
+
+ rxq_data->mark |= flow->mark;
+ if (!tunnel)
+ continue;
+ rxq_ctrl->tunnel_types[tunnel] += 1;
+ /* Clear tunnel type if more than one tunnel types set. */
+ for (j = 0; j != RTE_DIM(rxq_ctrl->tunnel_types); ++j) {
+ if (j == tunnel)
+ continue;
+ if (rxq_ctrl->tunnel_types[j] > 0) {
+ rxq_data->tunnel = 0;
+ break;
+ }
+ }
+ if (j == RTE_DIM(rxq_ctrl->tunnel_types))
+ rxq_data->tunnel = flow->tunnel;
+ }
+}
+
+/**
+ * Dump flow hash RX queue detail.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param flow
+ * Pointer to the rte_flow.
+ * @param hrxq_idx
+ * Hash RX queue index.
+ */
+static void
+mlx5_flow_dump(struct rte_eth_dev *dev __rte_unused,
+ struct rte_flow *flow __rte_unused,
+ unsigned int hrxq_idx __rte_unused)
+{
+#ifndef NDEBUG
+ uintptr_t spec_ptr;
+ uint16_t j;
+ char buf[256];
+ uint8_t off;
+
+ spec_ptr = (uintptr_t)(flow->frxq[hrxq_idx].ibv_attr + 1);
+ for (j = 0, off = 0; j < flow->frxq[hrxq_idx].ibv_attr->num_of_specs;
+ j++) {
+ struct ibv_flow_spec *spec = (void *)spec_ptr;
+ off += sprintf(buf + off, " %x(%hu)", spec->hdr.type,
+ spec->hdr.size);
+ spec_ptr += spec->hdr.size;
+ }
+ DRV_LOG(DEBUG,
+ "port %u Verbs flow %p type %u: hrxq:%p qp:%p ind:%p,"
+ " hash:%" PRIx64 "/%u specs:%hhu(%hu), priority:%hu, type:%d,"
+ " flags:%x, comp_mask:%x specs:%s",
+ dev->data->port_id, (void *)flow, hrxq_idx,
+ (void *)flow->frxq[hrxq_idx].hrxq,
+ (void *)flow->frxq[hrxq_idx].hrxq->qp,
+ (void *)flow->frxq[hrxq_idx].hrxq->ind_table,
+ flow->frxq[hrxq_idx].hash_fields |
+ (flow->tunnel &&
+ flow->rss_conf.level > 1 ? (uint32_t)IBV_RX_HASH_INNER : 0),
+ flow->rss_conf.queue_num,
+ flow->frxq[hrxq_idx].ibv_attr->num_of_specs,
+ flow->frxq[hrxq_idx].ibv_attr->size,
+ flow->frxq[hrxq_idx].ibv_attr->priority,
+ flow->frxq[hrxq_idx].ibv_attr->type,
+ flow->frxq[hrxq_idx].ibv_attr->flags,
+ flow->frxq[hrxq_idx].ibv_attr->comp_mask,
+ buf);
+#endif
+}
+
+/**
+ * Complete flow rule creation.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param parser
+ * Internal parser structure.
+ * @param flow
+ * Pointer to the rte_flow.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
+ struct mlx5_flow_parse *parser,
+ struct rte_flow *flow,
+ struct rte_flow_error *error)
+{
+ struct priv *priv __rte_unused = dev->data->dev_private;
+ int ret;
+ unsigned int i;
+ unsigned int flows_n = 0;
+
+ assert(priv->pd);
+ assert(priv->ctx);
+ assert(!parser->drop);
+ ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
+ if (ret)
+ goto error;
+ if (parser->count)
+ flow->cs = parser->cs;
+ if (!dev->data->dev_started)
+ return 0;
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!flow->frxq[i].hrxq)
+ continue;
+ flow->frxq[i].ibv_flow =
+ mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
+ flow->frxq[i].ibv_attr);
+ mlx5_flow_dump(dev, flow, i);
+ if (!flow->frxq[i].ibv_flow) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_HANDLE,
+ NULL, "flow rule creation failure");
+ goto error;
+ }
+ ++flows_n;
+ }
+ if (!flows_n) {
+ rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE,
+ NULL, "internal error in flow creation");
+ goto error;
+ }
+ mlx5_flow_create_update_rxqs(dev, flow);
+ return 0;
+error:
+ ret = rte_errno; /* Save rte_errno before cleanup. */
+ assert(flow);
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (flow->frxq[i].ibv_flow) {
+ struct ibv_flow *ibv_flow = flow->frxq[i].ibv_flow;
+
+ claim_zero(mlx5_glue->destroy_flow(ibv_flow));
+ }
+ if (flow->frxq[i].hrxq)
+ mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
+ if (flow->frxq[i].ibv_attr)
+ rte_free(flow->frxq[i].ibv_attr);
+ }
+ if (flow->cs) {
+ claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
+ flow->cs = NULL;
+ parser->cs = NULL;
+ }
+ rte_errno = ret; /* Restore rte_errno. */
+ return -rte_errno;
+}
+
+/**
+ * Convert a flow.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param list
+ * Pointer to a TAILQ flow list.
+ * @param[in] attr
+ * Flow rule attributes.
+ * @param[in] pattern
+ * Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * A flow on success, NULL otherwise and rte_errno is set.
+ */
+static struct rte_flow *
+mlx5_flow_list_create(struct rte_eth_dev *dev,
+ struct mlx5_flows *list,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ struct mlx5_flow_parse parser = { .create = 1, };
+ struct rte_flow *flow = NULL;
+ unsigned int i;
+ int ret;
+
+ ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
+ if (ret)
+ goto exit;
+ flow = rte_calloc(__func__, 1,
+ sizeof(*flow) +
+ parser.rss_conf.queue_num * sizeof(uint16_t),
+ 0);
+ if (!flow) {
+ rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "cannot allocate flow memory");
+ return NULL;
+ }
+ /* Copy configuration. */
+ flow->queues = (uint16_t (*)[])(flow + 1);
+ flow->tunnel = parser.tunnel;
+ flow->rss_conf = (struct rte_flow_action_rss){
+ .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
+ .level = 0,
+ .types = parser.rss_conf.types,
+ .key_len = parser.rss_conf.key_len,
+ .queue_num = parser.rss_conf.queue_num,
+ .key = memcpy(flow->rss_key, parser.rss_conf.key,
+ sizeof(*parser.rss_conf.key) *
+ parser.rss_conf.key_len),
+ .queue = memcpy(flow->queues, parser.rss_conf.queue,
+ sizeof(*parser.rss_conf.queue) *
+ parser.rss_conf.queue_num),
+ };
+ flow->mark = parser.mark;
+ /* finalise the flow. */
+ if (parser.drop)
+ ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
+ error);
+ else
+ ret = mlx5_flow_create_action_queue(dev, &parser, flow, error);
+ if (ret)
+ goto exit;
+ TAILQ_INSERT_TAIL(list, flow, next);
+ DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id,
+ (void *)flow);
+ return flow;
+exit:
+ DRV_LOG(ERR, "port %u flow creation error: %s", dev->data->port_id,
+ error->message);
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (parser.queue[i].ibv_attr)
+ rte_free(parser.queue[i].ibv_attr);
+ }
+ rte_free(flow);
+ return NULL;
+}
+
+/**
+ * Validate a flow supported by the NIC.
+ *
+ * @see rte_flow_validate()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_validate(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ struct mlx5_flow_parse parser = { .create = 0, };
+
+ return mlx5_flow_convert(dev, attr, items, actions, error, &parser);
+}
+
+/**
+ * Create a flow.
+ *
+ * @see rte_flow_create()
+ * @see rte_flow_ops
+ */
+struct rte_flow *
+mlx5_flow_create(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ struct priv *priv = dev->data->dev_private;
+
+ return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
+ error);
+}
+
+/**
+ * Destroy a flow in a list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param list
+ * Pointer to a TAILQ flow list.
+ * @param[in] flow
+ * Flow to destroy.
+ */
+static void
+mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
+ struct rte_flow *flow)
+{
+ struct priv *priv = dev->data->dev_private;
+ unsigned int i;
+
+ if (flow->drop || !dev->data->dev_started)
+ goto free;
+ for (i = 0; flow->tunnel && i != flow->rss_conf.queue_num; ++i) {
+ /* Update queue tunnel type. */
+ struct mlx5_rxq_data *rxq_data = (*priv->rxqs)
+ [(*flow->queues)[i]];
+ struct mlx5_rxq_ctrl *rxq_ctrl =
+ container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
+ uint8_t tunnel = PTYPE_IDX(flow->tunnel);
+
+ assert(rxq_ctrl->tunnel_types[tunnel] > 0);
+ rxq_ctrl->tunnel_types[tunnel] -= 1;
+ if (!rxq_ctrl->tunnel_types[tunnel]) {
+ /* Update tunnel type. */
+ uint8_t j;
+ uint8_t types = 0;
+ uint8_t last;
+
+ for (j = 0; j < RTE_DIM(rxq_ctrl->tunnel_types); j++)
+ if (rxq_ctrl->tunnel_types[j]) {
+ types += 1;
+ last = j;
+ }
+ /* Keep same if more than one tunnel types left. */
+ if (types == 1)
+ rxq_data->tunnel = ptype_ext[last];
+ else if (types == 0)
+ /* No tunnel type left. */
+ rxq_data->tunnel = 0;
+ }
+ }
+ for (i = 0; flow->mark && i != flow->rss_conf.queue_num; ++i) {
+ struct rte_flow *tmp;
+ int mark = 0;
+
+ /*
+ * To remove the mark from the queue, the queue must not be
+ * present in any other marked flow (RSS or not).
+ */
+ TAILQ_FOREACH(tmp, list, next) {
+ unsigned int j;
+ uint16_t *tqs = NULL;
+ uint16_t tq_n = 0;
+
+ if (!tmp->mark)
+ continue;
+ for (j = 0; j != hash_rxq_init_n; ++j) {
+ if (!tmp->frxq[j].hrxq)
+ continue;
+ tqs = tmp->frxq[j].hrxq->ind_table->queues;
+ tq_n = tmp->frxq[j].hrxq->ind_table->queues_n;
+ }
+ if (!tq_n)
+ continue;
+ for (j = 0; (j != tq_n) && !mark; j++)
+ if (tqs[j] == (*flow->queues)[i])
+ mark = 1;
+ }
+ (*priv->rxqs)[(*flow->queues)[i]]->mark = mark;
+ }
+free:
+ if (flow->drop) {
+ if (flow->frxq[HASH_RXQ_ETH].ibv_flow)
+ claim_zero(mlx5_glue->destroy_flow
+ (flow->frxq[HASH_RXQ_ETH].ibv_flow));
+ rte_free(flow->frxq[HASH_RXQ_ETH].ibv_attr);
+ } else {
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ struct mlx5_flow *frxq = &flow->frxq[i];
+
+ if (frxq->ibv_flow)
+ claim_zero(mlx5_glue->destroy_flow
+ (frxq->ibv_flow));
+ if (frxq->hrxq)
+ mlx5_hrxq_release(dev, frxq->hrxq);
+ if (frxq->ibv_attr)
+ rte_free(frxq->ibv_attr);
+ }
+ }
+ if (flow->cs) {
+ claim_zero(mlx5_glue->destroy_counter_set(flow->cs));
+ flow->cs = NULL;
+ }
+ TAILQ_REMOVE(list, flow, next);
+ DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id,
+ (void *)flow);
+ rte_free(flow);
+}
+
+/**
+ * Destroy all flows.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param list
+ * Pointer to a TAILQ flow list.
+ */
+void
+mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
+{
+ while (!TAILQ_EMPTY(list)) {
+ struct rte_flow *flow;
+
+ flow = TAILQ_FIRST(list);
+ mlx5_flow_list_destroy(dev, list, flow);
+ }
+}
+
+/**
+ * Create drop queue.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_hrxq_drop *fdq = NULL;
+
+ assert(priv->pd);
+ assert(priv->ctx);
+ fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0);
+ if (!fdq) {
+ DRV_LOG(WARNING,
+ "port %u cannot allocate memory for drop queue",
+ dev->data->port_id);
+ rte_errno = ENOMEM;
+ return -rte_errno;
+ }
+ fdq->cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
+ if (!fdq->cq) {
+ DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue",
+ dev->data->port_id);
+ rte_errno = errno;
+ goto error;
+ }
+ fdq->wq = mlx5_glue->create_wq
+ (priv->ctx,
+ &(struct ibv_wq_init_attr){
+ .wq_type = IBV_WQT_RQ,
+ .max_wr = 1,
+ .max_sge = 1,
+ .pd = priv->pd,
+ .cq = fdq->cq,
+ });
+ if (!fdq->wq) {
+ DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue",
+ dev->data->port_id);
+ rte_errno = errno;
+ goto error;
+ }
+ fdq->ind_table = mlx5_glue->create_rwq_ind_table
+ (priv->ctx,
+ &(struct ibv_rwq_ind_table_init_attr){
+ .log_ind_tbl_size = 0,
+ .ind_tbl = &fdq->wq,
+ .comp_mask = 0,
+ });
+ if (!fdq->ind_table) {
+ DRV_LOG(WARNING,
+ "port %u cannot allocate indirection table for drop"
+ " queue",
+ dev->data->port_id);
+ rte_errno = errno;
+ goto error;
+ }
+ fdq->qp = mlx5_glue->create_qp_ex
+ (priv->ctx,
+ &(struct ibv_qp_init_attr_ex){
+ .qp_type = IBV_QPT_RAW_PACKET,
+ .comp_mask =
+ IBV_QP_INIT_ATTR_PD |
+ IBV_QP_INIT_ATTR_IND_TABLE |
+ IBV_QP_INIT_ATTR_RX_HASH,
+ .rx_hash_conf = (struct ibv_rx_hash_conf){
+ .rx_hash_function =
+ IBV_RX_HASH_FUNC_TOEPLITZ,
+ .rx_hash_key_len = rss_hash_default_key_len,
+ .rx_hash_key = rss_hash_default_key,
+ .rx_hash_fields_mask = 0,
+ },
+ .rwq_ind_tbl = fdq->ind_table,
+ .pd = priv->pd
+ });
+ if (!fdq->qp) {
+ DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue",
+ dev->data->port_id);
+ rte_errno = errno;
+ goto error;
+ }
+ priv->flow_drop_queue = fdq;
+ return 0;
+error:
+ if (fdq->qp)
+ claim_zero(mlx5_glue->destroy_qp(fdq->qp));
+ if (fdq->ind_table)
+ claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
+ if (fdq->wq)
+ claim_zero(mlx5_glue->destroy_wq(fdq->wq));
+ if (fdq->cq)
+ claim_zero(mlx5_glue->destroy_cq(fdq->cq));
+ if (fdq)
+ rte_free(fdq);
+ priv->flow_drop_queue = NULL;
+ return -rte_errno;
+}
+
+/**
+ * Delete drop queue.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
+
+ if (!fdq)
+ return;
+ if (fdq->qp)
+ claim_zero(mlx5_glue->destroy_qp(fdq->qp));
+ if (fdq->ind_table)
+ claim_zero(mlx5_glue->destroy_rwq_ind_table(fdq->ind_table));
+ if (fdq->wq)
+ claim_zero(mlx5_glue->destroy_wq(fdq->wq));
+ if (fdq->cq)
+ claim_zero(mlx5_glue->destroy_cq(fdq->cq));
+ rte_free(fdq);
+ priv->flow_drop_queue = NULL;
+}
+
+/**
+ * Remove all flows.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param list
+ * Pointer to a TAILQ flow list.
+ */
+void
+mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct rte_flow *flow;
+ unsigned int i;
+
+ TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
+ struct mlx5_ind_table_ibv *ind_tbl = NULL;
+
+ if (flow->drop) {
+ if (!flow->frxq[HASH_RXQ_ETH].ibv_flow)
+ continue;
+ claim_zero(mlx5_glue->destroy_flow
+ (flow->frxq[HASH_RXQ_ETH].ibv_flow));
+ flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL;
+ DRV_LOG(DEBUG, "port %u flow %p removed",
+ dev->data->port_id, (void *)flow);
+ /* Next flow. */
+ continue;
+ }
+ /* Verify the flow has not already been cleaned. */
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!flow->frxq[i].ibv_flow)
+ continue;
+ /*
+ * Indirection table may be necessary to remove the
+ * flags in the Rx queues.
+ * This helps to speed-up the process by avoiding
+ * another loop.
+ */
+ ind_tbl = flow->frxq[i].hrxq->ind_table;
+ break;
+ }
+ if (i == hash_rxq_init_n)
+ return;
+ if (flow->mark) {
+ assert(ind_tbl);
+ for (i = 0; i != ind_tbl->queues_n; ++i)
+ (*priv->rxqs)[ind_tbl->queues[i]]->mark = 0;
+ }
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!flow->frxq[i].ibv_flow)
+ continue;
+ claim_zero(mlx5_glue->destroy_flow
+ (flow->frxq[i].ibv_flow));
+ flow->frxq[i].ibv_flow = NULL;
+ mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
+ flow->frxq[i].hrxq = NULL;
+ }
+ DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id,
+ (void *)flow);
+ }
+ /* Cleanup Rx queue tunnel info. */
+ for (i = 0; i != priv->rxqs_n; ++i) {
+ struct mlx5_rxq_data *q = (*priv->rxqs)[i];
+ struct mlx5_rxq_ctrl *rxq_ctrl =
+ container_of(q, struct mlx5_rxq_ctrl, rxq);
+
+ if (!q)
+ continue;
+ memset((void *)rxq_ctrl->tunnel_types, 0,
+ sizeof(rxq_ctrl->tunnel_types));
+ q->tunnel = 0;
+ }
+}
+
+/**
+ * Add all flows.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param list
+ * Pointer to a TAILQ flow list.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct rte_flow *flow;
+
+ TAILQ_FOREACH(flow, list, next) {
+ unsigned int i;
+
+ if (flow->drop) {
+ flow->frxq[HASH_RXQ_ETH].ibv_flow =
+ mlx5_glue->create_flow
+ (priv->flow_drop_queue->qp,
+ flow->frxq[HASH_RXQ_ETH].ibv_attr);
+ if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) {
+ DRV_LOG(DEBUG,
+ "port %u flow %p cannot be applied",
+ dev->data->port_id, (void *)flow);
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ DRV_LOG(DEBUG, "port %u flow %p applied",
+ dev->data->port_id, (void *)flow);
+ /* Next flow. */
+ continue;
+ }
+ for (i = 0; i != hash_rxq_init_n; ++i) {
+ if (!flow->frxq[i].ibv_attr)
+ continue;
+ flow->frxq[i].hrxq =
+ mlx5_hrxq_get(dev, flow->rss_conf.key,
+ flow->rss_conf.key_len,
+ flow->frxq[i].hash_fields,
+ flow->rss_conf.queue,
+ flow->rss_conf.queue_num,
+ flow->tunnel,
+ flow->rss_conf.level);
+ if (flow->frxq[i].hrxq)
+ goto flow_create;
+ flow->frxq[i].hrxq =
+ mlx5_hrxq_new(dev, flow->rss_conf.key,
+ flow->rss_conf.key_len,
+ flow->frxq[i].hash_fields,
+ flow->rss_conf.queue,
+ flow->rss_conf.queue_num,
+ flow->tunnel,
+ flow->rss_conf.level);
+ if (!flow->frxq[i].hrxq) {
+ DRV_LOG(DEBUG,
+ "port %u flow %p cannot create hash"
+ " rxq",
+ dev->data->port_id, (void *)flow);
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+flow_create:
+ mlx5_flow_dump(dev, flow, i);
+ flow->frxq[i].ibv_flow =
+ mlx5_glue->create_flow(flow->frxq[i].hrxq->qp,
+ flow->frxq[i].ibv_attr);
+ if (!flow->frxq[i].ibv_flow) {
+ DRV_LOG(DEBUG,
+ "port %u flow %p type %u cannot be"
+ " applied",
+ dev->data->port_id, (void *)flow, i);
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ }
+ mlx5_flow_create_update_rxqs(dev, flow);
+ }
+ return 0;
+}
+
+/**
+ * Verify the flow list is empty
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ *
+ * @return the number of flows not released.
+ */
+int
+mlx5_flow_verify(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct rte_flow *flow;
+ int ret = 0;
+
+ TAILQ_FOREACH(flow, &priv->flows, next) {
+ DRV_LOG(DEBUG, "port %u flow %p still referenced",
+ dev->data->port_id, (void *)flow);
+ ++ret;
+ }
+ return ret;
+}
+
+/**
+ * Enable a control flow configured from the control plane.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param eth_spec
+ * An Ethernet flow spec to apply.
+ * @param eth_mask
+ * An Ethernet flow mask to apply.
+ * @param vlan_spec
+ * A VLAN flow spec to apply.
+ * @param vlan_mask
+ * A VLAN flow mask to apply.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
+ struct rte_flow_item_eth *eth_spec,
+ struct rte_flow_item_eth *eth_mask,
+ struct rte_flow_item_vlan *vlan_spec,
+ struct rte_flow_item_vlan *vlan_mask)
+{
+ struct priv *priv = dev->data->dev_private;
+ const struct rte_flow_attr attr = {
+ .ingress = 1,
+ .priority = MLX5_CTRL_FLOW_PRIORITY,
+ };
+ struct rte_flow_item items[] = {
+ {
+ .type = RTE_FLOW_ITEM_TYPE_ETH,
+ .spec = eth_spec,
+ .last = NULL,
+ .mask = eth_mask,
+ },
+ {
+ .type = (vlan_spec) ? RTE_FLOW_ITEM_TYPE_VLAN :
+ RTE_FLOW_ITEM_TYPE_END,
+ .spec = vlan_spec,
+ .last = NULL,
+ .mask = vlan_mask,
+ },
+ {
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ },
+ };
+ uint16_t queue[priv->reta_idx_n];
+ struct rte_flow_action_rss action_rss = {
+ .func = RTE_ETH_HASH_FUNCTION_DEFAULT,
+ .level = 0,
+ .types = priv->rss_conf.rss_hf,
+ .key_len = priv->rss_conf.rss_key_len,
+ .queue_num = priv->reta_idx_n,
+ .key = priv->rss_conf.rss_key,
+ .queue = queue,
+ };
+ struct rte_flow_action actions[] = {
+ {
+ .type = RTE_FLOW_ACTION_TYPE_RSS,
+ .conf = &action_rss,
+ },
+ {
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ },
+ };
+ struct rte_flow *flow;
+ struct rte_flow_error error;
+ unsigned int i;
+
+ if (!priv->reta_idx_n) {
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ for (i = 0; i != priv->reta_idx_n; ++i)
+ queue[i] = (*priv->reta_idx)[i];
+ flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
+ actions, &error);
+ if (!flow)
+ return -rte_errno;
+ return 0;
+}
+
+/**
+ * Enable a flow control configured from the control plane.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param eth_spec
+ * An Ethernet flow spec to apply.
+ * @param eth_mask
+ * An Ethernet flow mask to apply.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_ctrl_flow(struct rte_eth_dev *dev,
+ struct rte_flow_item_eth *eth_spec,
+ struct rte_flow_item_eth *eth_mask)
+{
+ return mlx5_ctrl_flow_vlan(dev, eth_spec, eth_mask, NULL, NULL);
+}
+
+/**
+ * Destroy a flow.
+ *
+ * @see rte_flow_destroy()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_destroy(struct rte_eth_dev *dev,
+ struct rte_flow *flow,
+ struct rte_flow_error *error __rte_unused)
+{
+ struct priv *priv = dev->data->dev_private;
+
+ mlx5_flow_list_destroy(dev, &priv->flows, flow);
+ return 0;
+}
+
+/**
+ * Destroy all flows.
+ *
+ * @see rte_flow_flush()
+ * @see rte_flow_ops
+ */
+int
+mlx5_flow_flush(struct rte_eth_dev *dev,
+ struct rte_flow_error *error __rte_unused)