+#ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
+#define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
+#endif
+
+union flow_dv_attr {
+ struct {
+ uint32_t valid:1;
+ uint32_t ipv4:1;
+ uint32_t ipv6:1;
+ uint32_t tcp:1;
+ uint32_t udp:1;
+ uint32_t reserved:27;
+ };
+ uint32_t attr;
+};
+
+/**
+ * Initialize flow attributes structure according to flow items' types.
+ *
+ * @param[in] item
+ * Pointer to item specification.
+ * @param[out] attr
+ * Pointer to flow attributes structure.
+ */
+static void
+flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr)
+{
+ for (; item->type != RTE_FLOW_ITEM_TYPE_END; item++) {
+ switch (item->type) {
+ case RTE_FLOW_ITEM_TYPE_IPV4:
+ attr->ipv4 = 1;
+ break;
+ case RTE_FLOW_ITEM_TYPE_IPV6:
+ attr->ipv6 = 1;
+ break;
+ case RTE_FLOW_ITEM_TYPE_UDP:
+ attr->udp = 1;
+ break;
+ case RTE_FLOW_ITEM_TYPE_TCP:
+ attr->tcp = 1;
+ break;
+ default:
+ break;
+ }
+ }
+ attr->valid = 1;
+}
+
+struct field_modify_info {
+ uint32_t size; /* Size of field in protocol header, in bytes. */
+ uint32_t offset; /* Offset of field in protocol header, in bytes. */
+ enum mlx5_modification_field id;
+};
+
+struct field_modify_info modify_eth[] = {
+ {4, 0, MLX5_MODI_OUT_DMAC_47_16},
+ {2, 4, MLX5_MODI_OUT_DMAC_15_0},
+ {4, 6, MLX5_MODI_OUT_SMAC_47_16},
+ {2, 10, MLX5_MODI_OUT_SMAC_15_0},
+ {0, 0, 0},
+};
+
+struct field_modify_info modify_ipv4[] = {
+ {1, 8, MLX5_MODI_OUT_IPV4_TTL},
+ {4, 12, MLX5_MODI_OUT_SIPV4},
+ {4, 16, MLX5_MODI_OUT_DIPV4},
+ {0, 0, 0},
+};
+
+struct field_modify_info modify_ipv6[] = {
+ {1, 7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
+ {4, 8, MLX5_MODI_OUT_SIPV6_127_96},
+ {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
+ {4, 16, MLX5_MODI_OUT_SIPV6_63_32},
+ {4, 20, MLX5_MODI_OUT_SIPV6_31_0},
+ {4, 24, MLX5_MODI_OUT_DIPV6_127_96},
+ {4, 28, MLX5_MODI_OUT_DIPV6_95_64},
+ {4, 32, MLX5_MODI_OUT_DIPV6_63_32},
+ {4, 36, MLX5_MODI_OUT_DIPV6_31_0},
+ {0, 0, 0},
+};
+
+struct field_modify_info modify_udp[] = {
+ {2, 0, MLX5_MODI_OUT_UDP_SPORT},
+ {2, 2, MLX5_MODI_OUT_UDP_DPORT},
+ {0, 0, 0},
+};
+
+struct field_modify_info modify_tcp[] = {
+ {2, 0, MLX5_MODI_OUT_TCP_SPORT},
+ {2, 2, MLX5_MODI_OUT_TCP_DPORT},
+ {0, 0, 0},
+};
+
+/**
+ * Acquire the synchronizing object to protect multithreaded access
+ * to shared dv context. Lock occurs only if context is actually
+ * shared, i.e. we have multiport IB device and representors are
+ * created.
+ *
+ * @param[in] dev
+ * Pointer to the rte_eth_dev structure.
+ */
+static void
+flow_d_shared_lock(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
+
+ if (sh->dv_refcnt > 1) {
+ int ret;
+
+ ret = pthread_mutex_lock(&sh->dv_mutex);
+ assert(!ret);
+ (void)ret;
+ }
+}
+
+static void
+flow_d_shared_unlock(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
+
+ if (sh->dv_refcnt > 1) {
+ int ret;
+
+ ret = pthread_mutex_unlock(&sh->dv_mutex);
+ assert(!ret);
+ (void)ret;
+ }
+}
+
+/**
+ * Convert modify-header action to DV specification.
+ *
+ * @param[in] item
+ * Pointer to item specification.
+ * @param[in] field
+ * Pointer to field modification information.
+ * @param[in,out] resource
+ * Pointer to the modify-header resource.
+ * @param[in] type
+ * Type of modification.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_modify_action(struct rte_flow_item *item,
+ struct field_modify_info *field,
+ struct mlx5_flow_dv_modify_hdr_resource *resource,
+ uint32_t type,
+ struct rte_flow_error *error)
+{
+ uint32_t i = resource->actions_num;
+ struct mlx5_modification_cmd *actions = resource->actions;
+ const uint8_t *spec = item->spec;
+ const uint8_t *mask = item->mask;
+ uint32_t set;
+
+ while (field->size) {
+ set = 0;
+ /* Generate modify command for each mask segment. */
+ memcpy(&set, &mask[field->offset], field->size);
+ if (set) {
+ if (i >= MLX5_MODIFY_NUM)
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "too many items to modify");
+ actions[i].action_type = type;
+ actions[i].field = field->id;
+ actions[i].length = field->size ==
+ 4 ? 0 : field->size * 8;
+ rte_memcpy(&actions[i].data[4 - field->size],
+ &spec[field->offset], field->size);
+ actions[i].data0 = rte_cpu_to_be_32(actions[i].data0);
+ ++i;
+ }
+ if (resource->actions_num != i)
+ resource->actions_num = i;
+ field++;
+ }
+ if (!resource->actions_num)
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "invalid modification flow item");
+ return 0;
+}
+
+/**
+ * Convert modify-header set IPv4 address action to DV specification.
+ *
+ * @param[in,out] resource
+ * Pointer to the modify-header resource.
+ * @param[in] action
+ * Pointer to action specification.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_action_modify_ipv4
+ (struct mlx5_flow_dv_modify_hdr_resource *resource,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error)
+{
+ const struct rte_flow_action_set_ipv4 *conf =
+ (const struct rte_flow_action_set_ipv4 *)(action->conf);
+ struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
+ struct rte_flow_item_ipv4 ipv4;
+ struct rte_flow_item_ipv4 ipv4_mask;
+
+ memset(&ipv4, 0, sizeof(ipv4));
+ memset(&ipv4_mask, 0, sizeof(ipv4_mask));
+ if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC) {
+ ipv4.hdr.src_addr = conf->ipv4_addr;
+ ipv4_mask.hdr.src_addr = rte_flow_item_ipv4_mask.hdr.src_addr;
+ } else {
+ ipv4.hdr.dst_addr = conf->ipv4_addr;
+ ipv4_mask.hdr.dst_addr = rte_flow_item_ipv4_mask.hdr.dst_addr;
+ }
+ item.spec = &ipv4;
+ item.mask = &ipv4_mask;
+ return flow_dv_convert_modify_action(&item, modify_ipv4, resource,
+ MLX5_MODIFICATION_TYPE_SET, error);
+}
+
+/**
+ * Convert modify-header set IPv6 address action to DV specification.
+ *
+ * @param[in,out] resource
+ * Pointer to the modify-header resource.
+ * @param[in] action
+ * Pointer to action specification.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_action_modify_ipv6
+ (struct mlx5_flow_dv_modify_hdr_resource *resource,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error)
+{
+ const struct rte_flow_action_set_ipv6 *conf =
+ (const struct rte_flow_action_set_ipv6 *)(action->conf);
+ struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
+ struct rte_flow_item_ipv6 ipv6;
+ struct rte_flow_item_ipv6 ipv6_mask;
+
+ memset(&ipv6, 0, sizeof(ipv6));
+ memset(&ipv6_mask, 0, sizeof(ipv6_mask));
+ if (action->type == RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC) {
+ memcpy(&ipv6.hdr.src_addr, &conf->ipv6_addr,
+ sizeof(ipv6.hdr.src_addr));
+ memcpy(&ipv6_mask.hdr.src_addr,
+ &rte_flow_item_ipv6_mask.hdr.src_addr,
+ sizeof(ipv6.hdr.src_addr));
+ } else {
+ memcpy(&ipv6.hdr.dst_addr, &conf->ipv6_addr,
+ sizeof(ipv6.hdr.dst_addr));
+ memcpy(&ipv6_mask.hdr.dst_addr,
+ &rte_flow_item_ipv6_mask.hdr.dst_addr,
+ sizeof(ipv6.hdr.dst_addr));
+ }
+ item.spec = &ipv6;
+ item.mask = &ipv6_mask;
+ return flow_dv_convert_modify_action(&item, modify_ipv6, resource,
+ MLX5_MODIFICATION_TYPE_SET, error);
+}
+
+/**
+ * Convert modify-header set MAC address action to DV specification.
+ *
+ * @param[in,out] resource
+ * Pointer to the modify-header resource.
+ * @param[in] action
+ * Pointer to action specification.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_action_modify_mac
+ (struct mlx5_flow_dv_modify_hdr_resource *resource,
+ const struct rte_flow_action *action,
+ struct rte_flow_error *error)
+{
+ const struct rte_flow_action_set_mac *conf =
+ (const struct rte_flow_action_set_mac *)(action->conf);
+ struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_ETH };
+ struct rte_flow_item_eth eth;
+ struct rte_flow_item_eth eth_mask;
+
+ memset(ð, 0, sizeof(eth));
+ memset(ð_mask, 0, sizeof(eth_mask));
+ if (action->type == RTE_FLOW_ACTION_TYPE_SET_MAC_SRC) {
+ memcpy(ð.src.addr_bytes, &conf->mac_addr,
+ sizeof(eth.src.addr_bytes));
+ memcpy(ð_mask.src.addr_bytes,
+ &rte_flow_item_eth_mask.src.addr_bytes,
+ sizeof(eth_mask.src.addr_bytes));
+ } else {
+ memcpy(ð.dst.addr_bytes, &conf->mac_addr,
+ sizeof(eth.dst.addr_bytes));
+ memcpy(ð_mask.dst.addr_bytes,
+ &rte_flow_item_eth_mask.dst.addr_bytes,
+ sizeof(eth_mask.dst.addr_bytes));
+ }
+ item.spec = ð
+ item.mask = ð_mask;
+ return flow_dv_convert_modify_action(&item, modify_eth, resource,
+ MLX5_MODIFICATION_TYPE_SET, error);
+}
+
+/**
+ * Convert modify-header set TP action to DV specification.
+ *
+ * @param[in,out] resource
+ * Pointer to the modify-header resource.
+ * @param[in] action
+ * Pointer to action specification.
+ * @param[in] items
+ * Pointer to rte_flow_item objects list.
+ * @param[in] attr
+ * Pointer to flow attributes structure.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_action_modify_tp
+ (struct mlx5_flow_dv_modify_hdr_resource *resource,
+ const struct rte_flow_action *action,
+ const struct rte_flow_item *items,
+ union flow_dv_attr *attr,
+ struct rte_flow_error *error)
+{
+ const struct rte_flow_action_set_tp *conf =
+ (const struct rte_flow_action_set_tp *)(action->conf);
+ struct rte_flow_item item;
+ struct rte_flow_item_udp udp;
+ struct rte_flow_item_udp udp_mask;
+ struct rte_flow_item_tcp tcp;
+ struct rte_flow_item_tcp tcp_mask;
+ struct field_modify_info *field;
+
+ if (!attr->valid)
+ flow_dv_attr_init(items, attr);
+ if (attr->udp) {
+ memset(&udp, 0, sizeof(udp));
+ memset(&udp_mask, 0, sizeof(udp_mask));
+ if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
+ udp.hdr.src_port = conf->port;
+ udp_mask.hdr.src_port =
+ rte_flow_item_udp_mask.hdr.src_port;
+ } else {
+ udp.hdr.dst_port = conf->port;
+ udp_mask.hdr.dst_port =
+ rte_flow_item_udp_mask.hdr.dst_port;
+ }
+ item.type = RTE_FLOW_ITEM_TYPE_UDP;
+ item.spec = &udp;
+ item.mask = &udp_mask;
+ field = modify_udp;
+ }
+ if (attr->tcp) {
+ memset(&tcp, 0, sizeof(tcp));
+ memset(&tcp_mask, 0, sizeof(tcp_mask));
+ if (action->type == RTE_FLOW_ACTION_TYPE_SET_TP_SRC) {
+ tcp.hdr.src_port = conf->port;
+ tcp_mask.hdr.src_port =
+ rte_flow_item_tcp_mask.hdr.src_port;
+ } else {
+ tcp.hdr.dst_port = conf->port;
+ tcp_mask.hdr.dst_port =
+ rte_flow_item_tcp_mask.hdr.dst_port;
+ }
+ item.type = RTE_FLOW_ITEM_TYPE_TCP;
+ item.spec = &tcp;
+ item.mask = &tcp_mask;
+ field = modify_tcp;
+ }
+ return flow_dv_convert_modify_action(&item, field, resource,
+ MLX5_MODIFICATION_TYPE_SET, error);
+}
+
+/**
+ * Convert modify-header set TTL action to DV specification.
+ *
+ * @param[in,out] resource
+ * Pointer to the modify-header resource.
+ * @param[in] action
+ * Pointer to action specification.
+ * @param[in] items
+ * Pointer to rte_flow_item objects list.
+ * @param[in] attr
+ * Pointer to flow attributes structure.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_action_modify_ttl
+ (struct mlx5_flow_dv_modify_hdr_resource *resource,
+ const struct rte_flow_action *action,
+ const struct rte_flow_item *items,
+ union flow_dv_attr *attr,
+ struct rte_flow_error *error)
+{
+ const struct rte_flow_action_set_ttl *conf =
+ (const struct rte_flow_action_set_ttl *)(action->conf);
+ struct rte_flow_item item;
+ struct rte_flow_item_ipv4 ipv4;
+ struct rte_flow_item_ipv4 ipv4_mask;
+ struct rte_flow_item_ipv6 ipv6;
+ struct rte_flow_item_ipv6 ipv6_mask;
+ struct field_modify_info *field;
+
+ if (!attr->valid)
+ flow_dv_attr_init(items, attr);
+ if (attr->ipv4) {
+ memset(&ipv4, 0, sizeof(ipv4));
+ memset(&ipv4_mask, 0, sizeof(ipv4_mask));
+ ipv4.hdr.time_to_live = conf->ttl_value;
+ ipv4_mask.hdr.time_to_live = 0xFF;
+ item.type = RTE_FLOW_ITEM_TYPE_IPV4;
+ item.spec = &ipv4;
+ item.mask = &ipv4_mask;
+ field = modify_ipv4;
+ }
+ if (attr->ipv6) {
+ memset(&ipv6, 0, sizeof(ipv6));
+ memset(&ipv6_mask, 0, sizeof(ipv6_mask));
+ ipv6.hdr.hop_limits = conf->ttl_value;
+ ipv6_mask.hdr.hop_limits = 0xFF;
+ item.type = RTE_FLOW_ITEM_TYPE_IPV6;
+ item.spec = &ipv6;
+ item.mask = &ipv6_mask;
+ field = modify_ipv6;
+ }
+ return flow_dv_convert_modify_action(&item, field, resource,
+ MLX5_MODIFICATION_TYPE_SET, error);
+}
+
+/**
+ * Convert modify-header decrement TTL action to DV specification.
+ *
+ * @param[in,out] resource
+ * Pointer to the modify-header resource.
+ * @param[in] action
+ * Pointer to action specification.
+ * @param[in] items
+ * Pointer to rte_flow_item objects list.
+ * @param[in] attr
+ * Pointer to flow attributes structure.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_action_modify_dec_ttl
+ (struct mlx5_flow_dv_modify_hdr_resource *resource,
+ const struct rte_flow_item *items,
+ union flow_dv_attr *attr,
+ struct rte_flow_error *error)
+{
+ struct rte_flow_item item;
+ struct rte_flow_item_ipv4 ipv4;
+ struct rte_flow_item_ipv4 ipv4_mask;
+ struct rte_flow_item_ipv6 ipv6;
+ struct rte_flow_item_ipv6 ipv6_mask;
+ struct field_modify_info *field;
+
+ if (!attr->valid)
+ flow_dv_attr_init(items, attr);
+ if (attr->ipv4) {
+ memset(&ipv4, 0, sizeof(ipv4));
+ memset(&ipv4_mask, 0, sizeof(ipv4_mask));
+ ipv4.hdr.time_to_live = 0xFF;
+ ipv4_mask.hdr.time_to_live = 0xFF;
+ item.type = RTE_FLOW_ITEM_TYPE_IPV4;
+ item.spec = &ipv4;
+ item.mask = &ipv4_mask;
+ field = modify_ipv4;
+ }
+ if (attr->ipv6) {
+ memset(&ipv6, 0, sizeof(ipv6));
+ memset(&ipv6_mask, 0, sizeof(ipv6_mask));
+ ipv6.hdr.hop_limits = 0xFF;
+ ipv6_mask.hdr.hop_limits = 0xFF;
+ item.type = RTE_FLOW_ITEM_TYPE_IPV6;
+ item.spec = &ipv6;
+ item.mask = &ipv6_mask;
+ field = modify_ipv6;
+ }
+ return flow_dv_convert_modify_action(&item, field, resource,
+ MLX5_MODIFICATION_TYPE_ADD, error);
+}
+