+/**
+ * Get metadata split action information.
+ *
+ * @param[in] actions
+ * Pointer to the list of actions.
+ * @param[out] qrss
+ * Pointer to the return pointer.
+ * @param[out] qrss_type
+ * Pointer to the action type to return. RTE_FLOW_ACTION_TYPE_END is returned
+ * if no QUEUE/RSS is found.
+ * @param[out] encap_idx
+ * Pointer to the index of the encap action if exists, otherwise the last
+ * action index.
+ *
+ * @return
+ * Total number of actions.
+ */
+static int
+flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[],
+ const struct rte_flow_action **qrss,
+ int *encap_idx)
+{
+ const struct rte_flow_action_raw_encap *raw_encap;
+ int actions_n = 0;
+ int raw_decap_idx = -1;
+
+ *encap_idx = -1;
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
+ case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
+ *encap_idx = actions_n;
+ break;
+ case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
+ raw_decap_idx = actions_n;
+ break;
+ case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
+ raw_encap = actions->conf;
+ if (raw_encap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
+ *encap_idx = raw_decap_idx != -1 ?
+ raw_decap_idx : actions_n;
+ break;
+ case RTE_FLOW_ACTION_TYPE_QUEUE:
+ case RTE_FLOW_ACTION_TYPE_RSS:
+ *qrss = actions;
+ break;
+ default:
+ break;
+ }
+ actions_n++;
+ }
+ if (*encap_idx == -1)
+ *encap_idx = actions_n;
+ /* Count RTE_FLOW_ACTION_TYPE_END. */
+ return actions_n + 1;
+}
+
+/**
+ * Check meter action from the action list.
+ *
+ * @param[in] actions
+ * Pointer to the list of actions.
+ * @param[out] mtr
+ * Pointer to the meter exist flag.
+ *
+ * @return
+ * Total number of actions.
+ */
+static int
+flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
+{
+ int actions_n = 0;
+
+ MLX5_ASSERT(mtr);
+ *mtr = 0;
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_METER:
+ *mtr = 1;
+ break;
+ default:
+ break;
+ }
+ actions_n++;
+ }
+ /* Count RTE_FLOW_ACTION_TYPE_END. */
+ return actions_n + 1;
+}
+
+/**
+ * Check if the flow should be splited due to hairpin.
+ * The reason for the split is that in current HW we can't
+ * support encap on Rx, so if a flow have encap we move it
+ * to Tx.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[in] attr
+ * Flow rule attributes.
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ *
+ * @return
+ * > 0 the number of actions and the flow should be split,
+ * 0 when no split required.
+ */
+static int
+flow_check_hairpin_split(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_action actions[])
+{
+ int queue_action = 0;
+ int action_n = 0;
+ int encap = 0;
+ const struct rte_flow_action_queue *queue;
+ const struct rte_flow_action_rss *rss;
+ const struct rte_flow_action_raw_encap *raw_encap;
+
+ if (!attr->ingress)
+ return 0;
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_QUEUE:
+ queue = actions->conf;
+ if (queue == NULL)
+ return 0;
+ if (mlx5_rxq_get_type(dev, queue->index) !=
+ MLX5_RXQ_TYPE_HAIRPIN)
+ return 0;
+ queue_action = 1;
+ action_n++;
+ break;
+ case RTE_FLOW_ACTION_TYPE_RSS:
+ rss = actions->conf;
+ if (rss == NULL || rss->queue_num == 0)
+ return 0;
+ if (mlx5_rxq_get_type(dev, rss->queue[0]) !=
+ MLX5_RXQ_TYPE_HAIRPIN)
+ return 0;
+ queue_action = 1;
+ action_n++;
+ break;
+ case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
+ case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
+ encap = 1;
+ action_n++;
+ break;
+ case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
+ raw_encap = actions->conf;
+ if (raw_encap->size >
+ (sizeof(struct rte_flow_item_eth) +
+ sizeof(struct rte_flow_item_ipv4)))
+ encap = 1;
+ action_n++;
+ break;
+ default:
+ action_n++;
+ break;
+ }
+ }
+ if (encap == 1 && queue_action)
+ return action_n;
+ return 0;
+}
+
+/* Declare flow create/destroy prototype in advance. */
+static struct rte_flow *
+flow_list_create(struct rte_eth_dev *dev, struct mlx5_flows *list,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external, struct rte_flow_error *error);
+
+static void
+flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
+ struct rte_flow *flow);
+
+/**
+ * Add a flow of copying flow metadata registers in RX_CP_TBL.
+ *
+ * As mark_id is unique, if there's already a registered flow for the mark_id,
+ * return by increasing the reference counter of the resource. Otherwise, create
+ * the resource (mcp_res) and flow.
+ *
+ * Flow looks like,
+ * - If ingress port is ANY and reg_c[1] is mark_id,
+ * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
+ *
+ * For default flow (zero mark_id), flow is like,
+ * - If ingress port is ANY,
+ * reg_b := reg_c[0] and jump to RX_ACT_TBL.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mark_id
+ * ID of MARK action, zero means default flow for META.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * Associated resource on success, NULL otherwise and rte_errno is set.
+ */
+static struct mlx5_flow_mreg_copy_resource *
+flow_mreg_add_copy_action(struct rte_eth_dev *dev, uint32_t mark_id,
+ struct rte_flow_error *error)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct rte_flow_attr attr = {
+ .group = MLX5_FLOW_MREG_CP_TABLE_GROUP,
+ .ingress = 1,
+ };
+ struct mlx5_rte_flow_item_tag tag_spec = {
+ .data = mark_id,
+ };
+ struct rte_flow_item items[] = {
+ [1] = { .type = RTE_FLOW_ITEM_TYPE_END, },
+ };
+ struct rte_flow_action_mark ftag = {
+ .id = mark_id,
+ };
+ struct mlx5_flow_action_copy_mreg cp_mreg = {
+ .dst = REG_B,
+ .src = 0,
+ };
+ struct rte_flow_action_jump jump = {
+ .group = MLX5_FLOW_MREG_ACT_TABLE_GROUP,
+ };
+ struct rte_flow_action actions[] = {
+ [3] = { .type = RTE_FLOW_ACTION_TYPE_END, },
+ };
+ struct mlx5_flow_mreg_copy_resource *mcp_res;
+ int ret;
+
+ /* Fill the register fileds in the flow. */
+ ret = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, error);
+ if (ret < 0)
+ return NULL;
+ tag_spec.id = ret;
+ ret = mlx5_flow_get_reg_id(dev, MLX5_METADATA_RX, 0, error);
+ if (ret < 0)
+ return NULL;
+ cp_mreg.src = ret;
+ /* Check if already registered. */
+ MLX5_ASSERT(priv->mreg_cp_tbl);
+ mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl, mark_id);
+ if (mcp_res) {
+ /* For non-default rule. */
+ if (mark_id != MLX5_DEFAULT_COPY_ID)
+ mcp_res->refcnt++;
+ MLX5_ASSERT(mark_id != MLX5_DEFAULT_COPY_ID ||
+ mcp_res->refcnt == 1);
+ return mcp_res;
+ }
+ /* Provide the full width of FLAG specific value. */
+ if (mark_id == (priv->sh->dv_regc0_mask & MLX5_FLOW_MARK_DEFAULT))
+ tag_spec.data = MLX5_FLOW_MARK_DEFAULT;
+ /* Build a new flow. */
+ if (mark_id != MLX5_DEFAULT_COPY_ID) {
+ items[0] = (struct rte_flow_item){
+ .type = MLX5_RTE_FLOW_ITEM_TYPE_TAG,
+ .spec = &tag_spec,
+ };
+ items[1] = (struct rte_flow_item){
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ };
+ actions[0] = (struct rte_flow_action){
+ .type = MLX5_RTE_FLOW_ACTION_TYPE_MARK,
+ .conf = &ftag,
+ };
+ actions[1] = (struct rte_flow_action){
+ .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
+ .conf = &cp_mreg,
+ };
+ actions[2] = (struct rte_flow_action){
+ .type = RTE_FLOW_ACTION_TYPE_JUMP,
+ .conf = &jump,
+ };
+ actions[3] = (struct rte_flow_action){
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ };
+ } else {
+ /* Default rule, wildcard match. */
+ attr.priority = MLX5_FLOW_PRIO_RSVD;
+ items[0] = (struct rte_flow_item){
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ };
+ actions[0] = (struct rte_flow_action){
+ .type = MLX5_RTE_FLOW_ACTION_TYPE_COPY_MREG,
+ .conf = &cp_mreg,
+ };
+ actions[1] = (struct rte_flow_action){
+ .type = RTE_FLOW_ACTION_TYPE_JUMP,
+ .conf = &jump,
+ };
+ actions[2] = (struct rte_flow_action){
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ };
+ }
+ /* Build a new entry. */
+ mcp_res = rte_zmalloc(__func__, sizeof(*mcp_res), 0);
+ if (!mcp_res) {
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+ /*
+ * The copy Flows are not included in any list. There
+ * ones are referenced from other Flows and can not
+ * be applied, removed, deleted in ardbitrary order
+ * by list traversing.
+ */
+ mcp_res->flow = flow_list_create(dev, NULL, &attr, items,
+ actions, false, error);
+ if (!mcp_res->flow)
+ goto error;
+ mcp_res->refcnt++;
+ mcp_res->hlist_ent.key = mark_id;
+ ret = mlx5_hlist_insert(priv->mreg_cp_tbl,
+ &mcp_res->hlist_ent);
+ MLX5_ASSERT(!ret);
+ if (ret)
+ goto error;
+ return mcp_res;
+error:
+ if (mcp_res->flow)
+ flow_list_destroy(dev, NULL, mcp_res->flow);
+ rte_free(mcp_res);
+ return NULL;
+}
+
+/**
+ * Release flow in RX_CP_TBL.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @flow
+ * Parent flow for wich copying is provided.
+ */
+static void
+flow_mreg_del_copy_action(struct rte_eth_dev *dev,
+ struct rte_flow *flow)
+{
+ struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ if (!mcp_res || !priv->mreg_cp_tbl)
+ return;
+ if (flow->copy_applied) {
+ MLX5_ASSERT(mcp_res->appcnt);
+ flow->copy_applied = 0;
+ --mcp_res->appcnt;
+ if (!mcp_res->appcnt)
+ flow_drv_remove(dev, mcp_res->flow);
+ }
+ /*
+ * We do not check availability of metadata registers here,
+ * because copy resources are not allocated in this case.
+ */
+ if (--mcp_res->refcnt)
+ return;
+ MLX5_ASSERT(mcp_res->flow);
+ flow_list_destroy(dev, NULL, mcp_res->flow);
+ mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
+ rte_free(mcp_res);
+ flow->mreg_copy = NULL;
+}
+
+/**
+ * Start flow in RX_CP_TBL.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @flow
+ * Parent flow for wich copying is provided.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_mreg_start_copy_action(struct rte_eth_dev *dev,
+ struct rte_flow *flow)
+{
+ struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
+ int ret;
+
+ if (!mcp_res || flow->copy_applied)
+ return 0;
+ if (!mcp_res->appcnt) {
+ ret = flow_drv_apply(dev, mcp_res->flow, NULL);
+ if (ret)
+ return ret;
+ }
+ ++mcp_res->appcnt;
+ flow->copy_applied = 1;
+ return 0;
+}
+
+/**
+ * Stop flow in RX_CP_TBL.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @flow
+ * Parent flow for wich copying is provided.
+ */
+static void
+flow_mreg_stop_copy_action(struct rte_eth_dev *dev,
+ struct rte_flow *flow)
+{
+ struct mlx5_flow_mreg_copy_resource *mcp_res = flow->mreg_copy;
+
+ if (!mcp_res || !flow->copy_applied)
+ return;
+ MLX5_ASSERT(mcp_res->appcnt);
+ --mcp_res->appcnt;
+ flow->copy_applied = 0;
+ if (!mcp_res->appcnt)
+ flow_drv_remove(dev, mcp_res->flow);
+}
+
+/**
+ * Remove the default copy action from RX_CP_TBL.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+flow_mreg_del_default_copy_action(struct rte_eth_dev *dev)
+{
+ struct mlx5_flow_mreg_copy_resource *mcp_res;
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ /* Check if default flow is registered. */
+ if (!priv->mreg_cp_tbl)
+ return;
+ mcp_res = (void *)mlx5_hlist_lookup(priv->mreg_cp_tbl,
+ MLX5_DEFAULT_COPY_ID);
+ if (!mcp_res)
+ return;
+ MLX5_ASSERT(mcp_res->flow);
+ flow_list_destroy(dev, NULL, mcp_res->flow);
+ mlx5_hlist_remove(priv->mreg_cp_tbl, &mcp_res->hlist_ent);
+ rte_free(mcp_res);
+}
+
+/**
+ * Add the default copy action in in RX_CP_TBL.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * 0 for success, negative value otherwise and rte_errno is set.
+ */
+static int
+flow_mreg_add_default_copy_action(struct rte_eth_dev *dev,
+ struct rte_flow_error *error)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_mreg_copy_resource *mcp_res;
+
+ /* Check whether extensive metadata feature is engaged. */
+ if (!priv->config.dv_flow_en ||
+ priv->config.dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
+ !mlx5_flow_ext_mreg_supported(dev) ||
+ !priv->sh->dv_regc0_mask)
+ return 0;
+ mcp_res = flow_mreg_add_copy_action(dev, MLX5_DEFAULT_COPY_ID, error);
+ if (!mcp_res)
+ return -rte_errno;
+ return 0;
+}
+
+/**
+ * Add a flow of copying flow metadata registers in RX_CP_TBL.
+ *
+ * All the flow having Q/RSS action should be split by
+ * flow_mreg_split_qrss_prep() to pass by RX_CP_TBL. A flow in the RX_CP_TBL
+ * performs the following,
+ * - CQE->flow_tag := reg_c[1] (MARK)
+ * - CQE->flow_table_metadata (reg_b) := reg_c[0] (META)
+ * As CQE's flow_tag is not a register, it can't be simply copied from reg_c[1]
+ * but there should be a flow per each MARK ID set by MARK action.
+ *
+ * For the aforementioned reason, if there's a MARK action in flow's action
+ * list, a corresponding flow should be added to the RX_CP_TBL in order to copy
+ * the MARK ID to CQE's flow_tag like,
+ * - If reg_c[1] is mark_id,
+ * flow_tag := mark_id, reg_b := reg_c[0] and jump to RX_ACT_TBL.
+ *
+ * For SET_META action which stores value in reg_c[0], as the destination is
+ * also a flow metadata register (reg_b), adding a default flow is enough. Zero
+ * MARK ID means the default flow. The default flow looks like,
+ * - For all flow, reg_b := reg_c[0] and jump to RX_ACT_TBL.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param flow
+ * Pointer to flow structure.
+ * @param[in] actions
+ * Pointer to the list of actions.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * 0 on success, negative value otherwise and rte_errno is set.
+ */
+static int
+flow_mreg_update_copy_table(struct rte_eth_dev *dev,
+ struct rte_flow *flow,
+ const struct rte_flow_action *actions,
+ struct rte_flow_error *error)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_dev_config *config = &priv->config;
+ struct mlx5_flow_mreg_copy_resource *mcp_res;
+ const struct rte_flow_action_mark *mark;
+
+ /* Check whether extensive metadata feature is engaged. */
+ if (!config->dv_flow_en ||
+ config->dv_xmeta_en == MLX5_XMETA_MODE_LEGACY ||
+ !mlx5_flow_ext_mreg_supported(dev) ||
+ !priv->sh->dv_regc0_mask)
+ return 0;
+ /* Find MARK action. */
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_FLAG:
+ mcp_res = flow_mreg_add_copy_action
+ (dev, MLX5_FLOW_MARK_DEFAULT, error);
+ if (!mcp_res)
+ return -rte_errno;
+ flow->mreg_copy = mcp_res;
+ if (dev->data->dev_started) {
+ mcp_res->appcnt++;
+ flow->copy_applied = 1;
+ }
+ return 0;
+ case RTE_FLOW_ACTION_TYPE_MARK:
+ mark = (const struct rte_flow_action_mark *)
+ actions->conf;
+ mcp_res =
+ flow_mreg_add_copy_action(dev, mark->id, error);
+ if (!mcp_res)
+ return -rte_errno;
+ flow->mreg_copy = mcp_res;
+ if (dev->data->dev_started) {
+ mcp_res->appcnt++;
+ flow->copy_applied = 1;
+ }
+ return 0;
+ default:
+ break;
+ }
+ }
+ return 0;
+}
+
+#define MLX5_MAX_SPLIT_ACTIONS 24
+#define MLX5_MAX_SPLIT_ITEMS 24
+
+/**
+ * Split the hairpin flow.
+ * Since HW can't support encap on Rx we move the encap to Tx.
+ * If the count action is after the encap then we also
+ * move the count action. in this case the count will also measure
+ * the outer bytes.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[out] actions_rx
+ * Rx flow actions.
+ * @param[out] actions_tx
+ * Tx flow actions..
+ * @param[out] pattern_tx
+ * The pattern items for the Tx flow.
+ * @param[out] flow_id
+ * The flow ID connected to this flow.
+ *
+ * @return
+ * 0 on success.
+ */
+static int
+flow_hairpin_split(struct rte_eth_dev *dev,
+ const struct rte_flow_action actions[],
+ struct rte_flow_action actions_rx[],
+ struct rte_flow_action actions_tx[],
+ struct rte_flow_item pattern_tx[],
+ uint32_t *flow_id)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ const struct rte_flow_action_raw_encap *raw_encap;
+ const struct rte_flow_action_raw_decap *raw_decap;
+ struct mlx5_rte_flow_action_set_tag *set_tag;
+ struct rte_flow_action *tag_action;
+ struct mlx5_rte_flow_item_tag *tag_item;
+ struct rte_flow_item *item;
+ char *addr;
+ int encap = 0;
+
+ mlx5_flow_id_get(priv->sh->flow_id_pool, flow_id);
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
+ case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
+ rte_memcpy(actions_tx, actions,
+ sizeof(struct rte_flow_action));
+ actions_tx++;
+ break;
+ case RTE_FLOW_ACTION_TYPE_COUNT:
+ if (encap) {
+ rte_memcpy(actions_tx, actions,
+ sizeof(struct rte_flow_action));
+ actions_tx++;
+ } else {
+ rte_memcpy(actions_rx, actions,
+ sizeof(struct rte_flow_action));
+ actions_rx++;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
+ raw_encap = actions->conf;
+ if (raw_encap->size >
+ (sizeof(struct rte_flow_item_eth) +
+ sizeof(struct rte_flow_item_ipv4))) {
+ memcpy(actions_tx, actions,
+ sizeof(struct rte_flow_action));
+ actions_tx++;
+ encap = 1;
+ } else {
+ rte_memcpy(actions_rx, actions,
+ sizeof(struct rte_flow_action));
+ actions_rx++;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
+ raw_decap = actions->conf;
+ if (raw_decap->size <
+ (sizeof(struct rte_flow_item_eth) +
+ sizeof(struct rte_flow_item_ipv4))) {
+ memcpy(actions_tx, actions,
+ sizeof(struct rte_flow_action));
+ actions_tx++;
+ } else {
+ rte_memcpy(actions_rx, actions,
+ sizeof(struct rte_flow_action));
+ actions_rx++;
+ }
+ break;
+ default:
+ rte_memcpy(actions_rx, actions,
+ sizeof(struct rte_flow_action));
+ actions_rx++;
+ break;
+ }
+ }
+ /* Add set meta action and end action for the Rx flow. */
+ tag_action = actions_rx;
+ tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG;
+ actions_rx++;
+ rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action));
+ actions_rx++;
+ set_tag = (void *)actions_rx;
+ set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL);
+ MLX5_ASSERT(set_tag->id > REG_NONE);
+ set_tag->data = *flow_id;
+ tag_action->conf = set_tag;
+ /* Create Tx item list. */
+ rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action));
+ addr = (void *)&pattern_tx[2];
+ item = pattern_tx;
+ item->type = MLX5_RTE_FLOW_ITEM_TYPE_TAG;
+ tag_item = (void *)addr;
+ tag_item->data = *flow_id;
+ tag_item->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_TX, 0, NULL);
+ MLX5_ASSERT(set_tag->id > REG_NONE);
+ item->spec = tag_item;
+ addr += sizeof(struct mlx5_rte_flow_item_tag);
+ tag_item = (void *)addr;
+ tag_item->data = UINT32_MAX;
+ tag_item->id = UINT16_MAX;
+ item->mask = tag_item;
+ addr += sizeof(struct mlx5_rte_flow_item_tag);
+ item->last = NULL;
+ item++;
+ item->type = RTE_FLOW_ITEM_TYPE_END;
+ return 0;
+}
+
+/**
+ * The last stage of splitting chain, just creates the subflow
+ * without any modification.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] flow
+ * Parent flow structure pointer.
+ * @param[in, out] sub_flow
+ * Pointer to return the created subflow, may be NULL.
+ * @param[in] prefix_layers
+ * Prefix subflow layers, may be 0.
+ * @param[in] attr
+ * Flow rule attributes.
+ * @param[in] items
+ * Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[in] external
+ * This flow rule is created by request external to PMD.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ * @return
+ * 0 on success, negative value otherwise
+ */
+static int
+flow_create_split_inner(struct rte_eth_dev *dev,
+ struct rte_flow *flow,
+ struct mlx5_flow **sub_flow,
+ uint64_t prefix_layers,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external, struct rte_flow_error *error)
+{
+ struct mlx5_flow *dev_flow;
+
+ dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, error);
+ if (!dev_flow)
+ return -rte_errno;
+ dev_flow->flow = flow;
+ dev_flow->external = external;
+ /* Subflow object was created, we must include one in the list. */
+ LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next);
+ /*
+ * If dev_flow is as one of the suffix flow, some actions in suffix
+ * flow may need some user defined item layer flags.
+ */
+ if (prefix_layers)
+ dev_flow->handle->layers = prefix_layers;
+ if (sub_flow)
+ *sub_flow = dev_flow;
+ return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
+}
+
+/**
+ * Split the meter flow.
+ *
+ * As meter flow will split to three sub flow, other than meter
+ * action, the other actions make sense to only meter accepts
+ * the packet. If it need to be dropped, no other additional
+ * actions should be take.
+ *
+ * One kind of special action which decapsulates the L3 tunnel
+ * header will be in the prefix sub flow, as not to take the
+ * L3 tunnel header into account.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[in] items
+ * Pattern specification (list terminated by the END pattern item).
+ * @param[out] sfx_items
+ * Suffix flow match items (list terminated by the END pattern item).
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[out] actions_sfx
+ * Suffix flow actions.
+ * @param[out] actions_pre
+ * Prefix flow actions.
+ * @param[out] pattern_sfx
+ * The pattern items for the suffix flow.
+ * @param[out] tag_sfx
+ * Pointer to suffix flow tag.
+ *
+ * @return
+ * 0 on success.
+ */
+static int
+flow_meter_split_prep(struct rte_eth_dev *dev,
+ const struct rte_flow_item items[],
+ struct rte_flow_item sfx_items[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_action actions_sfx[],
+ struct rte_flow_action actions_pre[])
+{
+ struct rte_flow_action *tag_action = NULL;
+ struct rte_flow_item *tag_item;
+ struct mlx5_rte_flow_action_set_tag *set_tag;
+ struct rte_flow_error error;
+ const struct rte_flow_action_raw_encap *raw_encap;
+ const struct rte_flow_action_raw_decap *raw_decap;
+ struct mlx5_rte_flow_item_tag *tag_spec;
+ struct mlx5_rte_flow_item_tag *tag_mask;
+ uint32_t tag_id;
+ bool copy_vlan = false;
+
+ /* Prepare the actions for prefix and suffix flow. */
+ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
+ struct rte_flow_action **action_cur = NULL;
+
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_METER:
+ /* Add the extra tag action first. */
+ tag_action = actions_pre;
+ tag_action->type = MLX5_RTE_FLOW_ACTION_TYPE_TAG;
+ actions_pre++;
+ action_cur = &actions_pre;
+ break;
+ case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
+ case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
+ action_cur = &actions_pre;
+ break;
+ case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
+ raw_encap = actions->conf;
+ if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
+ action_cur = &actions_pre;
+ break;
+ case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
+ raw_decap = actions->conf;
+ if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
+ action_cur = &actions_pre;
+ break;
+ case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
+ case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
+ copy_vlan = true;
+ break;
+ default:
+ break;
+ }
+ if (!action_cur)
+ action_cur = &actions_sfx;
+ memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
+ (*action_cur)++;