X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_flow_dv.c;h=bbce67b6768c124a0db4e78f001666ce9095af4f;hb=1485d961e2c9d77dfa4900a16b5ba70cbaa4c7ce;hp=7a012f7bb948ca8d968a4723c4f7f32270303e72;hpb=fe3620aabacd504a2b6c807bd0abe18cc1a2da12;p=dpdk.git

diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 7a012f7bb9..bbce67b676 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -162,6 +162,7 @@ flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
 		case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
 		case RTE_FLOW_ITEM_TYPE_GENEVE:
 		case RTE_FLOW_ITEM_TYPE_MPLS:
+		case RTE_FLOW_ITEM_TYPE_GTP:
 			if (tunnel_decap)
 				attr->attr = 0;
 			break;
@@ -2873,8 +2874,6 @@ flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
 {
 	const struct rte_flow_action_of_push_vlan *push_vlan = action->conf;
 	const struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	bool direction_error = false;
 
 	if (push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_VLAN) &&
 	    push_vlan->ethertype != RTE_BE16(RTE_ETHER_TYPE_QINQ))
@@ -2886,22 +2885,6 @@ flow_dv_validate_action_push_vlan(struct rte_eth_dev *dev,
 					  RTE_FLOW_ERROR_TYPE_ACTION, action,
 					  "wrong action order, port_id should "
 					  "be after push VLAN");
-	/* Push VLAN is not supported in ingress except for CX6 FDB mode. */
-	if (attr->transfer) {
-		bool fdb_tx = priv->representor_id != UINT16_MAX;
-		bool is_cx5 = sh->steering_format_version ==
-		    MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
-
-		if (!fdb_tx && is_cx5)
-			direction_error = true;
-	} else if (attr->ingress) {
-		direction_error = true;
-	}
-	if (direction_error)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
-					  NULL,
-					  "push vlan action not supported for ingress");
 	if (!attr->transfer && priv->representor)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -3277,6 +3260,25 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
 	return 0;
 }
 
+/**
+ * Indicates whether ASO aging is supported.
+ *
+ * @param[in] sh
+ *   Pointer to shared device context structure.
+ * @param[in] attr
+ *   Attributes of flow that includes AGE action.
+ *
+ * @return
+ *   True when ASO aging is supported, false otherwise.
+ */
+static inline bool
+flow_hit_aso_supported(const struct mlx5_dev_ctx_shared *sh,
+		const struct rte_flow_attr *attr)
+{
+	MLX5_ASSERT(sh && attr);
+	return (sh->flow_hit_aso_en && (attr->transfer || attr->group));
+}
+
 /**
  * Validate count action.
  *
@@ -3286,6 +3288,8 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
  *   Indicator if action is shared.
  * @param[in] action_flags
  *   Holds the actions detected until now.
+ * @param[in] attr
+ *   Attributes of flow that includes this action.
  * @param[out] error
  *   Pointer to error structure.
  *
@@ -3295,6 +3299,7 @@ flow_dv_validate_action_set_tag(struct rte_eth_dev *dev,
 static int
 flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
 			      uint64_t action_flags,
+			      const struct rte_flow_attr *attr,
 			      struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
@@ -3306,10 +3311,10 @@ flow_dv_validate_action_count(struct rte_eth_dev *dev, bool shared,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "duplicate count actions set");
 	if (shared && (action_flags & MLX5_FLOW_ACTION_AGE) &&
-	    !priv->sh->flow_hit_aso_en)
+	    !flow_hit_aso_supported(priv->sh, attr))
 		return rte_flow_error_set(error, EINVAL,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
-					  "old age and shared count combination is not supported");
+					  "old age and indirect count combination is not supported");
 #ifdef HAVE_IBV_FLOW_DEVX_COUNTERS
 	return 0;
 #endif
@@ -5679,7 +5684,7 @@ flow_dv_validate_action_sample(uint64_t *action_flags,
 		case RTE_FLOW_ACTION_TYPE_COUNT:
 			ret = flow_dv_validate_action_count
 				(dev, false, *action_flags | sub_action_flags,
-				 error);
+				 attr, error);
 			if (ret < 0)
 				return ret;
 			*count = act->conf;
@@ -5771,8 +5776,7 @@ flow_dv_validate_action_sample(uint64_t *action_flags,
 	}
 	/* Continue validation for Xcap actions.*/
 	if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) &&
-	    (queue_index == 0xFFFF ||
-	     mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN)) {
+	    (queue_index == 0xFFFF || !mlx5_rxq_is_hairpin(dev, queue_index))) {
 		if ((sub_action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
 		     MLX5_FLOW_XCAP_ACTIONS)
 			return rte_flow_error_set(error, ENOTSUP,
@@ -6530,13 +6534,13 @@ flow_dv_mtr_alloc(struct rte_eth_dev *dev)
 			struct mlx5_aso_mtr_pool,
 			mtrs[mtr_free->offset]);
 	mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
-	if (!mtr_free->fm.meter_action) {
+	if (!mtr_free->fm.meter_action_g) {
 #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
 		struct rte_flow_error error;
 		uint8_t reg_id;
 
 		reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
-		mtr_free->fm.meter_action =
+		mtr_free->fm.meter_action_g =
 			mlx5_glue->dv_create_flow_action_aso
 						(priv->sh->rx_domain,
 						 pool->devx_obj->obj,
@@ -6544,7 +6548,7 @@ flow_dv_mtr_alloc(struct rte_eth_dev *dev)
 						 (1 << MLX5_FLOW_COLOR_GREEN),
 						 reg_id - REG_C_0);
 #endif /* HAVE_MLX5_DR_CREATE_ACTION_ASO */
-		if (!mtr_free->fm.meter_action) {
+		if (!mtr_free->fm.meter_action_g) {
 			flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
 			return 0;
 		}
@@ -6841,7 +6845,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 		 bool external, int hairpin, struct rte_flow_error *error)
 {
 	int ret;
-	uint64_t action_flags = 0;
+	uint64_t aso_mask, action_flags = 0;
 	uint64_t item_flags = 0;
 	uint64_t last_item = 0;
 	uint8_t next_protocol = 0xff;
@@ -6908,7 +6912,11 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 	const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
 	const struct rte_flow_item *port_id_item = NULL;
 	bool def_policy = false;
+	bool shared_count = false;
 	uint16_t udp_dport = 0;
+	uint32_t tag_id = 0;
+	const struct rte_flow_action_age *non_shared_age = NULL;
+	const struct rte_flow_action_count *count = NULL;
 
 	if (items == NULL)
 		return -1;
@@ -6948,6 +6956,14 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 		switch (type) {
 		case RTE_FLOW_ITEM_TYPE_VOID:
 			break;
+		case RTE_FLOW_ITEM_TYPE_ESP:
+			ret = mlx5_flow_validate_item_esp(items, item_flags,
+							  next_protocol,
+							  error);
+			if (ret < 0)
+				return ret;
+			last_item = MLX5_FLOW_ITEM_ESP;
+			break;
 		case RTE_FLOW_ITEM_TYPE_PORT_ID:
 			ret = flow_dv_validate_item_port_id
 					(dev, items, attr, item_flags, error);
@@ -7109,6 +7125,13 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 			gre_item = items;
 			last_item = MLX5_FLOW_LAYER_GRE;
 			break;
+		case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
+			ret = mlx5_flow_validate_item_gre_option(dev, items, item_flags,
+							  attr, gre_item, error);
+			if (ret < 0)
+				return ret;
+			last_item = MLX5_FLOW_LAYER_GRE;
+			break;
 		case RTE_FLOW_ITEM_TYPE_NVGRE:
 			ret = mlx5_flow_validate_item_nvgre(items, item_flags,
 							    next_protocol,
@@ -7206,8 +7229,10 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 				return ret;
 			last_item = MLX5_FLOW_ITEM_TAG;
 			break;
-		case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
 		case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
+			last_item = MLX5_FLOW_ITEM_TX_QUEUE;
+			break;
+		case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
 			break;
 		case RTE_FLOW_ITEM_TYPE_GTP:
 			ret = flow_dv_validate_item_gtp(dev, items, item_flags,
@@ -7278,7 +7303,6 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 	}
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
 		int type = actions->type;
-		bool shared_count = false;
 
 		if (!mlx5_flow_os_action_supported(type))
 			return rte_flow_error_set(error, ENOTSUP,
@@ -7377,6 +7401,8 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 				++actions_n;
 			if (action_flags & MLX5_FLOW_ACTION_SAMPLE)
 				modify_after_mirror = 1;
+			tag_id = ((const struct rte_flow_action_set_tag *)
+				  actions->conf)->index;
 			action_flags |= MLX5_FLOW_ACTION_SET_TAG;
 			rw_act_num += MLX5_ACT_NUM_SET_TAG;
 			break;
@@ -7435,9 +7461,10 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 		case RTE_FLOW_ACTION_TYPE_COUNT:
 			ret = flow_dv_validate_action_count(dev, shared_count,
 							    action_flags,
-							    error);
+							    attr, error);
 			if (ret < 0)
 				return ret;
+			count = actions->conf;
 			action_flags |= MLX5_FLOW_ACTION_COUNT;
 			++actions_n;
 			break;
@@ -7743,6 +7770,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 			++actions_n;
 			break;
 		case RTE_FLOW_ACTION_TYPE_AGE:
+			non_shared_age = actions->conf;
 			ret = flow_dv_validate_action_age(action_flags,
 							  actions, dev,
 							  error);
@@ -7750,15 +7778,15 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 				return ret;
 			/*
 			 * Validate the regular AGE action (using counter)
-			 * mutual exclusion with share counter actions.
+			 * mutual exclusion with indirect counter actions.
 			 */
-			if (!priv->sh->flow_hit_aso_en) {
+			if (!flow_hit_aso_supported(priv->sh, attr)) {
 				if (shared_count)
 					return rte_flow_error_set
 						(error, EINVAL,
 						RTE_FLOW_ERROR_TYPE_ACTION,
 						NULL,
-						"old age and shared count combination is not supported");
+						"old age and indirect count combination is not supported");
 				if (sample_count)
 					return rte_flow_error_set
 						(error, EINVAL,
@@ -7811,6 +7839,11 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 							     error);
 			if (ret < 0)
 				return ret;
+			if ((action_flags & MLX5_FLOW_ACTION_SET_TAG) &&
+			    tag_id == 0 && priv->mtr_color_reg == REG_NON)
+				return rte_flow_error_set(error, EINVAL,
+					RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					"sample after tag action causes metadata tag index 0 corruption");
 			action_flags |= MLX5_FLOW_ACTION_SAMPLE;
 			++actions_n;
 			break;
@@ -7957,8 +7990,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 	 */
 	if ((action_flags & (MLX5_FLOW_XCAP_ACTIONS |
 			     MLX5_FLOW_VLAN_ACTIONS)) &&
-	    (queue_index == 0xFFFF ||
-	     mlx5_rxq_get_type(dev, queue_index) != MLX5_RXQ_TYPE_HAIRPIN ||
+	    (queue_index == 0xFFFF || !mlx5_rxq_is_hairpin(dev, queue_index) ||
 	     ((conf = mlx5_rxq_get_hairpin_conf(dev, queue_index)) != NULL &&
 	     conf->tx_explicit != 0))) {
 		if ((action_flags & MLX5_FLOW_XCAP_ACTIONS) ==
@@ -7967,6 +7999,28 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
 						  NULL, "encap and decap "
 						  "combination aren't supported");
+		/* Push VLAN is not supported in ingress except for NICs newer than CX5. */
+		if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN) {
+			struct mlx5_dev_ctx_shared *sh = priv->sh;
+			bool direction_error = false;
+
+			if (attr->transfer) {
+				bool fdb_tx = priv->representor_id != UINT16_MAX;
+				bool is_cx5 = sh->steering_format_version ==
+				    MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5;
+
+				if (!fdb_tx && is_cx5)
+					direction_error = true;
+			} else if (attr->ingress) {
+				direction_error = true;
+			}
+			if (direction_error)
+				return rte_flow_error_set(error, ENOTSUP,
+							  RTE_FLOW_ERROR_TYPE_ATTR_INGRESS,
+							  NULL,
+							  "push VLAN action not supported "
+							  "for ingress");
+		}
 		if (!attr->transfer && attr->ingress) {
 			if (action_flags & MLX5_FLOW_ACTION_ENCAP)
 				return rte_flow_error_set
@@ -7974,12 +8028,6 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 						 RTE_FLOW_ERROR_TYPE_ACTION,
 						 NULL, "encap is not supported"
 						 " for ingress traffic");
-			else if (action_flags & MLX5_FLOW_ACTION_OF_PUSH_VLAN)
-				return rte_flow_error_set
-						(error, ENOTSUP,
-						 RTE_FLOW_ERROR_TYPE_ACTION,
-						 NULL, "push VLAN action not "
-						 "supported for ingress");
 			else if ((action_flags & MLX5_FLOW_VLAN_ACTIONS) ==
 					MLX5_FLOW_VLAN_ACTIONS)
 				return rte_flow_error_set
@@ -8019,6 +8067,20 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 					"cannot be done before meter action");
 		}
 	}
+	/*
+	 * Only support one ASO action in a single flow rule.
+	 * non-shared AGE + counter will fallback to use HW counter, no ASO hit object.
+	 * Group 0 uses HW counter for AGE too even if no counter action.
+	 */
+	aso_mask = (action_flags & MLX5_FLOW_ACTION_METER && priv->sh->meter_aso_en) << 2 |
+		   (action_flags & MLX5_FLOW_ACTION_CT && priv->sh->ct_aso_en) << 1 |
+		   (action_flags & MLX5_FLOW_ACTION_AGE &&
+		    !(non_shared_age && count) &&
+		    (attr->group || (attr->transfer && priv->fdb_def_rule)) &&
+		    priv->sh->flow_hit_aso_en);
+	if (__builtin_popcountl(aso_mask) > 1)
+		return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
+					  NULL, "unsupported combining AGE, METER, CT ASO actions in a single rule");
 	/*
 	 * Hairpin flow will add one more TAG action in TX implicit mode.
 	 * In TX explicit mode, there will be no hairpin flow ID.
@@ -8042,6 +8104,18 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
 		return rte_flow_error_set(error, EINVAL,
 				RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 				"sample before modify action is not supported");
+	/*
+	 * Validation the NIC Egress flow on representor, except implicit
+	 * hairpin default egress flow with TX_QUEUE item, other flows not
+	 * work due to metadata regC0 mismatch.
+	 */
+	if ((!attr->transfer && attr->egress) && priv->representor &&
+	    !(item_flags & MLX5_FLOW_ITEM_TX_QUEUE))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_ITEM,
+					  NULL,
+					  "NIC egress rules on representors"
+					  " is not supported");
 	return 0;
 }
 
@@ -8664,6 +8738,58 @@ flow_dv_translate_item_tcp(void *matcher, void *key,
 		 (tcp_v->hdr.tcp_flags & tcp_m->hdr.tcp_flags));
 }
 
+/**
+ * Add ESP item to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] inner
+ *   Item is inner pattern.
+ */
+static void
+flow_dv_translate_item_esp(void *matcher, void *key,
+			   const struct rte_flow_item *item,
+			   int inner)
+{
+	const struct rte_flow_item_esp *esp_m = item->mask;
+	const struct rte_flow_item_esp *esp_v = item->spec;
+	void *headers_m;
+	void *headers_v;
+	char *spi_m;
+	char *spi_v;
+
+	if (inner) {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 inner_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, inner_headers);
+	} else {
+		headers_m = MLX5_ADDR_OF(fte_match_param, matcher,
+					 outer_headers);
+		headers_v = MLX5_ADDR_OF(fte_match_param, key, outer_headers);
+	}
+	MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 0xff);
+	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_ESP);
+	if (!esp_v)
+		return;
+	if (!esp_m)
+		esp_m = &rte_flow_item_esp_mask;
+	headers_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
+	headers_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
+	if (inner) {
+		spi_m = MLX5_ADDR_OF(fte_match_set_misc, headers_m, inner_esp_spi);
+		spi_v = MLX5_ADDR_OF(fte_match_set_misc, headers_v, inner_esp_spi);
+	} else {
+		spi_m = MLX5_ADDR_OF(fte_match_set_misc, headers_m, outer_esp_spi);
+		spi_v = MLX5_ADDR_OF(fte_match_set_misc, headers_v, outer_esp_spi);
+	}
+	*(uint32_t *)spi_m = esp_m->hdr.spi;
+	*(uint32_t *)spi_v = esp_m->hdr.spi & esp_v->hdr.spi;
+}
+
 /**
  * Add UDP item to matcher and to the value.
  *
@@ -8829,6 +8955,110 @@ flow_dv_translate_item_gre(void *matcher, void *key,
 		 protocol_m & protocol_v);
 }
 
+/**
+ * Add GRE optional items to matcher and to the value.
+ *
+ * @param[in, out] matcher
+ *   Flow matcher.
+ * @param[in, out] key
+ *   Flow matcher value.
+ * @param[in] item
+ *   Flow pattern to translate.
+ * @param[in] gre_item
+ *   Pointer to gre_item.
+ * @param[in] pattern_flags
+ *   Accumulated pattern flags.
+ */
+static void
+flow_dv_translate_item_gre_option(void *matcher, void *key,
+				  const struct rte_flow_item *item,
+				  const struct rte_flow_item *gre_item,
+				  uint64_t pattern_flags)
+{
+	const struct rte_flow_item_gre_opt *option_m = item->mask;
+	const struct rte_flow_item_gre_opt *option_v = item->spec;
+	const struct rte_flow_item_gre *gre_m = gre_item->mask;
+	const struct rte_flow_item_gre *gre_v = gre_item->spec;
+	static const struct rte_flow_item_gre empty_gre = {0};
+	struct rte_flow_item gre_key_item;
+	uint16_t c_rsvd0_ver_m, c_rsvd0_ver_v;
+	uint16_t protocol_m, protocol_v;
+	void *misc5_m;
+	void *misc5_v;
+
+	/*
+	 * If only match key field, keep using misc for matching.
+	 * If need to match checksum or sequence, using misc5 and do
+	 * not need using misc.
+	 */
+	if (!(option_m->sequence.sequence ||
+	      option_m->checksum_rsvd.checksum)) {
+		flow_dv_translate_item_gre(matcher, key, gre_item,
+					   pattern_flags);
+		gre_key_item.spec = &option_v->key.key;
+		gre_key_item.mask = &option_m->key.key;
+		flow_dv_translate_item_gre_key(matcher, key, &gre_key_item);
+		return;
+	}
+	if (!gre_v) {
+		gre_v = &empty_gre;
+		gre_m = &empty_gre;
+	} else {
+		if (!gre_m)
+			gre_m = &rte_flow_item_gre_mask;
+	}
+	protocol_v = gre_v->protocol;
+	protocol_m = gre_m->protocol;
+	if (!protocol_m) {
+		/* Force next protocol to prevent matchers duplication */
+		uint16_t ether_type =
+			mlx5_translate_tunnel_etypes(pattern_flags);
+		if (ether_type) {
+			protocol_v = rte_be_to_cpu_16(ether_type);
+			protocol_m = UINT16_MAX;
+		}
+	}
+	c_rsvd0_ver_v = gre_v->c_rsvd0_ver;
+	c_rsvd0_ver_m = gre_m->c_rsvd0_ver;
+	if (option_m->sequence.sequence) {
+		c_rsvd0_ver_v |= RTE_BE16(0x1000);
+		c_rsvd0_ver_m |= RTE_BE16(0x1000);
+	}
+	if (option_m->key.key) {
+		c_rsvd0_ver_v |= RTE_BE16(0x2000);
+		c_rsvd0_ver_m |= RTE_BE16(0x2000);
+	}
+	if (option_m->checksum_rsvd.checksum) {
+		c_rsvd0_ver_v |= RTE_BE16(0x8000);
+		c_rsvd0_ver_m |= RTE_BE16(0x8000);
+	}
+	/*
+	 * Hardware parses GRE optional field into the fixed location,
+	 * do not need to adjust the tunnel dword indices.
+	 */
+	misc5_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_5);
+	misc5_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_5);
+	MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_0,
+		 rte_be_to_cpu_32((c_rsvd0_ver_v | protocol_v << 16) &
+				  (c_rsvd0_ver_m | protocol_m << 16)));
+	MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_0,
+		 rte_be_to_cpu_32(c_rsvd0_ver_m | protocol_m << 16));
+	MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_1,
+		 rte_be_to_cpu_32(option_v->checksum_rsvd.checksum &
+				  option_m->checksum_rsvd.checksum));
+	MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_1,
+		 rte_be_to_cpu_32(option_m->checksum_rsvd.checksum));
+	MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_2,
+		 rte_be_to_cpu_32(option_v->key.key & option_m->key.key));
+	MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_2,
+		 rte_be_to_cpu_32(option_m->key.key));
+	MLX5_SET(fte_match_set_misc5, misc5_v, tunnel_header_3,
+		 rte_be_to_cpu_32(option_v->sequence.sequence &
+				  option_m->sequence.sequence));
+	MLX5_SET(fte_match_set_misc5, misc5_m, tunnel_header_3,
+		 rte_be_to_cpu_32(option_m->sequence.sequence));
+}
+
 /**
  * Add NVGRE item to matcher and to the value.
  *
@@ -10167,7 +10397,7 @@ flow_dv_translate_item_flex(struct rte_eth_dev *dev, void *matcher, void *key,
 		/* Don't count both inner and outer flex items in one rule. */
 		if (mlx5_flex_acquire_index(dev, spec->handle, true) != index)
 			MLX5_ASSERT(false);
-		dev_flow->handle->flex_item |= RTE_BIT32(index);
+		dev_flow->handle->flex_item |= (uint8_t)RTE_BIT32(index);
 	}
 	mlx5_flex_flow_translate_item(dev, matcher, key, item, is_inner);
 }
@@ -10511,7 +10741,8 @@ flow_dv_tbl_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
 			tbl_data->tunnel->tunnel_id : 0,
 			tbl_data->group_id);
 	}
-	mlx5_list_destroy(tbl_data->matchers);
+	if (tbl_data->matchers)
+		mlx5_list_destroy(tbl_data->matchers);
 	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
@@ -10948,10 +11179,8 @@ flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
 {
 	const struct mlx5_rte_flow_item_tx_queue *queue_m;
 	const struct mlx5_rte_flow_item_tx_queue *queue_v;
-	void *misc_m =
-		MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
-	void *misc_v =
-		MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
+	void *misc_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters);
+	void *misc_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters);
 	struct mlx5_txq_ctrl *txq;
 	uint32_t queue, mask;
 
@@ -10962,7 +11191,7 @@ flow_dv_translate_item_tx_queue(struct rte_eth_dev *dev,
 	txq = mlx5_txq_get(dev, queue_v->queue);
 	if (!txq)
 		return;
-	if (txq->type == MLX5_TXQ_TYPE_HAIRPIN)
+	if (txq->is_hairpin)
 		queue = txq->obj->sq->id;
 	else
 		queue = txq->obj->sq_obj.sq->id;
@@ -11020,12 +11249,18 @@ flow_dv_hashfields_set(uint64_t item_flags,
 				fields |= MLX5_IPV6_IBV_RX_HASH;
 		}
 	}
-	if (fields == 0)
+	if (items & MLX5_FLOW_ITEM_ESP) {
+		if (rss_types & RTE_ETH_RSS_ESP)
+			fields |= IBV_RX_HASH_IPSEC_SPI;
+	}
+	if ((fields & ~IBV_RX_HASH_IPSEC_SPI) == 0) {
+		*hash_fields = fields;
 		/*
 		 * There is no match between the RSS types and the
 		 * L3 protocol (IPv4/IPv6) defined in the flow rule.
 		 */
 		return;
+	}
 	if ((rss_inner && (items & MLX5_FLOW_LAYER_INNER_L4_UDP)) ||
 	    (!rss_inner && (items & MLX5_FLOW_LAYER_OUTER_L4_UDP)) ||
 	    !items) {
@@ -11076,6 +11311,7 @@ flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
 		     uint32_t *hrxq_idx)
 {
 	struct mlx5_flow_handle *dh = dev_flow->handle;
+	uint32_t shared_rss = rss_desc->shared_rss;
 	struct mlx5_hrxq *hrxq;
 
 	MLX5_ASSERT(rss_desc->queue_num);
@@ -11087,6 +11323,7 @@ flow_dv_hrxq_prepare(struct rte_eth_dev *dev,
 		rss_desc->queue_num = 1;
 	hrxq = mlx5_hrxq_get(dev, rss_desc);
 	*hrxq_idx = hrxq ? hrxq->idx : 0;
+	rss_desc->shared_rss = shared_rss;
 	return hrxq;
 }
 
@@ -12709,6 +12946,7 @@ flow_dv_translate(struct rte_eth_dev *dev,
 	};
 	const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
 	const struct rte_flow_item *tunnel_item = NULL;
+	const struct rte_flow_item *gre_item = NULL;
 
 	if (!wks)
 		return rte_flow_error_set(error, ENOMEM,
@@ -13229,7 +13467,7 @@ flow_dv_translate(struct rte_eth_dev *dev,
 					NULL, "Failed to get meter in flow.");
 			/* Set the meter action. */
 			dev_flow->dv.actions[actions_n++] =
-				wks->fm->meter_action;
+				wks->fm->meter_action_g;
 			action_flags |= MLX5_FLOW_ACTION_METER;
 			break;
 		case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
@@ -13307,8 +13545,7 @@ flow_dv_translate(struct rte_eth_dev *dev,
 			 */
 			if (action_flags & MLX5_FLOW_ACTION_AGE) {
 				if ((non_shared_age && count) ||
-				    !(priv->sh->flow_hit_aso_en &&
-				      (attr->group || attr->transfer))) {
+				    !flow_hit_aso_supported(priv->sh, attr)) {
 					/* Creates age by counters. */
 					cnt_act = flow_dv_prepare_counter
 								(dev, dev_flow,
@@ -13368,6 +13605,11 @@ flow_dv_translate(struct rte_eth_dev *dev,
 						  RTE_FLOW_ERROR_TYPE_ITEM,
 						  NULL, "item not supported");
 		switch (item_type) {
+		case RTE_FLOW_ITEM_TYPE_ESP:
+			flow_dv_translate_item_esp(match_mask, match_value,
+						   items, tunnel);
+			last_item = MLX5_FLOW_ITEM_ESP;
+			break;
 		case RTE_FLOW_ITEM_TYPE_PORT_ID:
 			flow_dv_translate_item_port_id
 				(dev, match_mask, match_value, items, attr);
@@ -13481,12 +13723,18 @@ flow_dv_translate(struct rte_eth_dev *dev,
 			matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
 			last_item = MLX5_FLOW_LAYER_GRE;
 			tunnel_item = items;
+			gre_item = items;
 			break;
 		case RTE_FLOW_ITEM_TYPE_GRE_KEY:
 			flow_dv_translate_item_gre_key(match_mask,
 						       match_value, items);
 			last_item = MLX5_FLOW_LAYER_GRE_KEY;
 			break;
+		case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
+			matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
+			last_item = MLX5_FLOW_LAYER_GRE;
+			tunnel_item = items;
+			break;
 		case RTE_FLOW_ITEM_TYPE_NVGRE:
 			matcher.priority = MLX5_TUNNEL_PRIO_GET(rss_desc);
 			last_item = MLX5_FLOW_LAYER_GRE;
@@ -13539,11 +13787,13 @@ flow_dv_translate(struct rte_eth_dev *dev,
 		case RTE_FLOW_ITEM_TYPE_ICMP:
 			flow_dv_translate_item_icmp(match_mask, match_value,
 						    items, tunnel);
+			matcher.priority = MLX5_PRIORITY_MAP_L4;
 			last_item = MLX5_FLOW_LAYER_ICMP;
 			break;
 		case RTE_FLOW_ITEM_TYPE_ICMP6:
 			flow_dv_translate_item_icmp6(match_mask, match_value,
 						      items, tunnel);
+			matcher.priority = MLX5_PRIORITY_MAP_L4;
 			last_item = MLX5_FLOW_LAYER_ICMP6;
 			break;
 		case RTE_FLOW_ITEM_TYPE_TAG:
@@ -13618,11 +13868,13 @@ flow_dv_translate(struct rte_eth_dev *dev,
 	/*
 	 * When E-Switch mode is enabled, we have two cases where we need to
 	 * set the source port manually.
-	 * The first one, is in case of Nic steering rule, and the second is
-	 * E-Switch rule where no port_id item was found. In both cases
-	 * the source port is set according the current port in use.
+	 * The first one, is in case of NIC ingress steering rule, and the
+	 * second is E-Switch rule where no port_id item was found.
+	 * In both cases the source port is set according the current port
+	 * in use.
 	 */
-	if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && priv->sh->esw_mode) {
+	if (!(item_flags & MLX5_FLOW_ITEM_PORT_ID) && priv->sh->esw_mode &&
+	    !(attr->egress && !attr->transfer)) {
 		if (flow_dv_translate_item_port_id(dev, match_mask,
 						   match_value, NULL, attr))
 			return -rte_errno;
@@ -13645,6 +13897,9 @@ flow_dv_translate(struct rte_eth_dev *dev,
 		else if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_NVGRE)
 			flow_dv_translate_item_nvgre(match_mask, match_value,
 						     tunnel_item, item_flags);
+		else if (tunnel_item->type == RTE_FLOW_ITEM_TYPE_GRE_OPTION)
+			flow_dv_translate_item_gre_option(match_mask, match_value,
+					tunnel_item, gre_item, item_flags);
 		else
 			MLX5_ASSERT(false);
 	}
@@ -13824,6 +14079,15 @@ __flow_dv_action_rss_hrxq_set(struct mlx5_shared_action_rss *action,
 	case MLX5_RSS_HASH_NONE:
 		hrxqs[6] = hrxq_idx;
 		return 0;
+	case MLX5_RSS_HASH_IPV4_ESP:
+		hrxqs[7] = hrxq_idx;
+		return 0;
+	case MLX5_RSS_HASH_IPV6_ESP:
+		hrxqs[8] = hrxq_idx;
+		return 0;
+	case MLX5_RSS_HASH_ESP_SPI:
+		hrxqs[9] = hrxq_idx;
+		return 0;
 	default:
 		return -1;
 	}
@@ -13893,6 +14157,12 @@ flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
 		return hrxqs[5];
 	case MLX5_RSS_HASH_NONE:
 		return hrxqs[6];
+	case MLX5_RSS_HASH_IPV4_ESP:
+		return hrxqs[7];
+	case MLX5_RSS_HASH_IPV6_ESP:
+		return hrxqs[8];
+	case MLX5_RSS_HASH_ESP_SPI:
+		return hrxqs[9];
 	default:
 		return 0;
 	}
@@ -14510,7 +14780,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 			int index = rte_bsf32(dev_handle->flex_item);
 
 			mlx5_flex_release_index(dev, index);
-			dev_handle->flex_item &= ~RTE_BIT32(index);
+			dev_handle->flex_item &= ~(uint8_t)RTE_BIT32(index);
 		}
 		if (dev_handle->dvh.matcher)
 			flow_dv_matcher_release(dev, dev_handle);
@@ -14609,8 +14879,8 @@ __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
  * MLX5_RSS_HASH_IPV4_DST_ONLY are mutually exclusive so they can share
  * same slot in mlx5_rss_hash_fields.
  *
- * @param[in] rss_types
- *   RSS type.
+ * @param[in] orig_rss_types
+ *   RSS type as provided in shared RSS action.
  * @param[in, out] hash_field
  *   hash_field variable needed to be adjusted.
  *
@@ -14618,9 +14888,11 @@ __flow_dv_action_rss_hrxqs_release(struct rte_eth_dev *dev,
  *   void
  */
 void
-flow_dv_action_rss_l34_hash_adjust(uint64_t rss_types,
+flow_dv_action_rss_l34_hash_adjust(uint64_t orig_rss_types,
 				   uint64_t *hash_field)
 {
+	uint64_t rss_types = rte_eth_rss_hf_refine(orig_rss_types);
+
 	switch (*hash_field & ~IBV_RX_HASH_INNER) {
 	case MLX5_RSS_HASH_IPV4:
 		if (rss_types & MLX5_IPV4_LAYER_TYPES) {
@@ -15241,7 +15513,7 @@ __flow_dv_destroy_sub_policy_rules(struct rte_eth_dev *dev,
 
 	for (i = 0; i < RTE_COLORS; i++) {
 		next_fm = NULL;
-		if (i == RTE_COLOR_GREEN && policy &&
+		if (i <= RTE_COLOR_YELLOW && policy &&
 		    policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
 			next_fm = mlx5_flow_meter_find(priv,
 					policy->act_cnt[i].next_mtr_id, NULL);
@@ -15365,6 +15637,51 @@ flow_dv_destroy_mtr_policy_acts(struct rte_eth_dev *dev,
 		mtr_policy->dr_drop_action[j] = NULL;
 }
 
+/**
+ * Create yellow action for color aware meter.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in] fm
+ *   Meter information table.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL. Initialized in case of
+ *   error only.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+__flow_dv_create_mtr_yellow_action(struct rte_eth_dev *dev,
+				   struct mlx5_flow_meter_info *fm,
+				   struct rte_mtr_error *error)
+{
+#ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct rte_flow_error flow_err;
+	struct mlx5_aso_mtr *aso_mtr;
+	struct mlx5_aso_mtr_pool *pool;
+	uint8_t reg_id;
+
+	aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
+	pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool, mtrs[aso_mtr->offset]);
+	reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &flow_err);
+	fm->meter_action_y =
+		mlx5_glue->dv_create_flow_action_aso(priv->sh->rx_domain,
+						     pool->devx_obj->obj,
+						     aso_mtr->offset,
+						     (1 << MLX5_FLOW_COLOR_YELLOW),
+						     reg_id - REG_C_0);
+#else
+	RTE_SET_USED(dev);
+#endif
+	if (!fm->meter_action_y) {
+		return -rte_mtr_error_set(error, EINVAL, RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+					  "Fail to create yellow meter action.");
+	}
+	return 0;
+}
+
 /**
  * Create policy action per domain, lock free,
  * (mutex should be acquired by caller).
@@ -15407,9 +15724,7 @@ __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
 			    (MLX5_MAX_MODIFY_NUM + 1)];
 	} mhdr_dummy;
 	struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
-	struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
-	MLX5_ASSERT(wks);
 	egress = (domain == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
 	transfer = (domain == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
 	memset(&dh, 0, sizeof(struct mlx5_flow_handle));
@@ -15447,7 +15762,6 @@ __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
 					  NULL,
 					  "cannot create policy "
 					  "mark action for this color");
-				wks->mark = 1;
 				if (flow_dv_tag_resource_register(dev, tag_be,
 						  &dev_flow, &flow_err))
 					return -rte_mtr_error_set(error,
@@ -15459,6 +15773,7 @@ __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
 				act_cnt->rix_mark =
 					dev_flow.handle->dvh.rix_tag;
 				action_flags |= MLX5_FLOW_ACTION_MARK;
+				mtr_policy->mark = 1;
 				break;
 			}
 			case RTE_FLOW_ACTION_TYPE_SET_TAG:
@@ -15686,7 +16001,7 @@ __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
 				break;
 			}
 			/*
-			 * No need to check meter hierarchy for Y or R colors
+			 * No need to check meter hierarchy for R colors
 			 * here since it is done in the validation stage.
 			 */
 			case RTE_FLOW_ACTION_TYPE_METER:
@@ -15737,11 +16052,17 @@ __flow_dv_create_domain_policy_acts(struct rte_eth_dev *dev,
 					action_flags |=
 						MLX5_FLOW_ACTION_SET_TAG;
 				}
+				if (i == RTE_COLOR_YELLOW && next_fm->color_aware &&
+				    !next_fm->meter_action_y)
+					if (__flow_dv_create_mtr_yellow_action(dev, next_fm, error))
+						return -rte_errno;
 				act_cnt->fate_action = MLX5_FLOW_FATE_MTR;
 				act_cnt->next_mtr_id = next_fm->meter_id;
 				act_cnt->next_sub_policy = NULL;
 				mtr_policy->is_hierarchy = 1;
 				mtr_policy->dev = next_policy->dev;
+				if (next_policy->mark)
+					mtr_policy->mark = 1;
 				action_flags |=
 				MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
 				break;
@@ -16333,7 +16654,7 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
 	struct mlx5_flow_dv_tag_resource *tag;
 	struct mlx5_flow_dv_port_id_action_resource *port_action;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_flow_meter_info *next_fm = NULL;
+	struct mlx5_flow_meter_info *next_fm[RTE_COLORS] = {NULL};
 	struct mlx5_flow_meter_policy *next_policy;
 	struct mlx5_flow_meter_sub_policy *next_sub_policy;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
@@ -16354,30 +16675,31 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
 			acts[i].actions_n = 1;
 			continue;
 		}
-		if (i == RTE_COLOR_GREEN &&
-		    mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
+		if (mtr_policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR) {
 			struct rte_flow_attr attr = {
 				.transfer = transfer
 			};
 
-			next_fm = mlx5_flow_meter_find(priv,
+			next_fm[i] = mlx5_flow_meter_find(priv,
 					mtr_policy->act_cnt[i].next_mtr_id,
 					NULL);
-			if (!next_fm) {
+			if (!next_fm[i]) {
 				DRV_LOG(ERR,
 					"Failed to get next hierarchy meter.");
 				goto err_exit;
 			}
-			if (mlx5_flow_meter_attach(priv, next_fm,
+			if (mlx5_flow_meter_attach(priv, next_fm[i],
 						   &attr, &error)) {
 				DRV_LOG(ERR, "%s", error.message);
-				next_fm = NULL;
+				next_fm[i] = NULL;
 				goto err_exit;
 			}
 			/* Meter action must be the first for TX. */
 			if (mtr_first) {
 				acts[i].dv_actions[acts[i].actions_n] =
-					next_fm->meter_action;
+					(next_fm[i]->color_aware && i == RTE_COLOR_YELLOW) ?
+						next_fm[i]->meter_action_y :
+						next_fm[i]->meter_action_g;
 				acts[i].actions_n++;
 			}
 		}
@@ -16435,14 +16757,16 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
 				acts[i].actions_n++;
 				break;
 			case MLX5_FLOW_FATE_MTR:
-				if (!next_fm) {
+				if (!next_fm[i]) {
 					DRV_LOG(ERR,
 						"No next hierarchy meter.");
 					goto err_exit;
 				}
 				if (!mtr_first) {
 					acts[i].dv_actions[acts[i].actions_n] =
-							next_fm->meter_action;
+						(next_fm[i]->color_aware && i == RTE_COLOR_YELLOW) ?
+							next_fm[i]->meter_action_y :
+							next_fm[i]->meter_action_g;
 					acts[i].actions_n++;
 				}
 				if (mtr_policy->act_cnt[i].next_sub_policy) {
@@ -16451,7 +16775,7 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
 				} else {
 					next_policy =
 						mlx5_flow_meter_policy_find(dev,
-						next_fm->policy_id, NULL);
+								next_fm[i]->policy_id, NULL);
 					MLX5_ASSERT(next_policy);
 					next_sub_policy =
 					next_policy->sub_policys[domain][0];
@@ -16478,8 +16802,9 @@ __flow_dv_create_policy_acts_rules(struct rte_eth_dev *dev,
 	}
 	return 0;
 err_exit:
-	if (next_fm)
-		mlx5_flow_meter_detach(priv, next_fm);
+	for (i = 0; i < RTE_COLORS; i++)
+		if (next_fm[i])
+			mlx5_flow_meter_detach(priv, next_fm[i]);
 	return -1;
 }
 
@@ -16868,7 +17193,8 @@ __flow_dv_meter_get_rss_sub_policy(struct rte_eth_dev *dev,
 		}
 	}
 	/* Create sub policy. */
-	if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[0]) {
+	if (!mtr_policy->sub_policys[domain][0]->rix_hrxq[RTE_COLOR_GREEN] &&
+	    !mtr_policy->sub_policys[domain][0]->rix_hrxq[RTE_COLOR_YELLOW]) {
 		/* Reuse the first pre-allocated sub_policy. */
 		sub_policy = mtr_policy->sub_policys[domain][0];
 		sub_policy_idx = sub_policy->idx;
@@ -16994,8 +17320,9 @@ flow_dv_meter_sub_policy_rss_prepare(struct rte_eth_dev *dev,
 			DRV_LOG(ERR, "Exceed max meter number in hierarchy.");
 			return NULL;
 		}
-		next_fm = mlx5_flow_meter_find(priv,
-			mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
+		rte_spinlock_lock(&mtr_policy->sl);
+		next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, mtr_policy, NULL);
+		rte_spinlock_unlock(&mtr_policy->sl);
 		if (!next_fm) {
 			DRV_LOG(ERR, "Failed to get next meter in hierarchy.");
 			return NULL;
@@ -17052,6 +17379,68 @@ err_exit:
 	return NULL;
 }
 
+/**
+ * Check if need to create hierarchy tag rule.
+ *
+ * @param[in] priv
+ *   Pointer to mlx5_priv.
+ * @param[in] mtr_policy
+ *   Pointer to current meter policy.
+ * @param[in] src_port
+ *   The src port this extra rule should use.
+ * @param[out] next_fm
+ *   Pointer to next meter in hierarchy.
+ * @param[out] skip
+ *   Indicate if skip the tag rule creation.
+ * @param[out] error
+ *   Perform verbose error reporting if not NULL.
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_meter_hierarchy_skip_tag_rule(struct mlx5_priv *priv,
+				   struct mlx5_flow_meter_policy *mtr_policy,
+				   int32_t src_port,
+				   struct mlx5_flow_meter_info **next_fm,
+				   bool *skip,
+				   struct rte_flow_error *error)
+{
+	struct mlx5_flow_meter_sub_policy *sub_policy;
+	struct mlx5_sub_policy_color_rule *color_rule;
+	uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
+	int ret = 0;
+	int i;
+
+	*next_fm = NULL;
+	*skip = false;
+	rte_spinlock_lock(&mtr_policy->sl);
+	if (!mtr_policy->is_hierarchy)
+		goto exit;
+	*next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, mtr_policy, NULL);
+	if (!*next_fm) {
+		ret = rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+					 NULL, "Failed to find next meter in hierarchy.");
+		goto exit;
+	}
+	if (!(*next_fm)->drop_cnt) {
+		*skip = true;
+		goto exit;
+	}
+	sub_policy = mtr_policy->sub_policys[domain][0];
+	for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
+		if (mtr_policy->act_cnt[i].fate_action != MLX5_FLOW_FATE_MTR)
+			continue;
+		TAILQ_FOREACH(color_rule, &sub_policy->color_rules[i], next_port)
+			if (color_rule->src_port == src_port) {
+				*skip = true;
+				goto exit;
+			}
+	}
+exit:
+	rte_spinlock_unlock(&mtr_policy->sl);
+	return ret;
+}
+
 /**
  * Create the sub policy tag rule for all meters in hierarchy.
  *
@@ -17095,111 +17484,129 @@ flow_dv_meter_hierarchy_rule_create(struct rte_eth_dev *dev,
 		.reserved = 0,
 	};
 	uint32_t domain = MLX5_MTR_DOMAIN_TRANSFER;
-	int i;
+	struct {
+		struct mlx5_flow_meter_policy *fm_policy;
+		struct mlx5_flow_meter_info *next_fm;
+		struct mlx5_sub_policy_color_rule *tag_rule[MLX5_MTR_RTE_COLORS];
+	} fm_info[MLX5_MTR_CHAIN_MAX_NUM] = { {0} };
+	uint32_t fm_cnt = 0;
+	uint32_t i, j;
 
-	mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
-	MLX5_ASSERT(mtr_policy);
-	if (!mtr_policy->is_hierarchy)
-		return 0;
-	next_fm = mlx5_flow_meter_find(priv,
-			mtr_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
-	if (!next_fm) {
-		return rte_flow_error_set(error, EINVAL,
-				RTE_FLOW_ERROR_TYPE_ACTION, NULL,
-				"Failed to find next meter in hierarchy.");
-	}
-	if (!next_fm->drop_cnt)
-		goto exit;
 	color_reg_c_idx = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, error);
-	sub_policy = mtr_policy->sub_policys[domain][0];
-	for (i = 0; i < RTE_COLORS; i++) {
-		bool rule_exist = false;
-		struct mlx5_meter_policy_action_container *act_cnt;
+	/* Get all fms who need to create the tag color rule. */
+	do {
+		bool skip = false;
 
-		if (i >= RTE_COLOR_YELLOW)
-			break;
-		TAILQ_FOREACH(color_rule,
-			      &sub_policy->color_rules[i], next_port)
-			if (color_rule->src_port == src_port) {
-				rule_exist = true;
-				break;
-			}
-		if (rule_exist)
-			continue;
-		color_rule = mlx5_malloc(MLX5_MEM_ZERO,
-				sizeof(struct mlx5_sub_policy_color_rule),
-				0, SOCKET_ID_ANY);
-		if (!color_rule)
-			return rte_flow_error_set(error, ENOMEM,
-				RTE_FLOW_ERROR_TYPE_ACTION,
-				NULL, "No memory to create tag color rule.");
-		color_rule->src_port = src_port;
-		attr.priority = i;
-		next_policy = mlx5_flow_meter_policy_find(dev,
-						next_fm->policy_id, NULL);
-		MLX5_ASSERT(next_policy);
-		next_sub_policy = next_policy->sub_policys[domain][0];
-		tbl_data = container_of(next_sub_policy->tbl_rsc,
-					struct mlx5_flow_tbl_data_entry, tbl);
-		act_cnt = &mtr_policy->act_cnt[i];
-		if (mtr_first) {
-			acts.dv_actions[0] = next_fm->meter_action;
-			acts.dv_actions[1] = act_cnt->modify_hdr->action;
-		} else {
-			acts.dv_actions[0] = act_cnt->modify_hdr->action;
-			acts.dv_actions[1] = next_fm->meter_action;
-		}
-		acts.dv_actions[2] = tbl_data->jump.action;
-		acts.actions_n = 3;
-		if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) {
-			next_fm = NULL;
-			goto err_exit;
-		}
-		if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
-				MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
-				&attr, true, item,
-				&color_rule->matcher, error)) {
-			rte_flow_error_set(error, errno,
-				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				"Failed to create hierarchy meter matcher.");
+		mtr_policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
+		MLX5_ASSERT(mtr_policy);
+		if (mlx5_meter_hierarchy_skip_tag_rule(priv, mtr_policy, src_port,
+						       &next_fm, &skip, error))
 			goto err_exit;
+		if (next_fm && !skip) {
+			fm_info[fm_cnt].fm_policy = mtr_policy;
+			fm_info[fm_cnt].next_fm = next_fm;
+			if (++fm_cnt >= MLX5_MTR_CHAIN_MAX_NUM) {
+				rte_flow_error_set(error, errno,
+					RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+					"Exceed max meter number in hierarchy.");
+				goto err_exit;
+			}
 		}
-		if (__flow_dv_create_policy_flow(dev, color_reg_c_idx,
-					(enum rte_color)i,
-					color_rule->matcher->matcher_object,
-					acts.actions_n, acts.dv_actions,
-					true, item,
-					&color_rule->rule, &attr)) {
-			rte_flow_error_set(error, errno,
-				RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				"Failed to create hierarchy meter rule.");
-			goto err_exit;
+		fm = next_fm;
+	} while (fm);
+	/* Create tag color rules for all needed fms. */
+	for (i = 0; i < fm_cnt; i++) {
+		void *mtr_action;
+
+		mtr_policy = fm_info[i].fm_policy;
+		rte_spinlock_lock(&mtr_policy->sl);
+		sub_policy = mtr_policy->sub_policys[domain][0];
+		for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) {
+			if (mtr_policy->act_cnt[j].fate_action != MLX5_FLOW_FATE_MTR)
+				continue;
+			color_rule = mlx5_malloc(MLX5_MEM_ZERO,
+						 sizeof(struct mlx5_sub_policy_color_rule),
+						 0, SOCKET_ID_ANY);
+			if (!color_rule) {
+				rte_spinlock_unlock(&mtr_policy->sl);
+				rte_flow_error_set(error, ENOMEM,
+						   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+						   "No memory to create tag color rule.");
+				goto err_exit;
+			}
+			color_rule->src_port = src_port;
+			next_fm = fm_info[i].next_fm;
+			if (mlx5_flow_meter_attach(priv, next_fm, &attr, error)) {
+				mlx5_free(color_rule);
+				rte_spinlock_unlock(&mtr_policy->sl);
+				goto err_exit;
+			}
+			fm_info[i].tag_rule[j] = color_rule;
+			TAILQ_INSERT_TAIL(&sub_policy->color_rules[j], color_rule, next_port);
+			/* Prepare to create color rule. */
+			mtr_action = (next_fm->color_aware && j == RTE_COLOR_YELLOW) ?
+								next_fm->meter_action_y :
+								next_fm->meter_action_g;
+			next_policy = mlx5_flow_meter_policy_find(dev, next_fm->policy_id, NULL);
+			MLX5_ASSERT(next_policy);
+			next_sub_policy = next_policy->sub_policys[domain][0];
+			tbl_data = container_of(next_sub_policy->tbl_rsc,
+						struct mlx5_flow_tbl_data_entry, tbl);
+			if (mtr_first) {
+				acts.dv_actions[0] = mtr_action;
+				acts.dv_actions[1] = mtr_policy->act_cnt[j].modify_hdr->action;
+			} else {
+				acts.dv_actions[0] = mtr_policy->act_cnt[j].modify_hdr->action;
+				acts.dv_actions[1] = mtr_action;
+			}
+			acts.dv_actions[2] = tbl_data->jump.action;
+			acts.actions_n = 3;
+			if (__flow_dv_create_policy_matcher(dev, color_reg_c_idx,
+						MLX5_MTR_POLICY_MATCHER_PRIO, sub_policy,
+						&attr, true, item, &color_rule->matcher, error)) {
+				rte_spinlock_unlock(&mtr_policy->sl);
+				rte_flow_error_set(error, errno,
+						   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+						   "Failed to create hierarchy meter matcher.");
+				goto err_exit;
+			}
+			if (__flow_dv_create_policy_flow(dev, color_reg_c_idx, (enum rte_color)j,
+						color_rule->matcher->matcher_object,
+						acts.actions_n, acts.dv_actions,
+						true, item, &color_rule->rule, &attr)) {
+				rte_spinlock_unlock(&mtr_policy->sl);
+				rte_flow_error_set(error, errno,
+						   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+						   "Failed to create hierarchy meter rule.");
+				goto err_exit;
+			}
 		}
-		TAILQ_INSERT_TAIL(&sub_policy->color_rules[i],
-				  color_rule, next_port);
+		rte_spinlock_unlock(&mtr_policy->sl);
 	}
-exit:
-	/**
-	 * Recursive call to iterate all meters in hierarchy and
-	 * create needed rules.
-	 */
-	return flow_dv_meter_hierarchy_rule_create(dev, next_fm,
-						src_port, item, error);
+	return 0;
 err_exit:
-	if (color_rule) {
-		if (color_rule->rule)
-			mlx5_flow_os_destroy_flow(color_rule->rule);
-		if (color_rule->matcher) {
-			struct mlx5_flow_tbl_data_entry *tbl =
-				container_of(color_rule->matcher->tbl,
-						typeof(*tbl), tbl);
-			mlx5_list_unregister(tbl->matchers,
-						&color_rule->matcher->entry);
+	for (i = 0; i < fm_cnt; i++) {
+		mtr_policy = fm_info[i].fm_policy;
+		rte_spinlock_lock(&mtr_policy->sl);
+		sub_policy = mtr_policy->sub_policys[domain][0];
+		for (j = 0; j < MLX5_MTR_RTE_COLORS; j++) {
+			color_rule = fm_info[i].tag_rule[j];
+			if (!color_rule)
+				continue;
+			if (color_rule->rule)
+				mlx5_flow_os_destroy_flow(color_rule->rule);
+			if (color_rule->matcher) {
+				struct mlx5_flow_tbl_data_entry *tbl =
+					container_of(color_rule->matcher->tbl, typeof(*tbl), tbl);
+				mlx5_list_unregister(tbl->matchers, &color_rule->matcher->entry);
+			}
+			if (fm_info[i].next_fm)
+				mlx5_flow_meter_detach(priv, fm_info[i].next_fm);
+			TAILQ_REMOVE(&sub_policy->color_rules[j], color_rule, next_port);
+			mlx5_free(color_rule);
 		}
-		mlx5_free(color_rule);
+		rte_spinlock_unlock(&mtr_policy->sl);
 	}
-	if (next_fm)
-		mlx5_flow_meter_detach(priv, next_fm);
 	return -rte_errno;
 }
 
@@ -17592,7 +17999,7 @@ flow_dv_action_validate(struct rte_eth_dev *dev,
 						"Indirect age action not supported");
 		return flow_dv_validate_action_age(0, action, dev, err);
 	case RTE_FLOW_ACTION_TYPE_COUNT:
-		return flow_dv_validate_action_count(dev, true, 0, err);
+		return flow_dv_validate_action_count(dev, true, 0, NULL, err);
 	case RTE_FLOW_ACTION_TYPE_CONNTRACK:
 		if (!priv->sh->ct_aso_en)
 			return rte_flow_error_set(err, ENOTSUP,
@@ -17682,8 +18089,8 @@ flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
 					NULL,
 					"Multiple fate actions not supported.");
 	*hierarchy_domain = 0;
+	fm = mlx5_flow_meter_find(priv, meter_id, NULL);
 	while (true) {
-		fm = mlx5_flow_meter_find(priv, meter_id, NULL);
 		if (!fm)
 			return -rte_mtr_error_set(error, EINVAL,
 						RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
@@ -17692,6 +18099,10 @@ flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
 			return -rte_mtr_error_set(error, EINVAL,
 					RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 			"Non termination meter not supported in hierarchy.");
+		if (!fm->shared)
+			return -rte_mtr_error_set(error, EINVAL,
+					RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
+					"Only shared meter supported in hierarchy.");
 		policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
 		MLX5_ASSERT(policy);
 		/**
@@ -17713,7 +18124,9 @@ flow_dv_validate_policy_mtr_hierarchy(struct rte_eth_dev *dev,
 			*is_rss = policy->is_rss;
 			break;
 		}
-		meter_id = policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id;
+		rte_spinlock_lock(&policy->sl);
+		fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, NULL);
+		rte_spinlock_unlock(&policy->sl);
 		if (++cnt >= MLX5_MTR_CHAIN_MAX_NUM)
 			return -rte_mtr_error_set(error, EINVAL,
 					RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
@@ -17759,6 +18172,7 @@ flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
 	uint8_t def_domain = MLX5_MTR_ALL_DOMAIN_BIT;
 	uint8_t hierarchy_domain = 0;
 	const struct rte_flow_action_meter *mtr;
+	const struct rte_flow_action_meter *next_mtr = NULL;
 	bool def_green = false;
 	bool def_yellow = false;
 	const struct rte_flow_action_rss *rss_color[RTE_COLORS] = {NULL};
@@ -17942,25 +18356,12 @@ flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
 				++actions_n;
 				action_flags[i] |= MLX5_FLOW_ACTION_JUMP;
 				break;
-			/*
-			 * Only the last meter in the hierarchy will support
-			 * the YELLOW color steering. Then in the meter policy
-			 * actions list, there should be no other meter inside.
-			 */
 			case RTE_FLOW_ACTION_TYPE_METER:
-				if (i != RTE_COLOR_GREEN)
-					return -rte_mtr_error_set(error,
-						ENOTSUP,
-						RTE_MTR_ERROR_TYPE_METER_POLICY,
-						NULL,
-						"Meter hierarchy only supports GREEN color.");
-				if (*policy_mode != MLX5_MTR_POLICY_MODE_OG)
-					return -rte_mtr_error_set(error,
-						ENOTSUP,
-						RTE_MTR_ERROR_TYPE_METER_POLICY,
-						NULL,
-						"No yellow policy should be provided in meter hierarchy.");
 				mtr = act->conf;
+				if (next_mtr && next_mtr->mtr_id != mtr->mtr_id)
+					return -rte_mtr_error_set(error, ENOTSUP,
+						RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
+						"Green and Yellow must use the same meter.");
 				ret = flow_dv_validate_policy_mtr_hierarchy(dev,
 							mtr->mtr_id,
 							action_flags[i],
@@ -17972,6 +18373,7 @@ flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
 				++actions_n;
 				action_flags[i] |=
 				MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY;
+				next_mtr = mtr;
 				break;
 			default:
 				return -rte_mtr_error_set(error, ENOTSUP,
@@ -18057,6 +18459,13 @@ flow_dv_validate_mtr_policy_acts(struct rte_eth_dev *dev,
 			}
 		}
 	}
+	if (next_mtr && *policy_mode == MLX5_MTR_POLICY_MODE_ALL) {
+		if (!(action_flags[RTE_COLOR_GREEN] & action_flags[RTE_COLOR_YELLOW] &
+		      MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY))
+			return -rte_mtr_error_set(error, EINVAL, RTE_MTR_ERROR_TYPE_METER_POLICY,
+						  NULL,
+						  "Meter hierarchy supports meter action only.");
+	}
 	/* If both colors have RSS, the attributes should be the same. */
 	if (flow_dv_mtr_policy_rss_compare(rss_color[RTE_COLOR_GREEN],
 					   rss_color[RTE_COLOR_YELLOW]))
@@ -18248,4 +18657,3 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
 };
 
 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */
-