net/mlx5: engage free on completion queue
[dpdk.git] / drivers / net / mlx5 / mlx5_flow_dv.c
index 5bd1b1c..c02517a 100644 (file)
@@ -157,6 +157,7 @@ struct field_modify_info modify_vlan_out_first_vid[] = {
 };
 
 struct field_modify_info modify_ipv4[] = {
+       {1,  1, MLX5_MODI_OUT_IP_DSCP},
        {1,  8, MLX5_MODI_OUT_IPV4_TTL},
        {4, 12, MLX5_MODI_OUT_SIPV4},
        {4, 16, MLX5_MODI_OUT_DIPV4},
@@ -164,6 +165,7 @@ struct field_modify_info modify_ipv4[] = {
 };
 
 struct field_modify_info modify_ipv6[] = {
+       {1,  0, MLX5_MODI_OUT_IP_DSCP},
        {1,  7, MLX5_MODI_OUT_IPV6_HOPLIMIT},
        {4,  8, MLX5_MODI_OUT_SIPV6_127_96},
        {4, 12, MLX5_MODI_OUT_SIPV6_95_64},
@@ -1088,6 +1090,14 @@ flow_dv_convert_action_mark(struct rte_eth_dev *dev,
        if (reg < 0)
                return reg;
        assert(reg > 0);
+       if (reg == REG_C_0) {
+               uint32_t msk_c0 = priv->sh->dv_regc0_mask;
+               uint32_t shl_c0 = rte_bsf32(msk_c0);
+
+               data = rte_cpu_to_be_32(rte_cpu_to_be_32(data) << shl_c0);
+               mask = rte_cpu_to_be_32(mask) & msk_c0;
+               mask = rte_cpu_to_be_32(mask << shl_c0);
+       }
        reg_c_x[0].id = reg_to_field[reg];
        return flow_dv_convert_modify_action(&item, reg_c_x, NULL, resource,
                                             MLX5_MODIFICATION_TYPE_SET, error);
@@ -1190,6 +1200,82 @@ flow_dv_convert_action_set_meta
                                             MLX5_MODIFICATION_TYPE_SET, error);
 }
 
+/**
+ * Convert modify-header set IPv4 DSCP action to DV specification.
+ *
+ * @param[in,out] resource
+ *   Pointer to the modify-header resource.
+ * @param[in] action
+ *   Pointer to action specification.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_action_modify_ipv4_dscp
+                       (struct mlx5_flow_dv_modify_hdr_resource *resource,
+                        const struct rte_flow_action *action,
+                        struct rte_flow_error *error)
+{
+       const struct rte_flow_action_set_dscp *conf =
+               (const struct rte_flow_action_set_dscp *)(action->conf);
+       struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV4 };
+       struct rte_flow_item_ipv4 ipv4;
+       struct rte_flow_item_ipv4 ipv4_mask;
+
+       memset(&ipv4, 0, sizeof(ipv4));
+       memset(&ipv4_mask, 0, sizeof(ipv4_mask));
+       ipv4.hdr.type_of_service = conf->dscp;
+       ipv4_mask.hdr.type_of_service = RTE_IPV4_HDR_DSCP_MASK >> 2;
+       item.spec = &ipv4;
+       item.mask = &ipv4_mask;
+       return flow_dv_convert_modify_action(&item, modify_ipv4, NULL, resource,
+                                            MLX5_MODIFICATION_TYPE_SET, error);
+}
+
+/**
+ * Convert modify-header set IPv6 DSCP action to DV specification.
+ *
+ * @param[in,out] resource
+ *   Pointer to the modify-header resource.
+ * @param[in] action
+ *   Pointer to action specification.
+ * @param[out] error
+ *   Pointer to the error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_convert_action_modify_ipv6_dscp
+                       (struct mlx5_flow_dv_modify_hdr_resource *resource,
+                        const struct rte_flow_action *action,
+                        struct rte_flow_error *error)
+{
+       const struct rte_flow_action_set_dscp *conf =
+               (const struct rte_flow_action_set_dscp *)(action->conf);
+       struct rte_flow_item item = { .type = RTE_FLOW_ITEM_TYPE_IPV6 };
+       struct rte_flow_item_ipv6 ipv6;
+       struct rte_flow_item_ipv6 ipv6_mask;
+
+       memset(&ipv6, 0, sizeof(ipv6));
+       memset(&ipv6_mask, 0, sizeof(ipv6_mask));
+       /*
+        * Even though the DSCP bits offset of IPv6 is not byte aligned,
+        * rdma-core only accept the DSCP bits byte aligned start from
+        * bit 0 to 5 as to be compatible with IPv4. No need to shift the
+        * bits in IPv6 case as rdma-core requires byte aligned value.
+        */
+       ipv6.hdr.vtc_flow = conf->dscp;
+       ipv6_mask.hdr.vtc_flow = RTE_IPV6_HDR_DSCP_MASK >> 22;
+       item.spec = &ipv6;
+       item.mask = &ipv6_mask;
+       return flow_dv_convert_modify_action(&item, modify_ipv6, NULL, resource,
+                                            MLX5_MODIFICATION_TYPE_SET, error);
+}
+
 /**
  * Validate MARK item.
  *
@@ -3438,6 +3524,74 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
        return 0;
 }
 
+/**
+ * Validate the modify-header IPv4 DSCP actions.
+ *
+ * @param[in] action_flags
+ *   Holds the actions detected until now.
+ * @param[in] action
+ *   Pointer to the modify action.
+ * @param[in] item_flags
+ *   Holds the items detected.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_validate_action_modify_ipv4_dscp(const uint64_t action_flags,
+                                        const struct rte_flow_action *action,
+                                        const uint64_t item_flags,
+                                        struct rte_flow_error *error)
+{
+       int ret = 0;
+
+       ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
+       if (!ret) {
+               if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV4))
+                       return rte_flow_error_set(error, EINVAL,
+                                                 RTE_FLOW_ERROR_TYPE_ACTION,
+                                                 NULL,
+                                                 "no ipv4 item in pattern");
+       }
+       return ret;
+}
+
+/**
+ * Validate the modify-header IPv6 DSCP actions.
+ *
+ * @param[in] action_flags
+ *   Holds the actions detected until now.
+ * @param[in] action
+ *   Pointer to the modify action.
+ * @param[in] item_flags
+ *   Holds the items detected.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
+                                        const struct rte_flow_action *action,
+                                        const uint64_t item_flags,
+                                        struct rte_flow_error *error)
+{
+       int ret = 0;
+
+       ret = flow_dv_validate_action_modify_hdr(action_flags, action, error);
+       if (!ret) {
+               if (!(item_flags & MLX5_FLOW_LAYER_L3_IPV6))
+                       return rte_flow_error_set(error, EINVAL,
+                                                 RTE_FLOW_ERROR_TYPE_ACTION,
+                                                 NULL,
+                                                 "no ipv6 item in pattern");
+       }
+       return ret;
+}
+
 /**
  * Find existing modify-header resource or create and register a new one.
  *
@@ -4812,6 +4966,32 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
                        action_flags |= MLX5_FLOW_ACTION_METER;
                        ++actions_n;
                        break;
+               case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
+                       ret = flow_dv_validate_action_modify_ipv4_dscp
+                                                        (action_flags,
+                                                         actions,
+                                                         item_flags,
+                                                         error);
+                       if (ret < 0)
+                               return ret;
+                       /* Count all modify-header actions as one action. */
+                       if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
+                               ++actions_n;
+                       action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
+                       break;
+               case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
+                       ret = flow_dv_validate_action_modify_ipv6_dscp
+                                                               (action_flags,
+                                                                actions,
+                                                                item_flags,
+                                                                error);
+                       if (ret < 0)
+                               return ret;
+                       /* Count all modify-header actions as one action. */
+                       if (!(action_flags & MLX5_FLOW_MODIFY_HDR_ACTIONS))
+                               ++actions_n;
+                       action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
+                       break;
                default:
                        return rte_flow_error_set(error, ENOTSUP,
                                                  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -5748,6 +5928,7 @@ flow_dv_match_meta_reg(void *matcher, void *key,
                MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_2);
        void *misc2_v =
                MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
+       uint32_t temp;
 
        data &= mask;
        switch (reg_type) {
@@ -5760,8 +5941,18 @@ flow_dv_match_meta_reg(void *matcher, void *key,
                MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_b, data);
                break;
        case REG_C_0:
-               MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, mask);
-               MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, data);
+               /*
+                * The metadata register C0 field might be divided into
+                * source vport index and META item value, we should set
+                * this field according to specified mask, not as whole one.
+                */
+               temp = MLX5_GET(fte_match_set_misc2, misc2_m, metadata_reg_c_0);
+               temp |= mask;
+               MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_0, temp);
+               temp = MLX5_GET(fte_match_set_misc2, misc2_v, metadata_reg_c_0);
+               temp &= ~mask;
+               temp |= data;
+               MLX5_SET(fte_match_set_misc2, misc2_v, metadata_reg_c_0, temp);
                break;
        case REG_C_1:
                MLX5_SET(fte_match_set_misc2, misc2_m, metadata_reg_c_1, mask);
@@ -5831,6 +6022,15 @@ flow_dv_translate_item_mark(struct rte_eth_dev *dev,
                /* Get the metadata register index for the mark. */
                reg = mlx5_flow_get_reg_id(dev, MLX5_FLOW_MARK, 0, NULL);
                assert(reg > 0);
+               if (reg == REG_C_0) {
+                       struct mlx5_priv *priv = dev->data->dev_private;
+                       uint32_t msk_c0 = priv->sh->dv_regc0_mask;
+                       uint32_t shl_c0 = rte_bsf32(msk_c0);
+
+                       mask &= msk_c0;
+                       mask <<= shl_c0;
+                       value <<= shl_c0;
+               }
                flow_dv_match_meta_reg(matcher, key, reg, value, mask);
        }
 }
@@ -5881,8 +6081,12 @@ flow_dv_translate_item_meta(struct rte_eth_dev *dev,
                        struct mlx5_priv *priv = dev->data->dev_private;
                        uint32_t msk_c0 = priv->sh->dv_regc0_mask;
                        uint32_t shl_c0 = rte_bsf32(msk_c0);
+#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+                       uint32_t shr_c0 = __builtin_clz(priv->sh->dv_meta_mask);
 
-                       msk_c0 = rte_cpu_to_be_32(msk_c0);
+                       value >>= shr_c0;
+                       mask >>= shr_c0;
+#endif
                        value <<= shl_c0;
                        mask <<= shl_c0;
                        assert(msk_c0);
@@ -5912,6 +6116,8 @@ flow_dv_translate_item_meta_vport(void *matcher, void *key,
 /**
  * Add tag item to matcher
  *
+ * @param[in] dev
+ *   The devich to configure through.
  * @param[in, out] matcher
  *   Flow matcher.
  * @param[in, out] key
@@ -5920,15 +6126,27 @@ flow_dv_translate_item_meta_vport(void *matcher, void *key,
  *   Flow pattern to translate.
  */
 static void
-flow_dv_translate_mlx5_item_tag(void *matcher, void *key,
+flow_dv_translate_mlx5_item_tag(struct rte_eth_dev *dev,
+                               void *matcher, void *key,
                                const struct rte_flow_item *item)
 {
        const struct mlx5_rte_flow_item_tag *tag_v = item->spec;
        const struct mlx5_rte_flow_item_tag *tag_m = item->mask;
+       uint32_t mask, value;
 
        assert(tag_v);
-       flow_dv_match_meta_reg(matcher, key, tag_v->id, tag_v->data,
-                              tag_m ? tag_m->data : UINT32_MAX);
+       value = tag_v->data;
+       mask = tag_m ? tag_m->data : UINT32_MAX;
+       if (tag_v->id == REG_C_0) {
+               struct mlx5_priv *priv = dev->data->dev_private;
+               uint32_t msk_c0 = priv->sh->dv_regc0_mask;
+               uint32_t shl_c0 = rte_bsf32(msk_c0);
+
+               mask &= msk_c0;
+               mask <<= shl_c0;
+               value <<= shl_c0;
+       }
+       flow_dv_match_meta_reg(matcher, key, tag_v->id, value, mask);
 }
 
 /**
@@ -7107,6 +7325,18 @@ cnt_err:
                                flow->meter->mfts->meter_action;
                        action_flags |= MLX5_FLOW_ACTION_METER;
                        break;
+               case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
+                       if (flow_dv_convert_action_modify_ipv4_dscp(&mhdr_res,
+                                                             actions, error))
+                               return -rte_errno;
+                       action_flags |= MLX5_FLOW_ACTION_SET_IPV4_DSCP;
+                       break;
+               case RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP:
+                       if (flow_dv_convert_action_modify_ipv6_dscp(&mhdr_res,
+                                                             actions, error))
+                               return -rte_errno;
+                       action_flags |= MLX5_FLOW_ACTION_SET_IPV6_DSCP;
+                       break;
                case RTE_FLOW_ACTION_TYPE_END:
                        actions_end = true;
                        if (mhdr_res.actions_num) {
@@ -7275,7 +7505,7 @@ cnt_err:
                        last_item = MLX5_FLOW_ITEM_TAG;
                        break;
                case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
-                       flow_dv_translate_mlx5_item_tag(match_mask,
+                       flow_dv_translate_mlx5_item_tag(dev, match_mask,
                                                        match_value, items);
                        last_item = MLX5_FLOW_ITEM_TAG;
                        break;