drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133         uint32_t ls_factor;
 134         int ret;
 135         uint64_t cir, cbs;
 136         uint64_t eir, ebs;
 137         uint64_t pir, pbs;
 138
 139         /* Profile must not be NULL. */
 140         if (profile == NULL)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 143                                           NULL, "Meter profile is null.");
 144         /* Meter profile ID must be valid. */
 145         if (meter_profile_id == UINT32_MAX)
 146                 return -rte_mtr_error_set(error, EINVAL,
 147                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 148                                           NULL, "Meter profile id not valid.");
 149         /* Meter profile must not exist. */
 150         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 151         if (fmp)
 152                 return -rte_mtr_error_set(error, EEXIST,
 153                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 154                                           NULL,
 155                                           "Meter profile already exists.");
 156         if (!priv->sh->meter_aso_en) {
 157                 /* Old version is even not supported. */
 158                 if (!priv->config.hca_attr.qos.flow_meter_old)
 159                         return -rte_mtr_error_set(error, ENOTSUP,
 160                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 161                                 NULL, "Metering is not supported.");
 162                 /* Old FW metering only supports srTCM. */
 163                 if (profile->alg != RTE_MTR_SRTCM_RFC2697) {
 164                         return -rte_mtr_error_set(error, ENOTSUP,
 165                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 166                                 NULL, "Metering algorithm is not supported.");
 167                 } else if (profile->srtcm_rfc2697.ebs) {
 168                         /* EBS is not supported for old metering. */
 169                         return -rte_mtr_error_set(error, ENOTSUP,
 170                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 171                                 NULL, "EBS is not supported.");
 172                 }
 173                 if (profile->packet_mode)
 174                         return -rte_mtr_error_set(error, ENOTSUP,
 175                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 176                                 "Metering algorithm packet mode is not supported.");
 177         }
 178         ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0;
 179         switch (profile->alg) {
 180         case RTE_MTR_SRTCM_RFC2697:
 181                 cir = profile->srtcm_rfc2697.cir << ls_factor;
 182                 cbs = profile->srtcm_rfc2697.cbs << ls_factor;
 183                 ebs = profile->srtcm_rfc2697.ebs << ls_factor;
 184                 /* EBS could be zero for old metering. */
 185                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 186                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 187                     ebs <= MLX5_SRTCM_XBS_MAX) {
 188                         ret = 0;
 189                 } else {
 190                         ret = -rte_mtr_error_set(error, ENOTSUP,
 191                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 192                                         "Profile values out of range.");
 193                 }
 194                 break;
 195         case RTE_MTR_TRTCM_RFC2698:
 196                 cir = profile->trtcm_rfc2698.cir << ls_factor;
 197                 cbs = profile->trtcm_rfc2698.cbs << ls_factor;
 198                 pir = profile->trtcm_rfc2698.pir << ls_factor;
 199                 pbs = profile->trtcm_rfc2698.pbs << ls_factor;
 200                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 201                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 202                     pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) &&
 203                     pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) {
 204                         ret = 0;
 205                 } else {
 206                         ret = -rte_mtr_error_set(error, ENOTSUP,
 207                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 208                                         "Profile values out of range.");
 209                 }
 210                 break;
 211         case RTE_MTR_TRTCM_RFC4115:
 212                 cir = profile->trtcm_rfc4115.cir << ls_factor;
 213                 cbs = profile->trtcm_rfc4115.cbs << ls_factor;
 214                 eir = profile->trtcm_rfc4115.eir << ls_factor;
 215                 ebs = profile->trtcm_rfc4115.ebs << ls_factor;
 216                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 217                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 218                     eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) {
 219                         ret = 0;
 220                 } else {
 221                         ret = -rte_mtr_error_set(error, ENOTSUP,
 222                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 223                                         "Profile values out of range.");
 224                 }
 225                 break;
 226         default:
 227                 ret = -rte_mtr_error_set(error, ENOTSUP,
 228                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 229                                          "Unknown metering algorithm.");
 230                 break;
 231         }
 232         return ret;
 233 }
 234
 235 /*
 236  * Calculate mantissa and exponent for cir / eir.
 237  *
 238  * @param[in] xir
 239  *   Value to be calculated.
 240  * @param[out] man
 241  *   Pointer to the mantissa.
 242  * @param[out] exp
 243  *   Pointer to the exp.
 244  */
 245 static inline void
 246 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp)
 247 {
 248         int64_t _xir;
 249         int64_t delta = INT64_MAX;
 250         uint8_t _man = 0;
 251         uint8_t _exp = 0;
 252         uint64_t m, e;
 253
 254         /* Special case xir == 0 ? both exp and mantissa are 0. */
 255         if (xir == 0) {
 256                 *man = 0;
 257                 *exp = 0;
 258                 return;
 259         }
 260         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 261                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 262                         _xir = (1000000000ULL * m) >> e;
 263                         if (llabs(xir - _xir) <= delta) {
 264                                 delta = llabs(xir - _xir);
 265                                 _man = m;
 266                                 _exp = e;
 267                         }
 268                 }
 269         }
 270         *man = _man;
 271         *exp = _exp;
 272 }
 273
 274 /*
 275  * Calculate mantissa and exponent for xbs.
 276  *
 277  * @param[in] xbs
 278  *   Value to be calculated.
 279  * @param[out] man
 280  *   Pointer to the mantissa.
 281  * @param[out] exp
 282  *   Pointer to the exp.
 283  */
 284 static inline void
 285 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 286 {
 287         int _exp;
 288         double _man;
 289
 290         /* Special case xbs == 0 ? both exp and mantissa are 0. */
 291         if (xbs == 0) {
 292                 *man = 0;
 293                 *exp = 0;
 294                 return;
 295         }
 296         /* xbs = xbs_mantissa * 2^xbs_exponent */
 297         _man = frexp(xbs, &_exp);
 298         _man = _man * pow(2, MLX5_MAN_WIDTH);
 299         _exp = _exp - MLX5_MAN_WIDTH;
 300         *man = (uint8_t)ceil(_man);
 301         *exp = _exp;
 302 }
 303
 304 /**
 305  * Fill the prm meter parameter.
 306  *
 307  * @param[in,out] fmp
 308  *   Pointer to meter profile to be converted.
 309  * @param[out] error
 310  *   Pointer to the error structure.
 311  *
 312  * @return
 313  *   0 on success, a negative errno value otherwise and rte_errno is set.
 314  */
 315 static int
 316 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 317                            struct rte_mtr_error *error)
 318 {
 319         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 320         uint8_t man, exp;
 321         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 322         uint32_t eir_exp, eir_man, ebs_exp, ebs_man;
 323         uint64_t cir, cbs, eir, ebs;
 324
 325         switch (fmp->profile.alg) {
 326         case RTE_MTR_SRTCM_RFC2697:
 327                 cir = fmp->profile.srtcm_rfc2697.cir;
 328                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 329                 eir = 0;
 330                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 331                 break;
 332         case RTE_MTR_TRTCM_RFC2698:
 333                 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir >
 334                             fmp->profile.trtcm_rfc2698.cir &&
 335                             fmp->profile.trtcm_rfc2698.pbs >
 336                             fmp->profile.trtcm_rfc2698.cbs);
 337                 cir = fmp->profile.trtcm_rfc2698.cir;
 338                 cbs = fmp->profile.trtcm_rfc2698.cbs;
 339                 /* EIR / EBS are filled with PIR / PBS. */
 340                 eir = fmp->profile.trtcm_rfc2698.pir;
 341                 ebs = fmp->profile.trtcm_rfc2698.pbs;
 342                 break;
 343         case RTE_MTR_TRTCM_RFC4115:
 344                 cir = fmp->profile.trtcm_rfc4115.cir;
 345                 cbs = fmp->profile.trtcm_rfc4115.cbs;
 346                 eir = fmp->profile.trtcm_rfc4115.eir;
 347                 ebs = fmp->profile.trtcm_rfc4115.ebs;
 348                 break;
 349         default:
 350                 return -rte_mtr_error_set(error, EINVAL,
 351                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 352                                 "Metering algorithm mode is invalid");
 353         }
 354         /* Adjust the values for PPS mode. */
 355         if (fmp->profile.packet_mode) {
 356                 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 357                 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 358                 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 359                 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 360         }
 361         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 362         mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp);
 363         /* Check if cir mantissa is too large. */
 364         if (exp > ASO_DSEG_XIR_EXP_MASK)
 365                 return -rte_mtr_error_set(error, ENOTSUP,
 366                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 367                                           "meter profile parameter cir is not supported.");
 368         cir_man = man;
 369         cir_exp = exp;
 370          /* cbs = cbs_mantissa * 2^cbs_exponent */
 371         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 372         /* Check if cbs mantissa is too large. */
 373         if (exp > ASO_DSEG_EXP_MASK)
 374                 return -rte_mtr_error_set(error, ENOTSUP,
 375                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 376                                           "meter profile parameter cbs is not supported.");
 377         cbs_man = man;
 378         cbs_exp = exp;
 379         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 380                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 381                                           cir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 382                                           cir_man);
 383         mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp);
 384         /* Check if eir mantissa is too large. */
 385         if (exp > ASO_DSEG_XIR_EXP_MASK)
 386                 return -rte_mtr_error_set(error, ENOTSUP,
 387                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 388                                           "meter profile parameter eir is not supported.");
 389         eir_man = man;
 390         eir_exp = exp;
 391         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 392         /* Check if ebs mantissa is too large. */
 393         if (exp > ASO_DSEG_EXP_MASK)
 394                 return -rte_mtr_error_set(error, ENOTSUP,
 395                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 396                                           "meter profile parameter ebs is not supported.");
 397         ebs_man = man;
 398         ebs_exp = exp;
 399         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 400                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET |
 401                                           eir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 402                                           eir_man);
 403         if (srtcm->cbs_cir)
 404                 fmp->g_support = 1;
 405         if (srtcm->ebs_eir)
 406                 fmp->y_support = 1;
 407         return 0;
 408 }
 409
 410 /**
 411  * Callback to get MTR capabilities.
 412  *
 413  * @param[in] dev
 414  *   Pointer to Ethernet device.
 415  * @param[out] cap
 416  *   Pointer to save MTR capabilities.
 417  * @param[out] error
 418  *   Pointer to the error structure.
 419  *
 420  * @return
 421  *   0 on success, a negative errno value otherwise and rte_errno is set.
 422  */
 423 static int
 424 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 425                  struct rte_mtr_capabilities *cap,
 426                  struct rte_mtr_error *error __rte_unused)
 427 {
 428         struct mlx5_priv *priv = dev->data->dev_private;
 429         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 430
 431         if (!priv->mtr_en)
 432                 return -rte_mtr_error_set(error, ENOTSUP,
 433                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 434                                           "Meter is not supported");
 435         memset(cap, 0, sizeof(*cap));
 436         if (priv->sh->meter_aso_en) {
 437                 /* 2 meters per one ASO cache line. */
 438                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 439                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 440         } else {
 441                 cap->n_max = 1 << qattr->log_max_flow_meter;
 442                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 443         }
 444         cap->srtcm_rfc2697_byte_mode_supported = 1;
 445         cap->n_shared_max = cap->n_max;
 446         cap->identical = 1;
 447         cap->shared_identical = 1;
 448         cap->shared_n_flows_per_mtr_max = 4 << 20;
 449         /* 2M flows can share the same meter. */
 450         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 451         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 452         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 453         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 454                           RTE_MTR_STATS_N_PKTS_DROPPED;
 455         return 0;
 456 }
 457
 458 /**
 459  * Callback to add MTR profile.
 460  *
 461  * @param[in] dev
 462  *   Pointer to Ethernet device.
 463  * @param[in] meter_profile_id
 464  *   Meter profile id.
 465  * @param[in] profile
 466  *   Pointer to meter profile detail.
 467  * @param[out] error
 468  *   Pointer to the error structure.
 469  *
 470  * @return
 471  *   0 on success, a negative errno value otherwise and rte_errno is set.
 472  */
 473 static int
 474 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 475                        uint32_t meter_profile_id,
 476                        struct rte_mtr_meter_profile *profile,
 477                        struct rte_mtr_error *error)
 478 {
 479         struct mlx5_priv *priv = dev->data->dev_private;
 480         struct mlx5_flow_meter_profile *fmp;
 481         union mlx5_l3t_data data;
 482         int ret;
 483
 484         if (!priv->mtr_en)
 485                 return -rte_mtr_error_set(error, ENOTSUP,
 486                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 487                                           "Meter is not supported");
 488         /* Check input params. */
 489         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 490                                                profile, error);
 491         if (ret)
 492                 return ret;
 493         /* Meter profile memory allocation. */
 494         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 495                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 496         if (fmp == NULL)
 497                 return -rte_mtr_error_set(error, ENOMEM,
 498                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 499                                           NULL, "Meter profile memory "
 500                                           "alloc failed.");
 501         /* Fill profile info. */
 502         fmp->id = meter_profile_id;
 503         fmp->profile = *profile;
 504         /* Fill the flow meter parameters for the PRM. */
 505         ret = mlx5_flow_meter_param_fill(fmp, error);
 506         if (ret)
 507                 goto error;
 508         data.ptr = fmp;
 509         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 510                                  meter_profile_id, &data);
 511         if (ret)
 512                 return -rte_mtr_error_set(error, ENOTSUP,
 513                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 514                                           NULL, "Meter profile insert fail.");
 515         return 0;
 516 error:
 517         mlx5_free(fmp);
 518         return ret;
 519 }
 520
 521 /**
 522  * Callback to delete MTR profile.
 523  *
 524  * @param[in] dev
 525  *   Pointer to Ethernet device.
 526  * @param[in] meter_profile_id
 527  *   Meter profile id.
 528  * @param[out] error
 529  *   Pointer to the error structure.
 530  *
 531  * @return
 532  *   0 on success, a negative errno value otherwise and rte_errno is set.
 533  */
 534 static int
 535 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 536                           uint32_t meter_profile_id,
 537                           struct rte_mtr_error *error)
 538 {
 539         struct mlx5_priv *priv = dev->data->dev_private;
 540         struct mlx5_flow_meter_profile *fmp;
 541
 542         if (!priv->mtr_en)
 543                 return -rte_mtr_error_set(error, ENOTSUP,
 544                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 545                                           "Meter is not supported");
 546         /* Meter profile must exist. */
 547         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 548         if (fmp == NULL)
 549                 return -rte_mtr_error_set(error, ENOENT,
 550                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 551                                           &meter_profile_id,
 552                                           "Meter profile id is invalid.");
 553         /* Check profile is unused. */
 554         if (fmp->ref_cnt)
 555                 return -rte_mtr_error_set(error, EBUSY,
 556                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 557                                           NULL, "Meter profile is in use.");
 558         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 559                 return -rte_mtr_error_set(error, EBUSY,
 560                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 561                                           NULL, "Meter profile remove fail.");
 562         mlx5_free(fmp);
 563         return 0;
 564 }
 565
 566 /**
 567  * Find policy by id.
 568  *
 569  * @param[in] dev
 570  *   Pointer to Ethernet device.
 571  * @param policy_id
 572  *   Policy id.
 573  *
 574  * @return
 575  *   Pointer to the policy found on success, NULL otherwise.
 576  */
 577 struct mlx5_flow_meter_policy *
 578 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 579                             uint32_t policy_id,
 580                             uint32_t *policy_idx)
 581 {
 582         struct mlx5_priv *priv = dev->data->dev_private;
 583         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 584         union mlx5_l3t_data data;
 585
 586         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 587                 return NULL;
 588         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 589                                 !data.dword)
 590                 return NULL;
 591         if (policy_idx)
 592                 *policy_idx = data.dword;
 593         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 594                                         data.dword);
 595         /* Remove reference taken by the mlx5_l3t_get_entry. */
 596         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 597         if (sub_policy)
 598                 if (sub_policy->main_policy_id)
 599                         return sub_policy->main_policy;
 600         return NULL;
 601 }
 602
 603 /**
 604  * Get the last meter's policy from one meter's policy in hierarchy.
 605  *
 606  * @param[in] dev
 607  *   Pointer to Ethernet device.
 608  * @param[in] policy
 609  *   Pointer to flow meter policy.
 610  *
 611  * @return
 612  *   Pointer to the final meter's policy, or NULL when fail.
 613  */
 614 struct mlx5_flow_meter_policy *
 615 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 616                                         struct mlx5_flow_meter_policy *policy)
 617 {
 618         struct mlx5_priv *priv = dev->data->dev_private;
 619         struct mlx5_flow_meter_info *next_fm;
 620         struct mlx5_flow_meter_policy *next_policy = policy;
 621
 622         while (next_policy->is_hierarchy) {
 623                 next_fm = mlx5_flow_meter_find(priv,
 624                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 625                 if (!next_fm || next_fm->def_policy)
 626                         return NULL;
 627                 next_policy = mlx5_flow_meter_policy_find(dev,
 628                                                 next_fm->policy_id, NULL);
 629                 MLX5_ASSERT(next_policy);
 630         }
 631         return next_policy;
 632 }
 633
 634 /**
 635  * Callback to check MTR policy action validate
 636  *
 637  * @param[in] dev
 638  *   Pointer to Ethernet device.
 639  * @param[in] actions
 640  *   Pointer to meter policy action detail.
 641  * @param[out] error
 642  *   Pointer to the error structure.
 643  *
 644  * @return
 645  *   0 on success, a negative errno value otherwise and rte_errno is set.
 646  */
 647 static int
 648 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 649         struct rte_mtr_meter_policy_params *policy,
 650         struct rte_mtr_error *error)
 651 {
 652         struct mlx5_priv *priv = dev->data->dev_private;
 653         struct rte_flow_attr attr = { .transfer =
 654                         priv->config.dv_esw_en ? 1 : 0};
 655         bool is_rss = false;
 656         uint8_t policy_mode;
 657         uint8_t domain_bitmap;
 658         int ret;
 659
 660         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 661                 return -rte_mtr_error_set(error, ENOTSUP,
 662                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 663                                 NULL, "meter policy unsupported.");
 664         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 665                         &is_rss, &domain_bitmap, &policy_mode, error);
 666         if (ret)
 667                 return ret;
 668         return 0;
 669 }
 670
 671 static int
 672 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 673                         uint32_t policy_id,
 674                         struct mlx5_flow_meter_policy *mtr_policy,
 675                         struct rte_mtr_error *error,
 676                         bool clear_l3t)
 677 {
 678         struct mlx5_priv *priv = dev->data->dev_private;
 679         struct mlx5_flow_meter_sub_policy *sub_policy;
 680         uint32_t i, j;
 681         uint16_t sub_policy_num;
 682
 683         rte_spinlock_lock(&mtr_policy->sl);
 684         if (mtr_policy->ref_cnt) {
 685                 rte_spinlock_unlock(&mtr_policy->sl);
 686                 return -rte_mtr_error_set(error, EBUSY,
 687                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 688                                  NULL,
 689                                 "Meter policy object is being used.");
 690         }
 691         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 692         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 693         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 694                 sub_policy_num = (mtr_policy->sub_policy_num >>
 695                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 696                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 697                 if (sub_policy_num) {
 698                         for (j = 0; j < sub_policy_num; j++) {
 699                                 sub_policy = mtr_policy->sub_policys[i][j];
 700                                 if (sub_policy)
 701                                         mlx5_ipool_free
 702                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 703                                         sub_policy->idx);
 704                         }
 705                 }
 706         }
 707         if (priv->policy_idx_tbl && clear_l3t) {
 708                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 709                         rte_spinlock_unlock(&mtr_policy->sl);
 710                         return -rte_mtr_error_set(error, ENOTSUP,
 711                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 712                                 "Fail to delete policy in index table.");
 713                 }
 714         }
 715         rte_spinlock_unlock(&mtr_policy->sl);
 716         return 0;
 717 }
 718
 719 /**
 720  * Callback to add MTR policy.
 721  *
 722  * @param[in] dev
 723  *   Pointer to Ethernet device.
 724  * @param[out] policy_id
 725  *   Pointer to policy id
 726  * @param[in] actions
 727  *   Pointer to meter policy action detail.
 728  * @param[out] error
 729  *   Pointer to the error structure.
 730  *
 731  * @return
 732  *   0 on success, a negative errno value otherwise and rte_errno is set.
 733  */
 734 static int
 735 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 736                         uint32_t policy_id,
 737                         struct rte_mtr_meter_policy_params *policy,
 738                         struct rte_mtr_error *error)
 739 {
 740         struct mlx5_priv *priv = dev->data->dev_private;
 741         struct rte_flow_attr attr = { .transfer =
 742                         priv->config.dv_esw_en ? 1 : 0};
 743         uint32_t sub_policy_idx = 0;
 744         uint32_t policy_idx = 0;
 745         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 746         struct mlx5_flow_meter_sub_policy *sub_policy;
 747         bool is_rss = false;
 748         uint8_t policy_mode;
 749         uint32_t i;
 750         int ret;
 751         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 752         uint16_t sub_policy_num;
 753         uint8_t domain_bitmap = 0;
 754         union mlx5_l3t_data data;
 755         bool skip_rule = false;
 756
 757         if (!priv->mtr_en)
 758                 return -rte_mtr_error_set(error, ENOTSUP,
 759                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 760                                           NULL, "meter policy unsupported. ");
 761         if (policy_id == MLX5_INVALID_POLICY_ID)
 762                 return -rte_mtr_error_set(error, ENOTSUP,
 763                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 764                                           NULL, "policy ID is invalid. ");
 765         if (policy_id == priv->sh->mtrmng->def_policy_id)
 766                 return -rte_mtr_error_set(error, EEXIST,
 767                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 768                                           NULL, "default policy ID exists. ");
 769         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 770         if (mtr_policy)
 771                 return -rte_mtr_error_set(error, EEXIST,
 772                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 773                                           NULL, "policy ID exists. ");
 774         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 775                                           &is_rss, &domain_bitmap,
 776                                           &policy_mode, error);
 777         if (ret)
 778                 return ret;
 779         if (!domain_bitmap)
 780                 return -rte_mtr_error_set(error, ENOTSUP,
 781                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 782                                           NULL, "fail to find policy domain.");
 783         if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
 784                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 785                         return -rte_mtr_error_set(error, EEXIST,
 786                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 787                                 NULL, "a policy with similar actions "
 788                                 "is already configured");
 789                 if (mlx5_flow_create_def_policy(dev))
 790                         return -rte_mtr_error_set(error, ENOTSUP,
 791                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 792                                 NULL,
 793                                 "fail to create non-terminated policy.");
 794                 priv->sh->mtrmng->def_policy_id = policy_id;
 795                 return 0;
 796         }
 797         if (!priv->sh->meter_aso_en)
 798                 return -rte_mtr_error_set(error, ENOTSUP,
 799                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 800                         "no ASO capability to support the policy ");
 801         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 802                 if (!(domain_bitmap & (1 << i)))
 803                         continue;
 804                 /*
 805                  * If RSS is found, it means that only the ingress domain can
 806                  * be supported. It is invalid to support RSS for one color
 807                  * and egress / transfer domain actions for another. Drop and
 808                  * jump action should have no impact.
 809                  */
 810                 if (is_rss) {
 811                         policy_size +=
 812                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 813                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 814                         break;
 815                 }
 816                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 817         }
 818         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 819                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 820         if (!mtr_policy)
 821                 return -rte_mtr_error_set(error, ENOMEM,
 822                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 823                                 "Memory alloc failed for meter policy.");
 824         if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
 825                 mtr_policy->skip_y = 1;
 826         else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
 827                 mtr_policy->skip_g = 1;
 828         policy_size = sizeof(struct mlx5_flow_meter_policy);
 829         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 830                 if (!(domain_bitmap & (1 << i)))
 831                         continue;
 832                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 833                         mtr_policy->ingress = 1;
 834                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 835                         mtr_policy->egress = 1;
 836                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 837                         mtr_policy->transfer = 1;
 838                 sub_policy = mlx5_ipool_zmalloc
 839                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 840                                  &sub_policy_idx);
 841                 if (!sub_policy ||
 842                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 843                         goto policy_add_err;
 844                 sub_policy->idx = sub_policy_idx;
 845                 sub_policy->main_policy = mtr_policy;
 846                 if (!policy_idx) {
 847                         policy_idx = sub_policy_idx;
 848                         sub_policy->main_policy_id = 1;
 849                 }
 850                 mtr_policy->sub_policys[i] =
 851                         (struct mlx5_flow_meter_sub_policy **)
 852                         ((uint8_t *)mtr_policy + policy_size);
 853                 mtr_policy->sub_policys[i][0] = sub_policy;
 854                 sub_policy_num = (mtr_policy->sub_policy_num >>
 855                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 856                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 857                 sub_policy_num++;
 858                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 859                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 860                 mtr_policy->sub_policy_num |=
 861                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 862                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 863                 /*
 864                  * If RSS is found, it means that only the ingress domain can
 865                  * be supported. It is invalid to support RSS for one color
 866                  * and egress / transfer domain actions for another. Drop and
 867                  * jump action should have no impact.
 868                  */
 869                 if (is_rss) {
 870                         mtr_policy->is_rss = 1;
 871                         break;
 872                 }
 873                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 874         }
 875         rte_spinlock_init(&mtr_policy->sl);
 876         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 877                                         policy->actions, error);
 878         if (ret)
 879                 goto policy_add_err;
 880         if (mtr_policy->is_hierarchy) {
 881                 struct mlx5_flow_meter_policy *final_policy;
 882
 883                 final_policy =
 884                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 885                 if (!final_policy)
 886                         goto policy_add_err;
 887                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 888         }
 889         /*
 890          * If either Green or Yellow has queue / RSS action, all the policy
 891          * rules will be created later in the flow splitting stage.
 892          */
 893         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 894                 /* Create policy rules in HW. */
 895                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 896                 if (ret)
 897                         goto policy_add_err;
 898         }
 899         data.dword = policy_idx;
 900         if (!priv->policy_idx_tbl) {
 901                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 902                 if (!priv->policy_idx_tbl)
 903                         goto policy_add_err;
 904         }
 905         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 906                 goto policy_add_err;
 907         return 0;
 908 policy_add_err:
 909         if (mtr_policy) {
 910                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 911                         mtr_policy, error, false);
 912                 mlx5_free(mtr_policy);
 913                 if (ret)
 914                         return ret;
 915         }
 916         return -rte_mtr_error_set(error, ENOTSUP,
 917                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 918                                   NULL, "Failed to create devx policy.");
 919 }
 920
 921 /**
 922  * Callback to delete MTR policy.
 923  *
 924  * @param[in] dev
 925  *   Pointer to Ethernet device.
 926  * @param[in] policy_id
 927  *   Meter policy id.
 928  * @param[out] error
 929  *   Pointer to the error structure.
 930  *
 931  * @return
 932  *   0 on success, a negative errno value otherwise and rte_errno is set.
 933  */
 934 static int
 935 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 936                           uint32_t policy_id,
 937                           struct rte_mtr_error *error)
 938 {
 939         struct mlx5_priv *priv = dev->data->dev_private;
 940         struct mlx5_flow_meter_policy *mtr_policy;
 941         uint32_t policy_idx;
 942         int ret;
 943
 944         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 945                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 946                         return -rte_mtr_error_set(error, ENOTSUP,
 947                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 948                                 "Meter policy object is being used.");
 949                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 950                 return 0;
 951         }
 952         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 953         if (!mtr_policy)
 954                 return -rte_mtr_error_set(error, ENOTSUP,
 955                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 956                         "Meter policy id is invalid. ");
 957         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 958                                                 error, true);
 959         if (ret)
 960                 return ret;
 961         mlx5_free(mtr_policy);
 962         return 0;
 963 }
 964
 965 /**
 966  * Check meter validation.
 967  *
 968  * @param[in] priv
 969  *   Pointer to mlx5 private data structure.
 970  * @param[in] meter_id
 971  *   Meter id.
 972  * @param[in] params
 973  *   Pointer to rte meter parameters.
 974  * @param[out] error
 975  *   Pointer to rte meter error structure.
 976  *
 977  * @return
 978  *   0 on success, a negative errno value otherwise and rte_errno is set.
 979  */
 980 static int
 981 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 982                          struct rte_mtr_params *params,
 983                          struct rte_mtr_error *error)
 984 {
 985         /* Meter must use global drop action. */
 986         if (!priv->sh->dr_drop_action)
 987                 return -rte_mtr_error_set(error, ENOTSUP,
 988                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 989                                           NULL,
 990                                           "No drop action ready for meter.");
 991         /* Meter params must not be NULL. */
 992         if (params == NULL)
 993                 return -rte_mtr_error_set(error, EINVAL,
 994                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 995                                           NULL, "Meter object params null.");
 996         /* Previous meter color is not supported. */
 997         if (params->use_prev_mtr_color)
 998                 return -rte_mtr_error_set(error, ENOTSUP,
 999                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1000                                           NULL,
1001                                           "Previous meter color "
1002                                           "not supported.");
1003         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
1004                 return -rte_mtr_error_set(error, ENOENT,
1005                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1006                                 NULL, "Meter policy id not valid.");
1007         /* Validate meter id. */
1008         if (mlx5_flow_meter_find(priv, meter_id, NULL))
1009                 return -rte_mtr_error_set(error, EEXIST,
1010                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
1011                         "Meter object already exists.");
1012         return 0;
1013 }
1014
1015 /**
1016  * Modify the flow meter action.
1017  *
1018  * @param[in] priv
1019  *   Pointer to mlx5 private data structure.
1020  * @param[in] fm
1021  *   Pointer to flow meter to be modified.
1022  * @param[in] srtcm
1023  *   Pointer to meter srtcm description parameter.
1024  * @param[in] modify_bits
1025  *   The bit in srtcm to be updated.
1026  * @param[in] active_state
1027  *   The state to be updated.
1028  * @return
1029  *   0 on success, o negative value otherwise.
1030  */
1031 static int
1032 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
1033                 struct mlx5_flow_meter_info *fm,
1034                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
1035                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
1036 {
1037 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1038         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
1039         uint32_t *attr;
1040         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
1041         int ret;
1042         struct mlx5_aso_mtr *aso_mtr = NULL;
1043         uint32_t cbs_cir, ebs_eir, val;
1044
1045         if (priv->sh->meter_aso_en) {
1046                 fm->is_enable = !!is_enable;
1047                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1048                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1049                 if (ret)
1050                         return ret;
1051                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
1052                 if (ret)
1053                         return ret;
1054         } else {
1055                 /* Fill command parameters. */
1056                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
1057                 mod_attr.flow_meter_parameter = in;
1058                 mod_attr.flow_meter_parameter_sz =
1059                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
1060                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1061                         mod_attr.active = !!active_state;
1062                 else
1063                         mod_attr.active = 0;
1064                 attr = in;
1065                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
1066                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1067                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1068                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1069                                 ASO_DSEG_EXP_MASK;
1070                         MLX5_SET(flow_meter_parameters, attr,
1071                                 cbs_exponent, val);
1072                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1073                                 ASO_DSEG_MAN_MASK;
1074                         MLX5_SET(flow_meter_parameters, attr,
1075                                 cbs_mantissa, val);
1076                 }
1077                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1078                         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) &
1079                                 ASO_DSEG_EXP_MASK;
1080                         MLX5_SET(flow_meter_parameters, attr,
1081                                 cir_exponent, val);
1082                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1083                         MLX5_SET(flow_meter_parameters, attr,
1084                                 cir_mantissa, val);
1085                 }
1086                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1087                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1088                                 ASO_DSEG_EXP_MASK;
1089                         MLX5_SET(flow_meter_parameters, attr,
1090                                 ebs_exponent, val);
1091                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1092                                 ASO_DSEG_MAN_MASK;
1093                         MLX5_SET(flow_meter_parameters, attr,
1094                                 ebs_mantissa, val);
1095                 }
1096                 /* Apply modifications to meter only if it was created. */
1097                 if (fm->meter_action) {
1098                         ret = mlx5_glue->dv_modify_flow_action_meter
1099                                         (fm->meter_action, &mod_attr,
1100                                         rte_cpu_to_be_64(modify_bits));
1101                         if (ret)
1102                                 return ret;
1103                 }
1104                 /* Update succeeded modify meter parameters. */
1105                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1106                         fm->active_state = !!active_state;
1107         }
1108         return 0;
1109 #else
1110         (void)priv;
1111         (void)fm;
1112         (void)srtcm;
1113         (void)modify_bits;
1114         (void)active_state;
1115         (void)is_enable;
1116         return -ENOTSUP;
1117 #endif
1118 }
1119
1120 static int
1121 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1122                                 struct mlx5_flow_meter_info *fm,
1123                                 uint64_t stats_mask)
1124 {
1125         fm->bytes_dropped =
1126                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1127         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1128         if (fm->bytes_dropped || fm->pkts_dropped) {
1129                 if (!fm->drop_cnt) {
1130                         /* Alloc policer counters. */
1131                         fm->drop_cnt = mlx5_counter_alloc(dev);
1132                         if (!fm->drop_cnt)
1133                                 return -1;
1134                 }
1135         } else {
1136                 if (fm->drop_cnt) {
1137                         mlx5_counter_free(dev, fm->drop_cnt);
1138                         fm->drop_cnt = 0;
1139                 }
1140         }
1141         return 0;
1142 }
1143
1144 /**
1145  * Create meter rules.
1146  *
1147  * @param[in] dev
1148  *   Pointer to Ethernet device.
1149  * @param[in] meter_id
1150  *   Meter id.
1151  * @param[in] params
1152  *   Pointer to rte meter parameters.
1153  * @param[in] shared
1154  *   Meter shared with other flow or not.
1155  * @param[out] error
1156  *   Pointer to rte meter error structure.
1157  *
1158  * @return
1159  *   0 on success, a negative errno value otherwise and rte_errno is set.
1160  */
1161 static int
1162 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1163                        struct rte_mtr_params *params, int shared,
1164                        struct rte_mtr_error *error)
1165 {
1166         struct mlx5_priv *priv = dev->data->dev_private;
1167         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1168         struct mlx5_flow_meter_profile *fmp;
1169         struct mlx5_flow_meter_info *fm;
1170         /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */
1171         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1172         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1173         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1174                 .size = 0,
1175                 .trunk_size = 64,
1176                 .need_lock = 1,
1177                 .type = "mlx5_flow_mtr_flow_id_pool",
1178         };
1179         struct mlx5_aso_mtr *aso_mtr;
1180         uint32_t mtr_idx, policy_idx;
1181         union mlx5_l3t_data data;
1182         int ret;
1183         uint8_t domain_bitmap;
1184         uint8_t mtr_id_bits;
1185         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1186                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1187
1188         if (!priv->mtr_en)
1189                 return -rte_mtr_error_set(error, ENOTSUP,
1190                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1191                                         "Meter is not supported");
1192         /* Validate the parameters. */
1193         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1194         if (ret)
1195                 return ret;
1196         /* Meter profile must exist. */
1197         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1198         if (fmp == NULL)
1199                 return -rte_mtr_error_set(error, ENOENT,
1200                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1201                         NULL, "Meter profile id not valid.");
1202         /* Meter policy must exist. */
1203         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1204                 __atomic_add_fetch
1205                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1206                         1, __ATOMIC_RELAXED);
1207                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1208                 if (!priv->config.dv_esw_en)
1209                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1210         } else {
1211                 if (!priv->sh->meter_aso_en)
1212                         return -rte_mtr_error_set(error, ENOTSUP,
1213                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1214                                 "Part of the policies cannot be "
1215                                 "supported without ASO ");
1216                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1217                                 params->meter_policy_id, &policy_idx);
1218                 if (!mtr_policy)
1219                         return -rte_mtr_error_set(error, ENOENT,
1220                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1221                                 NULL, "Meter policy id not valid.");
1222                 domain_bitmap = (mtr_policy->ingress ?
1223                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1224                                 (mtr_policy->egress ?
1225                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1226                                 (mtr_policy->transfer ?
1227                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1228                 if (fmp->g_support && mtr_policy->skip_g)
1229                         return -rte_mtr_error_set(error, ENOTSUP,
1230                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1231                                         NULL, "Meter green policy is empty.");
1232                 if (fmp->y_support && mtr_policy->skip_y)
1233                         return -rte_mtr_error_set(error, ENOTSUP,
1234                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1235                                         NULL, "Meter yellow policy is empty.");
1236         }
1237         /* Allocate the flow meter memory. */
1238         if (priv->sh->meter_aso_en) {
1239                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1240                 if (!mtr_idx)
1241                         return -rte_mtr_error_set(error, ENOMEM,
1242                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1243                                 "Memory alloc failed for meter.");
1244                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1245                 fm = &aso_mtr->fm;
1246         } else {
1247                 if (fmp->y_support)
1248                         return -rte_mtr_error_set(error, ENOMEM,
1249                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1250                                 "Unsupported profile with yellow.");
1251                 legacy_fm = mlx5_ipool_zmalloc
1252                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1253                 if (legacy_fm == NULL)
1254                         return -rte_mtr_error_set(error, ENOMEM,
1255                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1256                                 "Memory alloc failed for meter.");
1257                 legacy_fm->idx = mtr_idx;
1258                 fm = &legacy_fm->fm;
1259         }
1260         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1261         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1262             mtr_reg_bits) {
1263                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1264                 goto error;
1265         }
1266         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1267                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1268         /* Fill the flow meter parameters. */
1269         fm->meter_id = meter_id;
1270         fm->policy_id = params->meter_policy_id;
1271         fm->profile = fmp;
1272         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1273                 goto error;
1274         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1275                 goto error;
1276         /* Add to the flow meter list. */
1277         if (!priv->sh->meter_aso_en) {
1278                 MLX5_ASSERT(legacy_fm != NULL);
1279                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1280         }
1281         /* Add to the flow meter list. */
1282         fm->active_state = 1; /* Config meter starts as active. */
1283         fm->is_enable = 1;
1284         fm->shared = !!shared;
1285         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1286         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1287                 fm->def_policy = 1;
1288                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1289                 if (!fm->flow_ipool)
1290                         goto error;
1291         }
1292         rte_spinlock_init(&fm->sl);
1293         /* If ASO meter supported, update ASO flow meter by wqe. */
1294         if (priv->sh->meter_aso_en) {
1295                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1296                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1297                 if (ret)
1298                         goto error;
1299                 if (!priv->mtr_idx_tbl) {
1300                         priv->mtr_idx_tbl =
1301                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1302                         if (!priv->mtr_idx_tbl)
1303                                 goto error;
1304                 }
1305                 data.dword = mtr_idx;
1306                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1307                         goto error;
1308         }
1309         if (mtr_policy)
1310                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1311         return 0;
1312 error:
1313         mlx5_flow_destroy_mtr_tbls(dev, fm);
1314         /* Free policer counters. */
1315         if (fm->drop_cnt)
1316                 mlx5_counter_free(dev, fm->drop_cnt);
1317         if (priv->sh->meter_aso_en)
1318                 mlx5_flow_mtr_free(dev, mtr_idx);
1319         else
1320                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1321         return -rte_mtr_error_set(error, ENOTSUP,
1322                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1323                 NULL, "Failed to create devx meter.");
1324 }
1325
1326 static int
1327 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1328                         struct mlx5_flow_meter_info *fm,
1329                         uint32_t mtr_idx)
1330 {
1331         struct mlx5_priv *priv = dev->data->dev_private;
1332         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1333         struct mlx5_flow_meter_profile *fmp;
1334         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1335         struct mlx5_flow_meter_policy *mtr_policy;
1336
1337         /* Meter object must not have any owner. */
1338         MLX5_ASSERT(!fm->ref_cnt);
1339         /* Get meter profile. */
1340         fmp = fm->profile;
1341         if (fmp == NULL)
1342                 return -1;
1343         /* Update dependencies. */
1344         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1345         fm->profile = NULL;
1346         /* Remove from list. */
1347         if (!priv->sh->meter_aso_en) {
1348                 legacy_fm = container_of(fm,
1349                         struct mlx5_legacy_flow_meter, fm);
1350                 TAILQ_REMOVE(fms, legacy_fm, next);
1351         }
1352         /* Free drop counters. */
1353         if (fm->drop_cnt)
1354                 mlx5_counter_free(dev, fm->drop_cnt);
1355         /* Free meter flow table. */
1356         if (fm->flow_ipool) {
1357                 mlx5_ipool_destroy(fm->flow_ipool);
1358                 fm->flow_ipool = 0;
1359         }
1360         mlx5_flow_destroy_mtr_tbls(dev, fm);
1361         if (fm->def_policy)
1362                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1363                                 1, __ATOMIC_RELAXED);
1364         if (priv->sh->meter_aso_en) {
1365                 if (!fm->def_policy) {
1366                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1367                                                 fm->policy_id, NULL);
1368                         if (mtr_policy)
1369                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1370                                                 1, __ATOMIC_RELAXED);
1371                         fm->policy_id = 0;
1372                 }
1373                 fm->def_policy = 0;
1374                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1375                         return -1;
1376                 mlx5_flow_mtr_free(dev, mtr_idx);
1377         } else {
1378                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1379                                         legacy_fm->idx);
1380         }
1381         return 0;
1382 }
1383
1384 /**
1385  * Destroy meter rules.
1386  *
1387  * @param[in] dev
1388  *   Pointer to Ethernet device.
1389  * @param[in] meter_id
1390  *   Meter id.
1391  * @param[out] error
1392  *   Pointer to rte meter error structure.
1393  *
1394  * @return
1395  *   0 on success, a negative errno value otherwise and rte_errno is set.
1396  */
1397 static int
1398 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1399                         struct rte_mtr_error *error)
1400 {
1401         struct mlx5_priv *priv = dev->data->dev_private;
1402         struct mlx5_flow_meter_info *fm;
1403         uint32_t mtr_idx = 0;
1404
1405         if (!priv->mtr_en)
1406                 return -rte_mtr_error_set(error, ENOTSUP,
1407                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1408                                           NULL,
1409                                           "Meter is not supported");
1410         /* Meter object must exist. */
1411         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1412         if (fm == NULL)
1413                 return -rte_mtr_error_set(error, ENOENT,
1414                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1415                                           NULL,
1416                                           "Meter object id not valid.");
1417         /* Meter object must not have any owner. */
1418         if (fm->ref_cnt > 0)
1419                 return -rte_mtr_error_set(error, EBUSY,
1420                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1421                                           NULL,
1422                                           "Meter object is being used.");
1423         /* Destroy the meter profile. */
1424         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1425                 return -rte_mtr_error_set(error, EINVAL,
1426                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1427                                         NULL,
1428                                         "MTR object meter profile invalid.");
1429         return 0;
1430 }
1431
1432 /**
1433  * Modify meter state.
1434  *
1435  * @param[in] priv
1436  *   Pointer to mlx5 private data structure.
1437  * @param[in] fm
1438  *   Pointer to flow meter.
1439  * @param[in] new_state
1440  *   New state to update.
1441  * @param[out] error
1442  *   Pointer to rte meter error structure.
1443  *
1444  * @return
1445  *   0 on success, a negative errno value otherwise and rte_errno is set.
1446  */
1447 static int
1448 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1449                              struct mlx5_flow_meter_info *fm,
1450                              uint32_t new_state,
1451                              struct rte_mtr_error *error)
1452 {
1453         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1454                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1455                 .ebs_eir = 0,
1456         };
1457         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1458                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1459         int ret;
1460
1461         if (new_state == MLX5_FLOW_METER_DISABLE)
1462                 ret = mlx5_flow_meter_action_modify(priv, fm,
1463                                 &srtcm, modify_bits, 0, 0);
1464         else
1465                 ret = mlx5_flow_meter_action_modify(priv, fm,
1466                                                     &fm->profile->srtcm_prm,
1467                                                     modify_bits, 0, 1);
1468         if (ret)
1469                 return -rte_mtr_error_set(error, -ret,
1470                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1471                                           NULL,
1472                                           new_state ?
1473                                           "Failed to enable meter." :
1474                                           "Failed to disable meter.");
1475         return 0;
1476 }
1477
1478 /**
1479  * Callback to enable flow meter.
1480  *
1481  * @param[in] dev
1482  *   Pointer to Ethernet device.
1483  * @param[in] meter_id
1484  *   Meter id.
1485  * @param[out] error
1486  *   Pointer to rte meter error structure.
1487  *
1488  * @return
1489  *   0 on success, a negative errno value otherwise and rte_errno is set.
1490  */
1491 static int
1492 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1493                        uint32_t meter_id,
1494                        struct rte_mtr_error *error)
1495 {
1496         struct mlx5_priv *priv = dev->data->dev_private;
1497         struct mlx5_flow_meter_info *fm;
1498         int ret;
1499
1500         if (!priv->mtr_en)
1501                 return -rte_mtr_error_set(error, ENOTSUP,
1502                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1503                                           "Meter is not supported");
1504         /* Meter object must exist. */
1505         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1506         if (fm == NULL)
1507                 return -rte_mtr_error_set(error, ENOENT,
1508                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1509                                           NULL, "Meter not found.");
1510         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1511                 return 0;
1512         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1513                                            error);
1514         if (!ret)
1515                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1516         return ret;
1517 }
1518
1519 /**
1520  * Callback to disable flow meter.
1521  *
1522  * @param[in] dev
1523  *   Pointer to Ethernet device.
1524  * @param[in] meter_id
1525  *   Meter id.
1526  * @param[out] error
1527  *   Pointer to rte meter error structure.
1528  *
1529  * @return
1530  *   0 on success, a negative errno value otherwise and rte_errno is set.
1531  */
1532 static int
1533 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1534                         uint32_t meter_id,
1535                         struct rte_mtr_error *error)
1536 {
1537         struct mlx5_priv *priv = dev->data->dev_private;
1538         struct mlx5_flow_meter_info *fm;
1539         int ret;
1540
1541         if (!priv->mtr_en)
1542                 return -rte_mtr_error_set(error, ENOTSUP,
1543                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1544                                           "Meter is not supported");
1545         /* Meter object must exist. */
1546         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1547         if (fm == NULL)
1548                 return -rte_mtr_error_set(error, ENOENT,
1549                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1550                                           NULL, "Meter not found.");
1551         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1552                 return 0;
1553         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1554                                            error);
1555         if (!ret)
1556                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1557         return ret;
1558 }
1559
1560 /**
1561  * Callback to update meter profile.
1562  *
1563  * @param[in] dev
1564  *   Pointer to Ethernet device.
1565  * @param[in] meter_id
1566  *   Meter id.
1567  * @param[in] meter_profile_id
1568  *   To be updated meter profile id.
1569  * @param[out] error
1570  *   Pointer to rte meter error structure.
1571  *
1572  * @return
1573  *   0 on success, a negative errno value otherwise and rte_errno is set.
1574  */
1575 static int
1576 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1577                                uint32_t meter_id,
1578                                uint32_t meter_profile_id,
1579                                struct rte_mtr_error *error)
1580 {
1581         struct mlx5_priv *priv = dev->data->dev_private;
1582         struct mlx5_flow_meter_profile *fmp;
1583         struct mlx5_flow_meter_profile *old_fmp;
1584         struct mlx5_flow_meter_info *fm;
1585         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1586                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1587         int ret;
1588
1589         if (!priv->mtr_en)
1590                 return -rte_mtr_error_set(error, ENOTSUP,
1591                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1592                                           "Meter is not supported");
1593         /* Meter profile must exist. */
1594         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1595         if (fmp == NULL)
1596                 return -rte_mtr_error_set(error, ENOENT,
1597                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1598                                           NULL, "Meter profile not found.");
1599         /* Meter object must exist. */
1600         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1601         if (fm == NULL)
1602                 return -rte_mtr_error_set(error, ENOENT,
1603                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1604                                           NULL, "Meter not found.");
1605         /* MTR object already set to meter profile id. */
1606         old_fmp = fm->profile;
1607         if (fmp == old_fmp)
1608                 return 0;
1609         /* Update the profile. */
1610         fm->profile = fmp;
1611         /* Update meter params in HW (if not disabled). */
1612         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1613                 return 0;
1614         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1615                                               modify_bits, fm->active_state, 1);
1616         if (ret) {
1617                 fm->profile = old_fmp;
1618                 return -rte_mtr_error_set(error, -ret,
1619                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1620                                           NULL, "Failed to update meter"
1621                                           " parameters in hardware.");
1622         }
1623         old_fmp->ref_cnt--;
1624         fmp->ref_cnt++;
1625         return 0;
1626 }
1627
1628 /**
1629  * Callback to update meter stats mask.
1630  *
1631  * @param[in] dev
1632  *   Pointer to Ethernet device.
1633  * @param[in] meter_id
1634  *   Meter id.
1635  * @param[in] stats_mask
1636  *   To be updated stats_mask.
1637  * @param[out] error
1638  *   Pointer to rte meter error structure.
1639  *
1640  * @return
1641  *   0 on success, a negative errno value otherwise and rte_errno is set.
1642  */
1643 static int
1644 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1645                              uint32_t meter_id,
1646                              uint64_t stats_mask,
1647                              struct rte_mtr_error *error)
1648 {
1649         struct mlx5_priv *priv = dev->data->dev_private;
1650         struct mlx5_flow_meter_info *fm;
1651
1652         if (!priv->mtr_en)
1653                 return -rte_mtr_error_set(error, ENOTSUP,
1654                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1655                                           "Meter is not supported");
1656         /* Meter object must exist. */
1657         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1658         if (fm == NULL)
1659                 return -rte_mtr_error_set(error, ENOENT,
1660                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1661                                           NULL, "Meter object id not valid.");
1662         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1663                 return -rte_mtr_error_set(error, ENOENT,
1664                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1665                                           NULL, "Fail to allocate "
1666                                           "counter for meter.");
1667         return 0;
1668 }
1669
1670 /**
1671  * Callback to read meter statistics.
1672  *
1673  * @param[in] dev
1674  *   Pointer to Ethernet device.
1675  * @param[in] meter_id
1676  *   Meter id.
1677  * @param[out] stats
1678  *   Pointer to store the statistics.
1679  * @param[out] stats_mask
1680  *   Pointer to store the stats_mask.
1681  * @param[in] clear
1682  *   Statistic to be cleared after read or not.
1683  * @param[out] error
1684  *   Pointer to rte meter error structure.
1685  *
1686  * @return
1687  *   0 on success, a negative errno value otherwise and rte_errno is set.
1688  */
1689 static int
1690 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1691                            uint32_t meter_id,
1692                            struct rte_mtr_stats *stats,
1693                            uint64_t *stats_mask,
1694                            int clear,
1695                            struct rte_mtr_error *error)
1696 {
1697         struct mlx5_priv *priv = dev->data->dev_private;
1698         struct mlx5_flow_meter_info *fm;
1699         uint64_t pkts;
1700         uint64_t bytes;
1701         int ret = 0;
1702
1703         if (!priv->mtr_en)
1704                 return -rte_mtr_error_set(error, ENOTSUP,
1705                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1706                                           "Meter is not supported");
1707         /* Meter object must exist. */
1708         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1709         if (fm == NULL)
1710                 return -rte_mtr_error_set(error, ENOENT,
1711                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1712                                           NULL, "Meter object id not valid.");
1713         *stats_mask = 0;
1714         if (fm->bytes_dropped)
1715                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1716         if (fm->pkts_dropped)
1717                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1718         memset(stats, 0, sizeof(*stats));
1719         if (fm->drop_cnt) {
1720                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1721                                                  &bytes);
1722                 if (ret)
1723                         goto error;
1724                 /* If need to read the packets, set it. */
1725                 if (fm->pkts_dropped)
1726                         stats->n_pkts_dropped = pkts;
1727                 /* If need to read the bytes, set it. */
1728                 if (fm->bytes_dropped)
1729                         stats->n_bytes_dropped = bytes;
1730         }
1731         return 0;
1732 error:
1733         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1734                                  "Failed to read meter drop counters.");
1735 }
1736
1737 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1738         .capabilities_get = mlx5_flow_mtr_cap_get,
1739         .meter_profile_add = mlx5_flow_meter_profile_add,
1740         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1741         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1742         .meter_policy_add = mlx5_flow_meter_policy_add,
1743         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1744         .create = mlx5_flow_meter_create,
1745         .destroy = mlx5_flow_meter_destroy,
1746         .meter_enable = mlx5_flow_meter_enable,
1747         .meter_disable = mlx5_flow_meter_disable,
1748         .meter_profile_update = mlx5_flow_meter_profile_update,
1749         .meter_dscp_table_update = NULL,
1750         .stats_update = mlx5_flow_meter_stats_update,
1751         .stats_read = mlx5_flow_meter_stats_read,
1752 };
1753
1754 /**
1755  * Get meter operations.
1756  *
1757  * @param dev
1758  *   Pointer to Ethernet device structure.
1759  * @param arg
1760  *   Pointer to set the mtr operations.
1761  *
1762  * @return
1763  *   Always 0.
1764  */
1765 int
1766 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1767 {
1768         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1769         return 0;
1770 }
1771
1772 /**
1773  * Find meter by id.
1774  *
1775  * @param priv
1776  *   Pointer to mlx5_priv.
1777  * @param meter_id
1778  *   Meter id.
1779  * @param mtr_idx
1780  *   Pointer to Meter index.
1781  *
1782  * @return
1783  *   Pointer to the meter info found on success, NULL otherwise.
1784  */
1785 struct mlx5_flow_meter_info *
1786 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1787                 uint32_t *mtr_idx)
1788 {
1789         struct mlx5_legacy_flow_meter *legacy_fm;
1790         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1791         struct mlx5_aso_mtr *aso_mtr;
1792         struct mlx5_aso_mtr_pools_mng *pools_mng =
1793                                 &priv->sh->mtrmng->pools_mng;
1794         union mlx5_l3t_data data;
1795         uint16_t n_valid;
1796
1797         if (priv->sh->meter_aso_en) {
1798                 rte_rwlock_read_lock(&pools_mng->resize_mtrwl);
1799                 n_valid = pools_mng->n_valid;
1800                 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl);
1801                 if (!n_valid || !priv->mtr_idx_tbl ||
1802                     (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1803                     !data.dword))
1804                         return NULL;
1805                 if (mtr_idx)
1806                         *mtr_idx = data.dword;
1807                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1808                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1809                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1810                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1811                         return NULL;
1812                 return &aso_mtr->fm;
1813         }
1814         TAILQ_FOREACH(legacy_fm, fms, next)
1815                 if (meter_id == legacy_fm->fm.meter_id) {
1816                         if (mtr_idx)
1817                                 *mtr_idx = legacy_fm->idx;
1818                         return &legacy_fm->fm;
1819                 }
1820         return NULL;
1821 }
1822
1823 /**
1824  * Find meter by index.
1825  *
1826  * @param priv
1827  *   Pointer to mlx5_priv.
1828  * @param idx
1829  *   Meter index.
1830  *
1831  * @return
1832  *   Pointer to the meter info found on success, NULL otherwise.
1833  */
1834 struct mlx5_flow_meter_info *
1835 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1836 {
1837         struct mlx5_aso_mtr *aso_mtr;
1838
1839         if (priv->sh->meter_aso_en) {
1840                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1841                 if (!aso_mtr)
1842                         return NULL;
1843                 return &aso_mtr->fm;
1844         } else {
1845                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1846         }
1847 }
1848
1849 /**
1850  * Attach meter to flow.
1851  * Unidirectional Meter creation can only be done
1852  * when flow direction is known, i.e. when calling meter_attach.
1853  *
1854  * @param [in] priv
1855  *  Pointer to mlx5 private data.
1856  * @param[in] fm
1857  *   Pointer to flow meter.
1858  * @param [in] attr
1859  *  Pointer to flow attributes.
1860  * @param [out] error
1861  *  Pointer to error structure.
1862  *
1863  * @return
1864  *   0 on success, a negative errno value otherwise and rte_errno is set.
1865  */
1866 int
1867 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1868                        struct mlx5_flow_meter_info *fm,
1869                        const struct rte_flow_attr *attr,
1870                        struct rte_flow_error *error)
1871 {
1872         int ret = 0;
1873
1874         if (priv->sh->meter_aso_en) {
1875                 struct mlx5_aso_mtr *aso_mtr;
1876
1877                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1878                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1879                         return rte_flow_error_set(error, ENOENT,
1880                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1881                                         NULL,
1882                                         "Timeout in meter configuration");
1883                 }
1884                 rte_spinlock_lock(&fm->sl);
1885                 if (fm->shared || !fm->ref_cnt) {
1886                         fm->ref_cnt++;
1887                 } else {
1888                         rte_flow_error_set(error, EINVAL,
1889                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1890                                    "Meter cannot be shared");
1891                         ret = -1;
1892                 }
1893                 rte_spinlock_unlock(&fm->sl);
1894         } else {
1895                 rte_spinlock_lock(&fm->sl);
1896                 if (fm->meter_action) {
1897                         if (fm->shared &&
1898                             attr->transfer == fm->transfer &&
1899                             attr->ingress == fm->ingress &&
1900                             attr->egress == fm->egress) {
1901                                 fm->ref_cnt++;
1902                         } else {
1903                                 rte_flow_error_set(error, EINVAL,
1904                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1905                                         fm->shared ?
1906                                         "Meter attr not match." :
1907                                         "Meter cannot be shared.");
1908                                 ret = -1;
1909                         }
1910                 } else {
1911                         fm->ingress = attr->ingress;
1912                         fm->egress = attr->egress;
1913                         fm->transfer = attr->transfer;
1914                         fm->ref_cnt = 1;
1915                         /* This also creates the meter object. */
1916                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1917                                                                          fm);
1918                         if (!fm->meter_action) {
1919                                 fm->ref_cnt = 0;
1920                                 fm->ingress = 0;
1921                                 fm->egress = 0;
1922                                 fm->transfer = 0;
1923                                 rte_flow_error_set(error, EINVAL,
1924                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1925                                         "Meter action create failed.");
1926                                 ret = -1;
1927                         }
1928                 }
1929                 rte_spinlock_unlock(&fm->sl);
1930         }
1931         return ret ? -rte_errno : 0;
1932 }
1933
1934 /**
1935  * Detach meter from flow.
1936  *
1937  * @param [in] priv
1938  *  Pointer to mlx5 private data.
1939  * @param [in] fm
1940  *  Pointer to flow meter.
1941  */
1942 void
1943 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1944                        struct mlx5_flow_meter_info *fm)
1945 {
1946 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1947         rte_spinlock_lock(&fm->sl);
1948         MLX5_ASSERT(fm->ref_cnt);
1949         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1950                 mlx5_glue->destroy_flow_action(fm->meter_action);
1951                 fm->meter_action = NULL;
1952                 fm->ingress = 0;
1953                 fm->egress = 0;
1954                 fm->transfer = 0;
1955         }
1956         rte_spinlock_unlock(&fm->sl);
1957 #else
1958         (void)priv;
1959         (void)fm;
1960 #endif
1961 }
1962
1963 /**
1964  * Flush meter with Rx queue configuration.
1965  *
1966  * @param[in] dev
1967  *   Pointer to Ethernet device.
1968  */
1969 void
1970 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1971 {
1972         struct mlx5_priv *priv = dev->data->dev_private;
1973         struct mlx5_flow_meter_sub_policy *sub_policy;
1974         struct mlx5_flow_meter_policy *mtr_policy;
1975         void *entry;
1976         uint32_t i, policy_idx;
1977
1978         if (!priv->mtr_en)
1979                 return;
1980         if (priv->policy_idx_tbl) {
1981                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1982                         policy_idx = *(uint32_t *)entry;
1983                         sub_policy = mlx5_ipool_get
1984                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1985                                 policy_idx);
1986                         if (!sub_policy || !sub_policy->main_policy)
1987                                 continue;
1988                         mtr_policy = sub_policy->main_policy;
1989                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1990                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1991                                         mtr_policy);
1992                 }
1993         }
1994 }
1995
1996 /**
1997  * Iterate a meter hierarchy and flush all meters and policies if possible.
1998  *
1999  * @param[in] dev
2000  *   Pointer to Ethernet device.
2001  * @param[in] fm
2002  *   Pointer to flow meter.
2003  * @param[in] mtr_idx
2004  *   .Meter's index
2005  * @param[out] error
2006  *   Pointer to rte meter error structure.
2007  *
2008  * @return
2009  *   0 on success, a negative errno value otherwise and rte_errno is set.
2010  */
2011 static int
2012 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
2013                                 struct mlx5_flow_meter_info *fm,
2014                                 uint32_t mtr_idx,
2015                                 struct rte_mtr_error *error)
2016 {
2017         struct mlx5_priv *priv = dev->data->dev_private;
2018         struct mlx5_flow_meter_policy *policy;
2019         uint32_t policy_id;
2020         struct mlx5_flow_meter_info *next_fm;
2021         uint32_t next_mtr_idx;
2022         struct mlx5_flow_meter_policy *next_policy = NULL;
2023
2024         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
2025         MLX5_ASSERT(policy);
2026         while (!fm->ref_cnt && policy->is_hierarchy) {
2027                 policy_id = fm->policy_id;
2028                 next_fm = mlx5_flow_meter_find(priv,
2029                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2030                                 &next_mtr_idx);
2031                 if (next_fm) {
2032                         next_policy = mlx5_flow_meter_policy_find(dev,
2033                                                         next_fm->policy_id,
2034                                                         NULL);
2035                         MLX5_ASSERT(next_policy);
2036                 }
2037                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
2038                         return -rte_mtr_error_set(error, ENOTSUP,
2039                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
2040                                                 NULL,
2041                                                 "Failed to flush meter.");
2042                 if (policy->ref_cnt)
2043                         break;
2044                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
2045                                                 policy, error, true))
2046                         return -rte_errno;
2047                 mlx5_free(policy);
2048                 if (!next_fm || !next_policy)
2049                         break;
2050                 fm = next_fm;
2051                 mtr_idx = next_mtr_idx;
2052                 policy = next_policy;
2053         }
2054         return 0;
2055 }
2056
2057 /**
2058  * Flush all the hierarchy meters and their policies.
2059  *
2060  * @param[in] dev
2061  *   Pointer to Ethernet device.
2062  * @param[out] error
2063  *   Pointer to rte meter error structure.
2064  *
2065  * @return
2066  *   0 on success, a negative errno value otherwise and rte_errno is set.
2067  */
2068 static int
2069 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
2070                                       struct rte_mtr_error *error)
2071 {
2072         struct mlx5_priv *priv = dev->data->dev_private;
2073         struct mlx5_flow_meter_info *fm;
2074         struct mlx5_flow_meter_policy *policy;
2075         struct mlx5_flow_meter_sub_policy *sub_policy;
2076         struct mlx5_flow_meter_info *next_fm;
2077         struct mlx5_aso_mtr *aso_mtr;
2078         uint32_t mtr_idx = 0;
2079         uint32_t i, policy_idx;
2080         void *entry;
2081
2082         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2083                 return 0;
2084         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2085                 mtr_idx = *(uint32_t *)entry;
2086                 if (!mtr_idx)
2087                         continue;
2088                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2089                 fm = &aso_mtr->fm;
2090                 if (fm->ref_cnt || fm->def_policy)
2091                         continue;
2092                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2093                         return -rte_errno;
2094         }
2095         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2096                 policy_idx = *(uint32_t *)entry;
2097                 sub_policy = mlx5_ipool_get
2098                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2099                                 policy_idx);
2100                 if (!sub_policy)
2101                         return -rte_mtr_error_set(error,
2102                                         EINVAL,
2103                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2104                                         NULL, "Meter policy invalid.");
2105                 policy = sub_policy->main_policy;
2106                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2107                         continue;
2108                 next_fm = mlx5_flow_meter_find(priv,
2109                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2110                                 &mtr_idx);
2111                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2112                                                     error, true))
2113                         return -rte_mtr_error_set(error,
2114                                         EINVAL,
2115                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2116                                         NULL, "Meter policy invalid.");
2117                 mlx5_free(policy);
2118                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2119                         continue;
2120                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2121                                                     mtr_idx, error))
2122                         return -rte_errno;
2123         }
2124         return 0;
2125 }
2126 /**
2127  * Flush meter configuration.
2128  *
2129  * @param[in] dev
2130  *   Pointer to Ethernet device.
2131  * @param[out] error
2132  *   Pointer to rte meter error structure.
2133  *
2134  * @return
2135  *   0 on success, a negative errno value otherwise and rte_errno is set.
2136  */
2137 int
2138 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2139 {
2140         struct mlx5_priv *priv = dev->data->dev_private;
2141         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2142         struct mlx5_flow_meter_profile *fmp;
2143         struct mlx5_legacy_flow_meter *legacy_fm;
2144         struct mlx5_flow_meter_info *fm;
2145         struct mlx5_flow_meter_sub_policy *sub_policy;
2146         void *tmp;
2147         uint32_t i, mtr_idx, policy_idx;
2148         void *entry;
2149         struct mlx5_aso_mtr *aso_mtr;
2150
2151         if (!priv->mtr_en)
2152                 return 0;
2153         if (priv->sh->meter_aso_en) {
2154                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2155                         return -rte_errno;
2156                 if (priv->mtr_idx_tbl) {
2157                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2158                                 mtr_idx = *(uint32_t *)entry;
2159                                 if (mtr_idx) {
2160                                         aso_mtr =
2161                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2162                                         fm = &aso_mtr->fm;
2163                                         (void)mlx5_flow_meter_params_flush(dev,
2164                                                 fm, mtr_idx);
2165                                 }
2166                         }
2167                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2168                         priv->mtr_idx_tbl = NULL;
2169                 }
2170         } else {
2171                 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2172                         fm = &legacy_fm->fm;
2173                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2174                                 return -rte_mtr_error_set(error, EINVAL,
2175                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2176                                 NULL, "MTR object meter profile invalid.");
2177                 }
2178         }
2179         if (priv->policy_idx_tbl) {
2180                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2181                         policy_idx = *(uint32_t *)entry;
2182                         sub_policy = mlx5_ipool_get
2183                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2184                                 policy_idx);
2185                         if (!sub_policy)
2186                                 return -rte_mtr_error_set(error,
2187                                                 EINVAL,
2188                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2189                                                 NULL, "MTR object "
2190                                                 "meter policy invalid.");
2191                         if (__mlx5_flow_meter_policy_delete(dev, i,
2192                                                 sub_policy->main_policy,
2193                                                 error, true))
2194                                 return -rte_mtr_error_set(error,
2195                                                 EINVAL,
2196                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2197                                                 NULL, "MTR object "
2198                                                 "meter policy invalid.");
2199                         mlx5_free(sub_policy->main_policy);
2200                 }
2201                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2202                 priv->policy_idx_tbl = NULL;
2203         }
2204         if (priv->mtr_profile_tbl) {
2205                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2206                         fmp = entry;
2207                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2208                                                            error))
2209                                 return -rte_mtr_error_set(error, EINVAL,
2210                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2211                                                 NULL, "Fail to destroy "
2212                                                 "meter profile.");
2213                 }
2214                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2215                 priv->mtr_profile_tbl = NULL;
2216         }
2217         /* Delete default policy table. */
2218         mlx5_flow_destroy_def_policy(dev);
2219         if (priv->sh->refcnt == 1)
2220                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2221         return 0;
2222 }