drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133         uint32_t ls_factor;
 134         int ret;
 135         uint64_t cir, cbs;
 136         uint64_t eir, ebs;
 137         uint64_t pir, pbs;
 138
 139         /* Profile must not be NULL. */
 140         if (profile == NULL)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 143                                           NULL, "Meter profile is null.");
 144         /* Meter profile ID must be valid. */
 145         if (meter_profile_id == UINT32_MAX)
 146                 return -rte_mtr_error_set(error, EINVAL,
 147                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 148                                           NULL, "Meter profile id not valid.");
 149         /* Meter profile must not exist. */
 150         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 151         if (fmp)
 152                 return -rte_mtr_error_set(error, EEXIST,
 153                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 154                                           NULL,
 155                                           "Meter profile already exists.");
 156         if (!priv->sh->meter_aso_en) {
 157                 /* Old version is even not supported. */
 158                 if (!priv->config.hca_attr.qos.flow_meter_old)
 159                         return -rte_mtr_error_set(error, ENOTSUP,
 160                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 161                                 NULL, "Metering is not supported.");
 162                 /* Old FW metering only supports srTCM. */
 163                 if (profile->alg != RTE_MTR_SRTCM_RFC2697) {
 164                         return -rte_mtr_error_set(error, ENOTSUP,
 165                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 166                                 NULL, "Metering algorithm is not supported.");
 167                 } else if (profile->srtcm_rfc2697.ebs) {
 168                         /* EBS is not supported for old metering. */
 169                         return -rte_mtr_error_set(error, ENOTSUP,
 170                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 171                                 NULL, "EBS is not supported.");
 172                 }
 173                 if (profile->packet_mode)
 174                         return -rte_mtr_error_set(error, ENOTSUP,
 175                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 176                                 "Metering algorithm packet mode is not supported.");
 177         }
 178         ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0;
 179         switch (profile->alg) {
 180         case RTE_MTR_SRTCM_RFC2697:
 181                 cir = profile->srtcm_rfc2697.cir << ls_factor;
 182                 cbs = profile->srtcm_rfc2697.cbs << ls_factor;
 183                 ebs = profile->srtcm_rfc2697.ebs << ls_factor;
 184                 /* EBS could be zero for old metering. */
 185                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 186                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 187                     ebs <= MLX5_SRTCM_XBS_MAX) {
 188                         ret = 0;
 189                 } else {
 190                         ret = -rte_mtr_error_set(error, ENOTSUP,
 191                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 192                                         "Profile values out of range.");
 193                 }
 194                 break;
 195         case RTE_MTR_TRTCM_RFC2698:
 196                 cir = profile->trtcm_rfc2698.cir << ls_factor;
 197                 cbs = profile->trtcm_rfc2698.cbs << ls_factor;
 198                 pir = profile->trtcm_rfc2698.pir << ls_factor;
 199                 pbs = profile->trtcm_rfc2698.pbs << ls_factor;
 200                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 201                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 202                     pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) &&
 203                     pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) {
 204                         ret = 0;
 205                 } else {
 206                         ret = -rte_mtr_error_set(error, ENOTSUP,
 207                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 208                                         "Profile values out of range.");
 209                 }
 210                 break;
 211         case RTE_MTR_TRTCM_RFC4115:
 212                 cir = profile->trtcm_rfc4115.cir << ls_factor;
 213                 cbs = profile->trtcm_rfc4115.cbs << ls_factor;
 214                 eir = profile->trtcm_rfc4115.eir << ls_factor;
 215                 ebs = profile->trtcm_rfc4115.ebs << ls_factor;
 216                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 217                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 218                     eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) {
 219                         ret = 0;
 220                 } else {
 221                         ret = -rte_mtr_error_set(error, ENOTSUP,
 222                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 223                                         "Profile values out of range.");
 224                 }
 225                 break;
 226         default:
 227                 ret = -rte_mtr_error_set(error, ENOTSUP,
 228                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 229                                          "Unknown metering algorithm.");
 230                 break;
 231         }
 232         return ret;
 233 }
 234
 235 /*
 236  * Calculate mantissa and exponent for cir / eir.
 237  *
 238  * @param[in] xir
 239  *   Value to be calculated.
 240  * @param[out] man
 241  *   Pointer to the mantissa.
 242  * @param[out] exp
 243  *   Pointer to the exp.
 244  */
 245 static inline void
 246 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp)
 247 {
 248         int64_t _xir;
 249         int64_t delta = INT64_MAX;
 250         uint8_t _man = 0;
 251         uint8_t _exp = 0;
 252         uint64_t m, e;
 253
 254         /* Special case xir == 0 ? both exp and mantissa are 0. */
 255         if (xir == 0) {
 256                 *man = 0;
 257                 *exp = 0;
 258                 return;
 259         }
 260         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 261                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 262                         _xir = (1000000000ULL * m) >> e;
 263                         if (llabs(xir - _xir) <= delta) {
 264                                 delta = llabs(xir - _xir);
 265                                 _man = m;
 266                                 _exp = e;
 267                         }
 268                 }
 269         }
 270         *man = _man;
 271         *exp = _exp;
 272 }
 273
 274 /*
 275  * Calculate mantissa and exponent for xbs.
 276  *
 277  * @param[in] xbs
 278  *   Value to be calculated.
 279  * @param[out] man
 280  *   Pointer to the mantissa.
 281  * @param[out] exp
 282  *   Pointer to the exp.
 283  */
 284 static inline void
 285 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 286 {
 287         int _exp;
 288         double _man;
 289
 290         /* Special case xbs == 0 ? both exp and mantissa are 0. */
 291         if (xbs == 0) {
 292                 *man = 0;
 293                 *exp = 0;
 294                 return;
 295         }
 296         /* xbs = xbs_mantissa * 2^xbs_exponent */
 297         _man = frexp(xbs, &_exp);
 298         if (_exp >= MLX5_MAN_WIDTH) {
 299                 _man = _man * pow(2, MLX5_MAN_WIDTH);
 300                 _exp = _exp - MLX5_MAN_WIDTH;
 301         }
 302         *man = (uint8_t)ceil(_man);
 303         *exp = _exp;
 304 }
 305
 306 /**
 307  * Fill the prm meter parameter.
 308  *
 309  * @param[in,out] fmp
 310  *   Pointer to meter profile to be converted.
 311  * @param[out] error
 312  *   Pointer to the error structure.
 313  *
 314  * @return
 315  *   0 on success, a negative errno value otherwise and rte_errno is set.
 316  */
 317 static int
 318 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 319                            struct rte_mtr_error *error)
 320 {
 321         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 322         uint8_t man, exp;
 323         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 324         uint32_t eir_exp, eir_man, ebs_exp, ebs_man;
 325         uint64_t cir, cbs, eir, ebs;
 326
 327         switch (fmp->profile.alg) {
 328         case RTE_MTR_SRTCM_RFC2697:
 329                 cir = fmp->profile.srtcm_rfc2697.cir;
 330                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 331                 eir = 0;
 332                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 333                 break;
 334         case RTE_MTR_TRTCM_RFC2698:
 335                 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir >
 336                             fmp->profile.trtcm_rfc2698.cir &&
 337                             fmp->profile.trtcm_rfc2698.pbs >
 338                             fmp->profile.trtcm_rfc2698.cbs);
 339                 cir = fmp->profile.trtcm_rfc2698.cir;
 340                 cbs = fmp->profile.trtcm_rfc2698.cbs;
 341                 /* EIR / EBS are filled with PIR / PBS. */
 342                 eir = fmp->profile.trtcm_rfc2698.pir;
 343                 ebs = fmp->profile.trtcm_rfc2698.pbs;
 344                 break;
 345         case RTE_MTR_TRTCM_RFC4115:
 346                 cir = fmp->profile.trtcm_rfc4115.cir;
 347                 cbs = fmp->profile.trtcm_rfc4115.cbs;
 348                 eir = fmp->profile.trtcm_rfc4115.eir;
 349                 ebs = fmp->profile.trtcm_rfc4115.ebs;
 350                 break;
 351         default:
 352                 return -rte_mtr_error_set(error, EINVAL,
 353                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 354                                 "Metering algorithm mode is invalid");
 355         }
 356         /* Adjust the values for PPS mode. */
 357         if (fmp->profile.packet_mode) {
 358                 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 359                 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 360                 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 361                 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 362         }
 363         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 364         mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp);
 365         /* Check if cir mantissa is too large. */
 366         if (exp > ASO_DSEG_XIR_EXP_MASK)
 367                 return -rte_mtr_error_set(error, ENOTSUP,
 368                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 369                                           "meter profile parameter cir is not supported.");
 370         cir_man = man;
 371         cir_exp = exp;
 372          /* cbs = cbs_mantissa * 2^cbs_exponent */
 373         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 374         /* Check if cbs mantissa is too large. */
 375         if (exp > ASO_DSEG_EXP_MASK)
 376                 return -rte_mtr_error_set(error, ENOTSUP,
 377                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 378                                           "meter profile parameter cbs is not supported.");
 379         cbs_man = man;
 380         cbs_exp = exp;
 381         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 382                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 383                                           cir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 384                                           cir_man);
 385         mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp);
 386         /* Check if eir mantissa is too large. */
 387         if (exp > ASO_DSEG_XIR_EXP_MASK)
 388                 return -rte_mtr_error_set(error, ENOTSUP,
 389                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 390                                           "meter profile parameter eir is not supported.");
 391         eir_man = man;
 392         eir_exp = exp;
 393         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 394         /* Check if ebs mantissa is too large. */
 395         if (exp > ASO_DSEG_EXP_MASK)
 396                 return -rte_mtr_error_set(error, ENOTSUP,
 397                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 398                                           "meter profile parameter ebs is not supported.");
 399         ebs_man = man;
 400         ebs_exp = exp;
 401         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 402                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET |
 403                                           eir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 404                                           eir_man);
 405         if (srtcm->cbs_cir)
 406                 fmp->g_support = 1;
 407         if (srtcm->ebs_eir)
 408                 fmp->y_support = 1;
 409         return 0;
 410 }
 411
 412 /**
 413  * Callback to get MTR capabilities.
 414  *
 415  * @param[in] dev
 416  *   Pointer to Ethernet device.
 417  * @param[out] cap
 418  *   Pointer to save MTR capabilities.
 419  * @param[out] error
 420  *   Pointer to the error structure.
 421  *
 422  * @return
 423  *   0 on success, a negative errno value otherwise and rte_errno is set.
 424  */
 425 static int
 426 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 427                  struct rte_mtr_capabilities *cap,
 428                  struct rte_mtr_error *error __rte_unused)
 429 {
 430         struct mlx5_priv *priv = dev->data->dev_private;
 431         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 432
 433         if (!priv->mtr_en)
 434                 return -rte_mtr_error_set(error, ENOTSUP,
 435                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 436                                           "Meter is not supported");
 437         memset(cap, 0, sizeof(*cap));
 438         if (priv->sh->meter_aso_en) {
 439                 /* 2 meters per one ASO cache line. */
 440                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 441                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 442         } else {
 443                 cap->n_max = 1 << qattr->log_max_flow_meter;
 444                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 445         }
 446         cap->srtcm_rfc2697_byte_mode_supported = 1;
 447         cap->n_shared_max = cap->n_max;
 448         cap->identical = 1;
 449         cap->shared_identical = 1;
 450         cap->shared_n_flows_per_mtr_max = 4 << 20;
 451         /* 2M flows can share the same meter. */
 452         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 453         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 454         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 455         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 456                           RTE_MTR_STATS_N_PKTS_DROPPED;
 457         return 0;
 458 }
 459
 460 /**
 461  * Callback to add MTR profile.
 462  *
 463  * @param[in] dev
 464  *   Pointer to Ethernet device.
 465  * @param[in] meter_profile_id
 466  *   Meter profile id.
 467  * @param[in] profile
 468  *   Pointer to meter profile detail.
 469  * @param[out] error
 470  *   Pointer to the error structure.
 471  *
 472  * @return
 473  *   0 on success, a negative errno value otherwise and rte_errno is set.
 474  */
 475 static int
 476 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 477                        uint32_t meter_profile_id,
 478                        struct rte_mtr_meter_profile *profile,
 479                        struct rte_mtr_error *error)
 480 {
 481         struct mlx5_priv *priv = dev->data->dev_private;
 482         struct mlx5_flow_meter_profile *fmp;
 483         union mlx5_l3t_data data;
 484         int ret;
 485
 486         if (!priv->mtr_en)
 487                 return -rte_mtr_error_set(error, ENOTSUP,
 488                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 489                                           "Meter is not supported");
 490         /* Check input params. */
 491         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 492                                                profile, error);
 493         if (ret)
 494                 return ret;
 495         /* Meter profile memory allocation. */
 496         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 497                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 498         if (fmp == NULL)
 499                 return -rte_mtr_error_set(error, ENOMEM,
 500                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 501                                           NULL, "Meter profile memory "
 502                                           "alloc failed.");
 503         /* Fill profile info. */
 504         fmp->id = meter_profile_id;
 505         fmp->profile = *profile;
 506         /* Fill the flow meter parameters for the PRM. */
 507         ret = mlx5_flow_meter_param_fill(fmp, error);
 508         if (ret)
 509                 goto error;
 510         data.ptr = fmp;
 511         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 512                                  meter_profile_id, &data);
 513         if (ret)
 514                 return -rte_mtr_error_set(error, ENOTSUP,
 515                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 516                                           NULL, "Meter profile insert fail.");
 517         return 0;
 518 error:
 519         mlx5_free(fmp);
 520         return ret;
 521 }
 522
 523 /**
 524  * Callback to delete MTR profile.
 525  *
 526  * @param[in] dev
 527  *   Pointer to Ethernet device.
 528  * @param[in] meter_profile_id
 529  *   Meter profile id.
 530  * @param[out] error
 531  *   Pointer to the error structure.
 532  *
 533  * @return
 534  *   0 on success, a negative errno value otherwise and rte_errno is set.
 535  */
 536 static int
 537 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 538                           uint32_t meter_profile_id,
 539                           struct rte_mtr_error *error)
 540 {
 541         struct mlx5_priv *priv = dev->data->dev_private;
 542         struct mlx5_flow_meter_profile *fmp;
 543
 544         if (!priv->mtr_en)
 545                 return -rte_mtr_error_set(error, ENOTSUP,
 546                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 547                                           "Meter is not supported");
 548         /* Meter profile must exist. */
 549         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 550         if (fmp == NULL)
 551                 return -rte_mtr_error_set(error, ENOENT,
 552                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 553                                           &meter_profile_id,
 554                                           "Meter profile id is invalid.");
 555         /* Check profile is unused. */
 556         if (fmp->ref_cnt)
 557                 return -rte_mtr_error_set(error, EBUSY,
 558                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 559                                           NULL, "Meter profile is in use.");
 560         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 561                 return -rte_mtr_error_set(error, EBUSY,
 562                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 563                                           NULL, "Meter profile remove fail.");
 564         mlx5_free(fmp);
 565         return 0;
 566 }
 567
 568 /**
 569  * Find policy by id.
 570  *
 571  * @param[in] dev
 572  *   Pointer to Ethernet device.
 573  * @param policy_id
 574  *   Policy id.
 575  *
 576  * @return
 577  *   Pointer to the policy found on success, NULL otherwise.
 578  */
 579 struct mlx5_flow_meter_policy *
 580 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 581                             uint32_t policy_id,
 582                             uint32_t *policy_idx)
 583 {
 584         struct mlx5_priv *priv = dev->data->dev_private;
 585         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 586         union mlx5_l3t_data data;
 587
 588         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 589                 return NULL;
 590         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 591                                 !data.dword)
 592                 return NULL;
 593         if (policy_idx)
 594                 *policy_idx = data.dword;
 595         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 596                                         data.dword);
 597         /* Remove reference taken by the mlx5_l3t_get_entry. */
 598         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 599         if (sub_policy)
 600                 if (sub_policy->main_policy_id)
 601                         return sub_policy->main_policy;
 602         return NULL;
 603 }
 604
 605 /**
 606  * Get the last meter's policy from one meter's policy in hierarchy.
 607  *
 608  * @param[in] dev
 609  *   Pointer to Ethernet device.
 610  * @param[in] policy
 611  *   Pointer to flow meter policy.
 612  *
 613  * @return
 614  *   Pointer to the final meter's policy, or NULL when fail.
 615  */
 616 struct mlx5_flow_meter_policy *
 617 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 618                                         struct mlx5_flow_meter_policy *policy)
 619 {
 620         struct mlx5_priv *priv = dev->data->dev_private;
 621         struct mlx5_flow_meter_info *next_fm;
 622         struct mlx5_flow_meter_policy *next_policy = policy;
 623
 624         while (next_policy->is_hierarchy) {
 625                 next_fm = mlx5_flow_meter_find(priv,
 626                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 627                 if (!next_fm || next_fm->def_policy)
 628                         return NULL;
 629                 next_policy = mlx5_flow_meter_policy_find(dev,
 630                                                 next_fm->policy_id, NULL);
 631                 MLX5_ASSERT(next_policy);
 632         }
 633         return next_policy;
 634 }
 635
 636 /**
 637  * Callback to check MTR policy action validate
 638  *
 639  * @param[in] dev
 640  *   Pointer to Ethernet device.
 641  * @param[in] actions
 642  *   Pointer to meter policy action detail.
 643  * @param[out] error
 644  *   Pointer to the error structure.
 645  *
 646  * @return
 647  *   0 on success, a negative errno value otherwise and rte_errno is set.
 648  */
 649 static int
 650 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 651         struct rte_mtr_meter_policy_params *policy,
 652         struct rte_mtr_error *error)
 653 {
 654         struct mlx5_priv *priv = dev->data->dev_private;
 655         struct rte_flow_attr attr = { .transfer =
 656                         priv->config.dv_esw_en ? 1 : 0};
 657         bool is_rss = false;
 658         uint8_t policy_mode;
 659         uint8_t domain_bitmap;
 660         int ret;
 661
 662         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 663                 return -rte_mtr_error_set(error, ENOTSUP,
 664                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 665                                 NULL, "meter policy unsupported.");
 666         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 667                         &is_rss, &domain_bitmap, &policy_mode, error);
 668         if (ret)
 669                 return ret;
 670         return 0;
 671 }
 672
 673 static int
 674 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 675                         uint32_t policy_id,
 676                         struct mlx5_flow_meter_policy *mtr_policy,
 677                         struct rte_mtr_error *error,
 678                         bool clear_l3t)
 679 {
 680         struct mlx5_priv *priv = dev->data->dev_private;
 681         struct mlx5_flow_meter_sub_policy *sub_policy;
 682         uint32_t i, j;
 683         uint16_t sub_policy_num;
 684
 685         rte_spinlock_lock(&mtr_policy->sl);
 686         if (mtr_policy->ref_cnt) {
 687                 rte_spinlock_unlock(&mtr_policy->sl);
 688                 return -rte_mtr_error_set(error, EBUSY,
 689                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 690                                  NULL,
 691                                 "Meter policy object is being used.");
 692         }
 693         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 694         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 695         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 696                 sub_policy_num = (mtr_policy->sub_policy_num >>
 697                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 698                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 699                 if (sub_policy_num) {
 700                         for (j = 0; j < sub_policy_num; j++) {
 701                                 sub_policy = mtr_policy->sub_policys[i][j];
 702                                 if (sub_policy)
 703                                         mlx5_ipool_free
 704                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 705                                         sub_policy->idx);
 706                         }
 707                 }
 708         }
 709         if (priv->policy_idx_tbl && clear_l3t) {
 710                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 711                         rte_spinlock_unlock(&mtr_policy->sl);
 712                         return -rte_mtr_error_set(error, ENOTSUP,
 713                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 714                                 "Fail to delete policy in index table.");
 715                 }
 716         }
 717         rte_spinlock_unlock(&mtr_policy->sl);
 718         return 0;
 719 }
 720
 721 /**
 722  * Callback to add MTR policy.
 723  *
 724  * @param[in] dev
 725  *   Pointer to Ethernet device.
 726  * @param[out] policy_id
 727  *   Pointer to policy id
 728  * @param[in] actions
 729  *   Pointer to meter policy action detail.
 730  * @param[out] error
 731  *   Pointer to the error structure.
 732  *
 733  * @return
 734  *   0 on success, a negative errno value otherwise and rte_errno is set.
 735  */
 736 static int
 737 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 738                         uint32_t policy_id,
 739                         struct rte_mtr_meter_policy_params *policy,
 740                         struct rte_mtr_error *error)
 741 {
 742         struct mlx5_priv *priv = dev->data->dev_private;
 743         struct rte_flow_attr attr = { .transfer =
 744                         priv->config.dv_esw_en ? 1 : 0};
 745         uint32_t sub_policy_idx = 0;
 746         uint32_t policy_idx = 0;
 747         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 748         struct mlx5_flow_meter_sub_policy *sub_policy;
 749         bool is_rss = false;
 750         uint8_t policy_mode;
 751         uint32_t i;
 752         int ret;
 753         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 754         uint16_t sub_policy_num;
 755         uint8_t domain_bitmap = 0;
 756         union mlx5_l3t_data data;
 757         bool skip_rule = false;
 758
 759         if (!priv->mtr_en)
 760                 return -rte_mtr_error_set(error, ENOTSUP,
 761                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 762                                           NULL, "meter policy unsupported. ");
 763         if (policy_id == MLX5_INVALID_POLICY_ID)
 764                 return -rte_mtr_error_set(error, ENOTSUP,
 765                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 766                                           NULL, "policy ID is invalid. ");
 767         if (policy_id == priv->sh->mtrmng->def_policy_id)
 768                 return -rte_mtr_error_set(error, EEXIST,
 769                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 770                                           NULL, "default policy ID exists. ");
 771         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 772         if (mtr_policy)
 773                 return -rte_mtr_error_set(error, EEXIST,
 774                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 775                                           NULL, "policy ID exists. ");
 776         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 777                                           &is_rss, &domain_bitmap,
 778                                           &policy_mode, error);
 779         if (ret)
 780                 return ret;
 781         if (!domain_bitmap)
 782                 return -rte_mtr_error_set(error, ENOTSUP,
 783                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 784                                           NULL, "fail to find policy domain.");
 785         if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
 786                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 787                         return -rte_mtr_error_set(error, EEXIST,
 788                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 789                                 NULL, "a policy with similar actions "
 790                                 "is already configured");
 791                 if (mlx5_flow_create_def_policy(dev))
 792                         return -rte_mtr_error_set(error, ENOTSUP,
 793                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 794                                 NULL,
 795                                 "fail to create non-terminated policy.");
 796                 priv->sh->mtrmng->def_policy_id = policy_id;
 797                 return 0;
 798         }
 799         if (!priv->sh->meter_aso_en)
 800                 return -rte_mtr_error_set(error, ENOTSUP,
 801                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 802                         "no ASO capability to support the policy ");
 803         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 804                 if (!(domain_bitmap & (1 << i)))
 805                         continue;
 806                 /*
 807                  * If RSS is found, it means that only the ingress domain can
 808                  * be supported. It is invalid to support RSS for one color
 809                  * and egress / transfer domain actions for another. Drop and
 810                  * jump action should have no impact.
 811                  */
 812                 if (is_rss) {
 813                         policy_size +=
 814                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 815                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 816                         break;
 817                 }
 818                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 819         }
 820         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 821                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 822         if (!mtr_policy)
 823                 return -rte_mtr_error_set(error, ENOMEM,
 824                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 825                                 "Memory alloc failed for meter policy.");
 826         if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
 827                 mtr_policy->skip_y = 1;
 828         else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
 829                 mtr_policy->skip_g = 1;
 830         policy_size = sizeof(struct mlx5_flow_meter_policy);
 831         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 832                 if (!(domain_bitmap & (1 << i)))
 833                         continue;
 834                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 835                         mtr_policy->ingress = 1;
 836                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 837                         mtr_policy->egress = 1;
 838                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 839                         mtr_policy->transfer = 1;
 840                 sub_policy = mlx5_ipool_zmalloc
 841                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 842                                  &sub_policy_idx);
 843                 if (!sub_policy ||
 844                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 845                         goto policy_add_err;
 846                 sub_policy->idx = sub_policy_idx;
 847                 sub_policy->main_policy = mtr_policy;
 848                 if (!policy_idx) {
 849                         policy_idx = sub_policy_idx;
 850                         sub_policy->main_policy_id = 1;
 851                 }
 852                 mtr_policy->sub_policys[i] =
 853                         (struct mlx5_flow_meter_sub_policy **)
 854                         ((uint8_t *)mtr_policy + policy_size);
 855                 mtr_policy->sub_policys[i][0] = sub_policy;
 856                 sub_policy_num = (mtr_policy->sub_policy_num >>
 857                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 858                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 859                 sub_policy_num++;
 860                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 861                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 862                 mtr_policy->sub_policy_num |=
 863                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 864                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 865                 /*
 866                  * If RSS is found, it means that only the ingress domain can
 867                  * be supported. It is invalid to support RSS for one color
 868                  * and egress / transfer domain actions for another. Drop and
 869                  * jump action should have no impact.
 870                  */
 871                 if (is_rss) {
 872                         mtr_policy->is_rss = 1;
 873                         break;
 874                 }
 875                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 876         }
 877         rte_spinlock_init(&mtr_policy->sl);
 878         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 879                                         policy->actions, error);
 880         if (ret)
 881                 goto policy_add_err;
 882         if (mtr_policy->is_hierarchy) {
 883                 struct mlx5_flow_meter_policy *final_policy;
 884
 885                 final_policy =
 886                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 887                 if (!final_policy)
 888                         goto policy_add_err;
 889                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 890         }
 891         /*
 892          * If either Green or Yellow has queue / RSS action, all the policy
 893          * rules will be created later in the flow splitting stage.
 894          */
 895         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 896                 /* Create policy rules in HW. */
 897                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 898                 if (ret)
 899                         goto policy_add_err;
 900         }
 901         data.dword = policy_idx;
 902         if (!priv->policy_idx_tbl) {
 903                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 904                 if (!priv->policy_idx_tbl)
 905                         goto policy_add_err;
 906         }
 907         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 908                 goto policy_add_err;
 909         return 0;
 910 policy_add_err:
 911         if (mtr_policy) {
 912                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 913                         mtr_policy, error, false);
 914                 mlx5_free(mtr_policy);
 915                 if (ret)
 916                         return ret;
 917         }
 918         return -rte_mtr_error_set(error, ENOTSUP,
 919                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 920                                   NULL, "Failed to create devx policy.");
 921 }
 922
 923 /**
 924  * Callback to delete MTR policy.
 925  *
 926  * @param[in] dev
 927  *   Pointer to Ethernet device.
 928  * @param[in] policy_id
 929  *   Meter policy id.
 930  * @param[out] error
 931  *   Pointer to the error structure.
 932  *
 933  * @return
 934  *   0 on success, a negative errno value otherwise and rte_errno is set.
 935  */
 936 static int
 937 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 938                           uint32_t policy_id,
 939                           struct rte_mtr_error *error)
 940 {
 941         struct mlx5_priv *priv = dev->data->dev_private;
 942         struct mlx5_flow_meter_policy *mtr_policy;
 943         uint32_t policy_idx;
 944         int ret;
 945
 946         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 947                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 948                         return -rte_mtr_error_set(error, ENOTSUP,
 949                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 950                                 "Meter policy object is being used.");
 951                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 952                 return 0;
 953         }
 954         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 955         if (!mtr_policy)
 956                 return -rte_mtr_error_set(error, ENOTSUP,
 957                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 958                         "Meter policy id is invalid. ");
 959         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 960                                                 error, true);
 961         if (ret)
 962                 return ret;
 963         mlx5_free(mtr_policy);
 964         return 0;
 965 }
 966
 967 /**
 968  * Check meter validation.
 969  *
 970  * @param[in] priv
 971  *   Pointer to mlx5 private data structure.
 972  * @param[in] meter_id
 973  *   Meter id.
 974  * @param[in] params
 975  *   Pointer to rte meter parameters.
 976  * @param[out] error
 977  *   Pointer to rte meter error structure.
 978  *
 979  * @return
 980  *   0 on success, a negative errno value otherwise and rte_errno is set.
 981  */
 982 static int
 983 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 984                          struct rte_mtr_params *params,
 985                          struct rte_mtr_error *error)
 986 {
 987         /* Meter must use global drop action. */
 988         if (!priv->sh->dr_drop_action)
 989                 return -rte_mtr_error_set(error, ENOTSUP,
 990                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 991                                           NULL,
 992                                           "No drop action ready for meter.");
 993         /* Meter params must not be NULL. */
 994         if (params == NULL)
 995                 return -rte_mtr_error_set(error, EINVAL,
 996                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 997                                           NULL, "Meter object params null.");
 998         /* Previous meter color is not supported. */
 999         if (params->use_prev_mtr_color)
1000                 return -rte_mtr_error_set(error, ENOTSUP,
1001                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1002                                           NULL,
1003                                           "Previous meter color "
1004                                           "not supported.");
1005         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
1006                 return -rte_mtr_error_set(error, ENOENT,
1007                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1008                                 NULL, "Meter policy id not valid.");
1009         /* Validate meter id. */
1010         if (mlx5_flow_meter_find(priv, meter_id, NULL))
1011                 return -rte_mtr_error_set(error, EEXIST,
1012                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
1013                         "Meter object already exists.");
1014         return 0;
1015 }
1016
1017 /**
1018  * Modify the flow meter action.
1019  *
1020  * @param[in] priv
1021  *   Pointer to mlx5 private data structure.
1022  * @param[in] fm
1023  *   Pointer to flow meter to be modified.
1024  * @param[in] srtcm
1025  *   Pointer to meter srtcm description parameter.
1026  * @param[in] modify_bits
1027  *   The bit in srtcm to be updated.
1028  * @param[in] active_state
1029  *   The state to be updated.
1030  * @return
1031  *   0 on success, o negative value otherwise.
1032  */
1033 static int
1034 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
1035                 struct mlx5_flow_meter_info *fm,
1036                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
1037                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
1038 {
1039 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1040         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
1041         uint32_t *attr;
1042         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
1043         int ret;
1044         struct mlx5_aso_mtr *aso_mtr = NULL;
1045         uint32_t cbs_cir, ebs_eir, val;
1046
1047         if (priv->sh->meter_aso_en) {
1048                 fm->is_enable = !!is_enable;
1049                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1050                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1051                 if (ret)
1052                         return ret;
1053                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
1054                 if (ret)
1055                         return ret;
1056         } else {
1057                 /* Fill command parameters. */
1058                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
1059                 mod_attr.flow_meter_parameter = in;
1060                 mod_attr.flow_meter_parameter_sz =
1061                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
1062                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1063                         mod_attr.active = !!active_state;
1064                 else
1065                         mod_attr.active = 0;
1066                 attr = in;
1067                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
1068                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1069                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1070                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1071                                 ASO_DSEG_EXP_MASK;
1072                         MLX5_SET(flow_meter_parameters, attr,
1073                                 cbs_exponent, val);
1074                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1075                                 ASO_DSEG_MAN_MASK;
1076                         MLX5_SET(flow_meter_parameters, attr,
1077                                 cbs_mantissa, val);
1078                 }
1079                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1080                         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) &
1081                                 ASO_DSEG_EXP_MASK;
1082                         MLX5_SET(flow_meter_parameters, attr,
1083                                 cir_exponent, val);
1084                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1085                         MLX5_SET(flow_meter_parameters, attr,
1086                                 cir_mantissa, val);
1087                 }
1088                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1089                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1090                                 ASO_DSEG_EXP_MASK;
1091                         MLX5_SET(flow_meter_parameters, attr,
1092                                 ebs_exponent, val);
1093                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1094                                 ASO_DSEG_MAN_MASK;
1095                         MLX5_SET(flow_meter_parameters, attr,
1096                                 ebs_mantissa, val);
1097                 }
1098                 /* Apply modifications to meter only if it was created. */
1099                 if (fm->meter_action) {
1100                         ret = mlx5_glue->dv_modify_flow_action_meter
1101                                         (fm->meter_action, &mod_attr,
1102                                         rte_cpu_to_be_64(modify_bits));
1103                         if (ret)
1104                                 return ret;
1105                 }
1106                 /* Update succeeded modify meter parameters. */
1107                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1108                         fm->active_state = !!active_state;
1109         }
1110         return 0;
1111 #else
1112         (void)priv;
1113         (void)fm;
1114         (void)srtcm;
1115         (void)modify_bits;
1116         (void)active_state;
1117         (void)is_enable;
1118         return -ENOTSUP;
1119 #endif
1120 }
1121
1122 static int
1123 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1124                                 struct mlx5_flow_meter_info *fm,
1125                                 uint64_t stats_mask)
1126 {
1127         fm->bytes_dropped =
1128                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1129         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1130         if (fm->bytes_dropped || fm->pkts_dropped) {
1131                 if (!fm->drop_cnt) {
1132                         /* Alloc policer counters. */
1133                         fm->drop_cnt = mlx5_counter_alloc(dev);
1134                         if (!fm->drop_cnt)
1135                                 return -1;
1136                 }
1137         } else {
1138                 if (fm->drop_cnt) {
1139                         mlx5_counter_free(dev, fm->drop_cnt);
1140                         fm->drop_cnt = 0;
1141                 }
1142         }
1143         return 0;
1144 }
1145
1146 /**
1147  * Create meter rules.
1148  *
1149  * @param[in] dev
1150  *   Pointer to Ethernet device.
1151  * @param[in] meter_id
1152  *   Meter id.
1153  * @param[in] params
1154  *   Pointer to rte meter parameters.
1155  * @param[in] shared
1156  *   Meter shared with other flow or not.
1157  * @param[out] error
1158  *   Pointer to rte meter error structure.
1159  *
1160  * @return
1161  *   0 on success, a negative errno value otherwise and rte_errno is set.
1162  */
1163 static int
1164 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1165                        struct rte_mtr_params *params, int shared,
1166                        struct rte_mtr_error *error)
1167 {
1168         struct mlx5_priv *priv = dev->data->dev_private;
1169         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1170         struct mlx5_flow_meter_profile *fmp;
1171         struct mlx5_flow_meter_info *fm;
1172         /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */
1173         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1174         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1175         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1176                 .size = 0,
1177                 .trunk_size = 64,
1178                 .need_lock = 1,
1179                 .type = "mlx5_flow_mtr_flow_id_pool",
1180         };
1181         struct mlx5_aso_mtr *aso_mtr;
1182         uint32_t mtr_idx, policy_idx;
1183         union mlx5_l3t_data data;
1184         int ret;
1185         uint8_t domain_bitmap;
1186         uint8_t mtr_id_bits;
1187         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1188                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1189
1190         if (!priv->mtr_en)
1191                 return -rte_mtr_error_set(error, ENOTSUP,
1192                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1193                                         "Meter is not supported");
1194         /* Validate the parameters. */
1195         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1196         if (ret)
1197                 return ret;
1198         /* Meter profile must exist. */
1199         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1200         if (fmp == NULL)
1201                 return -rte_mtr_error_set(error, ENOENT,
1202                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1203                         NULL, "Meter profile id not valid.");
1204         /* Meter policy must exist. */
1205         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1206                 __atomic_add_fetch
1207                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1208                         1, __ATOMIC_RELAXED);
1209                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1210                 if (!priv->config.dv_esw_en)
1211                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1212         } else {
1213                 if (!priv->sh->meter_aso_en)
1214                         return -rte_mtr_error_set(error, ENOTSUP,
1215                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1216                                 "Part of the policies cannot be "
1217                                 "supported without ASO ");
1218                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1219                                 params->meter_policy_id, &policy_idx);
1220                 if (!mtr_policy)
1221                         return -rte_mtr_error_set(error, ENOENT,
1222                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1223                                 NULL, "Meter policy id not valid.");
1224                 domain_bitmap = (mtr_policy->ingress ?
1225                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1226                                 (mtr_policy->egress ?
1227                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1228                                 (mtr_policy->transfer ?
1229                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1230                 if (fmp->g_support && mtr_policy->skip_g)
1231                         return -rte_mtr_error_set(error, ENOTSUP,
1232                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1233                                         NULL, "Meter green policy is empty.");
1234                 if (fmp->y_support && mtr_policy->skip_y)
1235                         return -rte_mtr_error_set(error, ENOTSUP,
1236                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1237                                         NULL, "Meter yellow policy is empty.");
1238         }
1239         /* Allocate the flow meter memory. */
1240         if (priv->sh->meter_aso_en) {
1241                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1242                 if (!mtr_idx)
1243                         return -rte_mtr_error_set(error, ENOMEM,
1244                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1245                                 "Memory alloc failed for meter.");
1246                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1247                 fm = &aso_mtr->fm;
1248         } else {
1249                 if (fmp->y_support)
1250                         return -rte_mtr_error_set(error, ENOMEM,
1251                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1252                                 "Unsupported profile with yellow.");
1253                 legacy_fm = mlx5_ipool_zmalloc
1254                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1255                 if (legacy_fm == NULL)
1256                         return -rte_mtr_error_set(error, ENOMEM,
1257                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1258                                 "Memory alloc failed for meter.");
1259                 legacy_fm->idx = mtr_idx;
1260                 fm = &legacy_fm->fm;
1261         }
1262         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1263         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1264             mtr_reg_bits) {
1265                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1266                 goto error;
1267         }
1268         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1269                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1270         /* Fill the flow meter parameters. */
1271         fm->meter_id = meter_id;
1272         fm->policy_id = params->meter_policy_id;
1273         fm->profile = fmp;
1274         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1275                 goto error;
1276         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1277                 goto error;
1278         /* Add to the flow meter list. */
1279         if (!priv->sh->meter_aso_en) {
1280                 MLX5_ASSERT(legacy_fm != NULL);
1281                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1282         }
1283         /* Add to the flow meter list. */
1284         fm->active_state = 1; /* Config meter starts as active. */
1285         fm->is_enable = 1;
1286         fm->shared = !!shared;
1287         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1288         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1289                 fm->def_policy = 1;
1290                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1291                 if (!fm->flow_ipool)
1292                         goto error;
1293         }
1294         rte_spinlock_init(&fm->sl);
1295         /* If ASO meter supported, update ASO flow meter by wqe. */
1296         if (priv->sh->meter_aso_en) {
1297                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1298                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1299                 if (ret)
1300                         goto error;
1301                 if (!priv->mtr_idx_tbl) {
1302                         priv->mtr_idx_tbl =
1303                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1304                         if (!priv->mtr_idx_tbl)
1305                                 goto error;
1306                 }
1307                 data.dword = mtr_idx;
1308                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1309                         goto error;
1310         }
1311         if (mtr_policy)
1312                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1313         return 0;
1314 error:
1315         mlx5_flow_destroy_mtr_tbls(dev, fm);
1316         /* Free policer counters. */
1317         if (fm->drop_cnt)
1318                 mlx5_counter_free(dev, fm->drop_cnt);
1319         if (priv->sh->meter_aso_en)
1320                 mlx5_flow_mtr_free(dev, mtr_idx);
1321         else
1322                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1323         return -rte_mtr_error_set(error, ENOTSUP,
1324                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1325                 NULL, "Failed to create devx meter.");
1326 }
1327
1328 static int
1329 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1330                         struct mlx5_flow_meter_info *fm,
1331                         uint32_t mtr_idx)
1332 {
1333         struct mlx5_priv *priv = dev->data->dev_private;
1334         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1335         struct mlx5_flow_meter_profile *fmp;
1336         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1337         struct mlx5_flow_meter_policy *mtr_policy;
1338
1339         /* Meter object must not have any owner. */
1340         MLX5_ASSERT(!fm->ref_cnt);
1341         /* Get meter profile. */
1342         fmp = fm->profile;
1343         if (fmp == NULL)
1344                 return -1;
1345         /* Update dependencies. */
1346         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1347         fm->profile = NULL;
1348         /* Remove from list. */
1349         if (!priv->sh->meter_aso_en) {
1350                 legacy_fm = container_of(fm,
1351                         struct mlx5_legacy_flow_meter, fm);
1352                 TAILQ_REMOVE(fms, legacy_fm, next);
1353         }
1354         /* Free drop counters. */
1355         if (fm->drop_cnt)
1356                 mlx5_counter_free(dev, fm->drop_cnt);
1357         /* Free meter flow table. */
1358         if (fm->flow_ipool) {
1359                 mlx5_ipool_destroy(fm->flow_ipool);
1360                 fm->flow_ipool = 0;
1361         }
1362         mlx5_flow_destroy_mtr_tbls(dev, fm);
1363         if (fm->def_policy)
1364                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1365                                 1, __ATOMIC_RELAXED);
1366         if (priv->sh->meter_aso_en) {
1367                 if (!fm->def_policy) {
1368                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1369                                                 fm->policy_id, NULL);
1370                         if (mtr_policy)
1371                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1372                                                 1, __ATOMIC_RELAXED);
1373                         fm->policy_id = 0;
1374                 }
1375                 fm->def_policy = 0;
1376                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1377                         return -1;
1378                 mlx5_flow_mtr_free(dev, mtr_idx);
1379         } else {
1380                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1381                                         legacy_fm->idx);
1382         }
1383         return 0;
1384 }
1385
1386 /**
1387  * Destroy meter rules.
1388  *
1389  * @param[in] dev
1390  *   Pointer to Ethernet device.
1391  * @param[in] meter_id
1392  *   Meter id.
1393  * @param[out] error
1394  *   Pointer to rte meter error structure.
1395  *
1396  * @return
1397  *   0 on success, a negative errno value otherwise and rte_errno is set.
1398  */
1399 static int
1400 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1401                         struct rte_mtr_error *error)
1402 {
1403         struct mlx5_priv *priv = dev->data->dev_private;
1404         struct mlx5_flow_meter_info *fm;
1405         uint32_t mtr_idx = 0;
1406
1407         if (!priv->mtr_en)
1408                 return -rte_mtr_error_set(error, ENOTSUP,
1409                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1410                                           NULL,
1411                                           "Meter is not supported");
1412         /* Meter object must exist. */
1413         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1414         if (fm == NULL)
1415                 return -rte_mtr_error_set(error, ENOENT,
1416                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1417                                           NULL,
1418                                           "Meter object id not valid.");
1419         /* Meter object must not have any owner. */
1420         if (fm->ref_cnt > 0)
1421                 return -rte_mtr_error_set(error, EBUSY,
1422                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1423                                           NULL,
1424                                           "Meter object is being used.");
1425         /* Destroy the meter profile. */
1426         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1427                 return -rte_mtr_error_set(error, EINVAL,
1428                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1429                                         NULL,
1430                                         "MTR object meter profile invalid.");
1431         return 0;
1432 }
1433
1434 /**
1435  * Modify meter state.
1436  *
1437  * @param[in] priv
1438  *   Pointer to mlx5 private data structure.
1439  * @param[in] fm
1440  *   Pointer to flow meter.
1441  * @param[in] new_state
1442  *   New state to update.
1443  * @param[out] error
1444  *   Pointer to rte meter error structure.
1445  *
1446  * @return
1447  *   0 on success, a negative errno value otherwise and rte_errno is set.
1448  */
1449 static int
1450 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1451                              struct mlx5_flow_meter_info *fm,
1452                              uint32_t new_state,
1453                              struct rte_mtr_error *error)
1454 {
1455         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1456                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1457                 .ebs_eir = 0,
1458         };
1459         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1460                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1461         int ret;
1462
1463         if (new_state == MLX5_FLOW_METER_DISABLE)
1464                 ret = mlx5_flow_meter_action_modify(priv, fm,
1465                                 &srtcm, modify_bits, 0, 0);
1466         else
1467                 ret = mlx5_flow_meter_action_modify(priv, fm,
1468                                                     &fm->profile->srtcm_prm,
1469                                                     modify_bits, 0, 1);
1470         if (ret)
1471                 return -rte_mtr_error_set(error, -ret,
1472                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1473                                           NULL,
1474                                           new_state ?
1475                                           "Failed to enable meter." :
1476                                           "Failed to disable meter.");
1477         return 0;
1478 }
1479
1480 /**
1481  * Callback to enable flow meter.
1482  *
1483  * @param[in] dev
1484  *   Pointer to Ethernet device.
1485  * @param[in] meter_id
1486  *   Meter id.
1487  * @param[out] error
1488  *   Pointer to rte meter error structure.
1489  *
1490  * @return
1491  *   0 on success, a negative errno value otherwise and rte_errno is set.
1492  */
1493 static int
1494 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1495                        uint32_t meter_id,
1496                        struct rte_mtr_error *error)
1497 {
1498         struct mlx5_priv *priv = dev->data->dev_private;
1499         struct mlx5_flow_meter_info *fm;
1500         int ret;
1501
1502         if (!priv->mtr_en)
1503                 return -rte_mtr_error_set(error, ENOTSUP,
1504                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1505                                           "Meter is not supported");
1506         /* Meter object must exist. */
1507         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1508         if (fm == NULL)
1509                 return -rte_mtr_error_set(error, ENOENT,
1510                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1511                                           NULL, "Meter not found.");
1512         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1513                 return 0;
1514         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1515                                            error);
1516         if (!ret)
1517                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1518         return ret;
1519 }
1520
1521 /**
1522  * Callback to disable flow meter.
1523  *
1524  * @param[in] dev
1525  *   Pointer to Ethernet device.
1526  * @param[in] meter_id
1527  *   Meter id.
1528  * @param[out] error
1529  *   Pointer to rte meter error structure.
1530  *
1531  * @return
1532  *   0 on success, a negative errno value otherwise and rte_errno is set.
1533  */
1534 static int
1535 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1536                         uint32_t meter_id,
1537                         struct rte_mtr_error *error)
1538 {
1539         struct mlx5_priv *priv = dev->data->dev_private;
1540         struct mlx5_flow_meter_info *fm;
1541         int ret;
1542
1543         if (!priv->mtr_en)
1544                 return -rte_mtr_error_set(error, ENOTSUP,
1545                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1546                                           "Meter is not supported");
1547         /* Meter object must exist. */
1548         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1549         if (fm == NULL)
1550                 return -rte_mtr_error_set(error, ENOENT,
1551                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1552                                           NULL, "Meter not found.");
1553         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1554                 return 0;
1555         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1556                                            error);
1557         if (!ret)
1558                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1559         return ret;
1560 }
1561
1562 /**
1563  * Callback to update meter profile.
1564  *
1565  * @param[in] dev
1566  *   Pointer to Ethernet device.
1567  * @param[in] meter_id
1568  *   Meter id.
1569  * @param[in] meter_profile_id
1570  *   To be updated meter profile id.
1571  * @param[out] error
1572  *   Pointer to rte meter error structure.
1573  *
1574  * @return
1575  *   0 on success, a negative errno value otherwise and rte_errno is set.
1576  */
1577 static int
1578 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1579                                uint32_t meter_id,
1580                                uint32_t meter_profile_id,
1581                                struct rte_mtr_error *error)
1582 {
1583         struct mlx5_priv *priv = dev->data->dev_private;
1584         struct mlx5_flow_meter_profile *fmp;
1585         struct mlx5_flow_meter_profile *old_fmp;
1586         struct mlx5_flow_meter_info *fm;
1587         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1588                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1589         int ret;
1590
1591         if (!priv->mtr_en)
1592                 return -rte_mtr_error_set(error, ENOTSUP,
1593                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1594                                           "Meter is not supported");
1595         /* Meter profile must exist. */
1596         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1597         if (fmp == NULL)
1598                 return -rte_mtr_error_set(error, ENOENT,
1599                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1600                                           NULL, "Meter profile not found.");
1601         /* Meter object must exist. */
1602         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1603         if (fm == NULL)
1604                 return -rte_mtr_error_set(error, ENOENT,
1605                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1606                                           NULL, "Meter not found.");
1607         /* MTR object already set to meter profile id. */
1608         old_fmp = fm->profile;
1609         if (fmp == old_fmp)
1610                 return 0;
1611         /* Update the profile. */
1612         fm->profile = fmp;
1613         /* Update meter params in HW (if not disabled). */
1614         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1615                 return 0;
1616         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1617                                               modify_bits, fm->active_state, 1);
1618         if (ret) {
1619                 fm->profile = old_fmp;
1620                 return -rte_mtr_error_set(error, -ret,
1621                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1622                                           NULL, "Failed to update meter"
1623                                           " parameters in hardware.");
1624         }
1625         old_fmp->ref_cnt--;
1626         fmp->ref_cnt++;
1627         return 0;
1628 }
1629
1630 /**
1631  * Callback to update meter stats mask.
1632  *
1633  * @param[in] dev
1634  *   Pointer to Ethernet device.
1635  * @param[in] meter_id
1636  *   Meter id.
1637  * @param[in] stats_mask
1638  *   To be updated stats_mask.
1639  * @param[out] error
1640  *   Pointer to rte meter error structure.
1641  *
1642  * @return
1643  *   0 on success, a negative errno value otherwise and rte_errno is set.
1644  */
1645 static int
1646 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1647                              uint32_t meter_id,
1648                              uint64_t stats_mask,
1649                              struct rte_mtr_error *error)
1650 {
1651         struct mlx5_priv *priv = dev->data->dev_private;
1652         struct mlx5_flow_meter_info *fm;
1653
1654         if (!priv->mtr_en)
1655                 return -rte_mtr_error_set(error, ENOTSUP,
1656                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1657                                           "Meter is not supported");
1658         /* Meter object must exist. */
1659         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1660         if (fm == NULL)
1661                 return -rte_mtr_error_set(error, ENOENT,
1662                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1663                                           NULL, "Meter object id not valid.");
1664         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1665                 return -rte_mtr_error_set(error, ENOENT,
1666                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1667                                           NULL, "Fail to allocate "
1668                                           "counter for meter.");
1669         return 0;
1670 }
1671
1672 /**
1673  * Callback to read meter statistics.
1674  *
1675  * @param[in] dev
1676  *   Pointer to Ethernet device.
1677  * @param[in] meter_id
1678  *   Meter id.
1679  * @param[out] stats
1680  *   Pointer to store the statistics.
1681  * @param[out] stats_mask
1682  *   Pointer to store the stats_mask.
1683  * @param[in] clear
1684  *   Statistic to be cleared after read or not.
1685  * @param[out] error
1686  *   Pointer to rte meter error structure.
1687  *
1688  * @return
1689  *   0 on success, a negative errno value otherwise and rte_errno is set.
1690  */
1691 static int
1692 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1693                            uint32_t meter_id,
1694                            struct rte_mtr_stats *stats,
1695                            uint64_t *stats_mask,
1696                            int clear,
1697                            struct rte_mtr_error *error)
1698 {
1699         struct mlx5_priv *priv = dev->data->dev_private;
1700         struct mlx5_flow_meter_info *fm;
1701         uint64_t pkts;
1702         uint64_t bytes;
1703         int ret = 0;
1704
1705         if (!priv->mtr_en)
1706                 return -rte_mtr_error_set(error, ENOTSUP,
1707                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1708                                           "Meter is not supported");
1709         /* Meter object must exist. */
1710         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1711         if (fm == NULL)
1712                 return -rte_mtr_error_set(error, ENOENT,
1713                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1714                                           NULL, "Meter object id not valid.");
1715         *stats_mask = 0;
1716         if (fm->bytes_dropped)
1717                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1718         if (fm->pkts_dropped)
1719                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1720         memset(stats, 0, sizeof(*stats));
1721         if (fm->drop_cnt) {
1722                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1723                                                  &bytes);
1724                 if (ret)
1725                         goto error;
1726                 /* If need to read the packets, set it. */
1727                 if (fm->pkts_dropped)
1728                         stats->n_pkts_dropped = pkts;
1729                 /* If need to read the bytes, set it. */
1730                 if (fm->bytes_dropped)
1731                         stats->n_bytes_dropped = bytes;
1732         }
1733         return 0;
1734 error:
1735         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1736                                  "Failed to read meter drop counters.");
1737 }
1738
1739 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1740         .capabilities_get = mlx5_flow_mtr_cap_get,
1741         .meter_profile_add = mlx5_flow_meter_profile_add,
1742         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1743         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1744         .meter_policy_add = mlx5_flow_meter_policy_add,
1745         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1746         .create = mlx5_flow_meter_create,
1747         .destroy = mlx5_flow_meter_destroy,
1748         .meter_enable = mlx5_flow_meter_enable,
1749         .meter_disable = mlx5_flow_meter_disable,
1750         .meter_profile_update = mlx5_flow_meter_profile_update,
1751         .meter_dscp_table_update = NULL,
1752         .stats_update = mlx5_flow_meter_stats_update,
1753         .stats_read = mlx5_flow_meter_stats_read,
1754 };
1755
1756 /**
1757  * Get meter operations.
1758  *
1759  * @param dev
1760  *   Pointer to Ethernet device structure.
1761  * @param arg
1762  *   Pointer to set the mtr operations.
1763  *
1764  * @return
1765  *   Always 0.
1766  */
1767 int
1768 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1769 {
1770         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1771         return 0;
1772 }
1773
1774 /**
1775  * Find meter by id.
1776  *
1777  * @param priv
1778  *   Pointer to mlx5_priv.
1779  * @param meter_id
1780  *   Meter id.
1781  * @param mtr_idx
1782  *   Pointer to Meter index.
1783  *
1784  * @return
1785  *   Pointer to the meter info found on success, NULL otherwise.
1786  */
1787 struct mlx5_flow_meter_info *
1788 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1789                 uint32_t *mtr_idx)
1790 {
1791         struct mlx5_legacy_flow_meter *legacy_fm;
1792         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1793         struct mlx5_aso_mtr *aso_mtr;
1794         struct mlx5_aso_mtr_pools_mng *pools_mng =
1795                                 &priv->sh->mtrmng->pools_mng;
1796         union mlx5_l3t_data data;
1797         uint16_t n_valid;
1798
1799         if (priv->sh->meter_aso_en) {
1800                 rte_rwlock_read_lock(&pools_mng->resize_mtrwl);
1801                 n_valid = pools_mng->n_valid;
1802                 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl);
1803                 if (!n_valid || !priv->mtr_idx_tbl ||
1804                     (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1805                     !data.dword))
1806                         return NULL;
1807                 if (mtr_idx)
1808                         *mtr_idx = data.dword;
1809                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1810                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1811                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1812                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1813                         return NULL;
1814                 return &aso_mtr->fm;
1815         }
1816         TAILQ_FOREACH(legacy_fm, fms, next)
1817                 if (meter_id == legacy_fm->fm.meter_id) {
1818                         if (mtr_idx)
1819                                 *mtr_idx = legacy_fm->idx;
1820                         return &legacy_fm->fm;
1821                 }
1822         return NULL;
1823 }
1824
1825 /**
1826  * Find meter by index.
1827  *
1828  * @param priv
1829  *   Pointer to mlx5_priv.
1830  * @param idx
1831  *   Meter index.
1832  *
1833  * @return
1834  *   Pointer to the meter info found on success, NULL otherwise.
1835  */
1836 struct mlx5_flow_meter_info *
1837 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1838 {
1839         struct mlx5_aso_mtr *aso_mtr;
1840
1841         if (priv->sh->meter_aso_en) {
1842                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1843                 if (!aso_mtr)
1844                         return NULL;
1845                 return &aso_mtr->fm;
1846         } else {
1847                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1848         }
1849 }
1850
1851 /**
1852  * Attach meter to flow.
1853  * Unidirectional Meter creation can only be done
1854  * when flow direction is known, i.e. when calling meter_attach.
1855  *
1856  * @param [in] priv
1857  *  Pointer to mlx5 private data.
1858  * @param[in] fm
1859  *   Pointer to flow meter.
1860  * @param [in] attr
1861  *  Pointer to flow attributes.
1862  * @param [out] error
1863  *  Pointer to error structure.
1864  *
1865  * @return
1866  *   0 on success, a negative errno value otherwise and rte_errno is set.
1867  */
1868 int
1869 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1870                        struct mlx5_flow_meter_info *fm,
1871                        const struct rte_flow_attr *attr,
1872                        struct rte_flow_error *error)
1873 {
1874         int ret = 0;
1875
1876         if (priv->sh->meter_aso_en) {
1877                 struct mlx5_aso_mtr *aso_mtr;
1878
1879                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1880                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1881                         return rte_flow_error_set(error, ENOENT,
1882                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1883                                         NULL,
1884                                         "Timeout in meter configuration");
1885                 }
1886                 rte_spinlock_lock(&fm->sl);
1887                 if (fm->shared || !fm->ref_cnt) {
1888                         fm->ref_cnt++;
1889                 } else {
1890                         rte_flow_error_set(error, EINVAL,
1891                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1892                                    "Meter cannot be shared");
1893                         ret = -1;
1894                 }
1895                 rte_spinlock_unlock(&fm->sl);
1896         } else {
1897                 rte_spinlock_lock(&fm->sl);
1898                 if (fm->meter_action) {
1899                         if (fm->shared &&
1900                             attr->transfer == fm->transfer &&
1901                             attr->ingress == fm->ingress &&
1902                             attr->egress == fm->egress) {
1903                                 fm->ref_cnt++;
1904                         } else {
1905                                 rte_flow_error_set(error, EINVAL,
1906                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1907                                         fm->shared ?
1908                                         "Meter attr not match." :
1909                                         "Meter cannot be shared.");
1910                                 ret = -1;
1911                         }
1912                 } else {
1913                         fm->ingress = attr->ingress;
1914                         fm->egress = attr->egress;
1915                         fm->transfer = attr->transfer;
1916                         fm->ref_cnt = 1;
1917                         /* This also creates the meter object. */
1918                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1919                                                                          fm);
1920                         if (!fm->meter_action) {
1921                                 fm->ref_cnt = 0;
1922                                 fm->ingress = 0;
1923                                 fm->egress = 0;
1924                                 fm->transfer = 0;
1925                                 rte_flow_error_set(error, EINVAL,
1926                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1927                                         "Meter action create failed.");
1928                                 ret = -1;
1929                         }
1930                 }
1931                 rte_spinlock_unlock(&fm->sl);
1932         }
1933         return ret ? -rte_errno : 0;
1934 }
1935
1936 /**
1937  * Detach meter from flow.
1938  *
1939  * @param [in] priv
1940  *  Pointer to mlx5 private data.
1941  * @param [in] fm
1942  *  Pointer to flow meter.
1943  */
1944 void
1945 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1946                        struct mlx5_flow_meter_info *fm)
1947 {
1948 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1949         rte_spinlock_lock(&fm->sl);
1950         MLX5_ASSERT(fm->ref_cnt);
1951         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1952                 mlx5_glue->destroy_flow_action(fm->meter_action);
1953                 fm->meter_action = NULL;
1954                 fm->ingress = 0;
1955                 fm->egress = 0;
1956                 fm->transfer = 0;
1957         }
1958         rte_spinlock_unlock(&fm->sl);
1959 #else
1960         (void)priv;
1961         (void)fm;
1962 #endif
1963 }
1964
1965 /**
1966  * Flush meter with Rx queue configuration.
1967  *
1968  * @param[in] dev
1969  *   Pointer to Ethernet device.
1970  */
1971 void
1972 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1973 {
1974         struct mlx5_priv *priv = dev->data->dev_private;
1975         struct mlx5_flow_meter_sub_policy *sub_policy;
1976         struct mlx5_flow_meter_policy *mtr_policy;
1977         void *entry;
1978         uint32_t i, policy_idx;
1979
1980         if (!priv->mtr_en)
1981                 return;
1982         if (priv->policy_idx_tbl) {
1983                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1984                         policy_idx = *(uint32_t *)entry;
1985                         sub_policy = mlx5_ipool_get
1986                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1987                                 policy_idx);
1988                         if (!sub_policy || !sub_policy->main_policy)
1989                                 continue;
1990                         mtr_policy = sub_policy->main_policy;
1991                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1992                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1993                                         mtr_policy);
1994                 }
1995         }
1996 }
1997
1998 /**
1999  * Iterate a meter hierarchy and flush all meters and policies if possible.
2000  *
2001  * @param[in] dev
2002  *   Pointer to Ethernet device.
2003  * @param[in] fm
2004  *   Pointer to flow meter.
2005  * @param[in] mtr_idx
2006  *   .Meter's index
2007  * @param[out] error
2008  *   Pointer to rte meter error structure.
2009  *
2010  * @return
2011  *   0 on success, a negative errno value otherwise and rte_errno is set.
2012  */
2013 static int
2014 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
2015                                 struct mlx5_flow_meter_info *fm,
2016                                 uint32_t mtr_idx,
2017                                 struct rte_mtr_error *error)
2018 {
2019         struct mlx5_priv *priv = dev->data->dev_private;
2020         struct mlx5_flow_meter_policy *policy;
2021         uint32_t policy_id;
2022         struct mlx5_flow_meter_info *next_fm;
2023         uint32_t next_mtr_idx;
2024         struct mlx5_flow_meter_policy *next_policy = NULL;
2025
2026         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
2027         MLX5_ASSERT(policy);
2028         while (!fm->ref_cnt && policy->is_hierarchy) {
2029                 policy_id = fm->policy_id;
2030                 next_fm = mlx5_flow_meter_find(priv,
2031                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2032                                 &next_mtr_idx);
2033                 if (next_fm) {
2034                         next_policy = mlx5_flow_meter_policy_find(dev,
2035                                                         next_fm->policy_id,
2036                                                         NULL);
2037                         MLX5_ASSERT(next_policy);
2038                 }
2039                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
2040                         return -rte_mtr_error_set(error, ENOTSUP,
2041                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
2042                                                 NULL,
2043                                                 "Failed to flush meter.");
2044                 if (policy->ref_cnt)
2045                         break;
2046                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
2047                                                 policy, error, true))
2048                         return -rte_errno;
2049                 mlx5_free(policy);
2050                 if (!next_fm || !next_policy)
2051                         break;
2052                 fm = next_fm;
2053                 mtr_idx = next_mtr_idx;
2054                 policy = next_policy;
2055         }
2056         return 0;
2057 }
2058
2059 /**
2060  * Flush all the hierarchy meters and their policies.
2061  *
2062  * @param[in] dev
2063  *   Pointer to Ethernet device.
2064  * @param[out] error
2065  *   Pointer to rte meter error structure.
2066  *
2067  * @return
2068  *   0 on success, a negative errno value otherwise and rte_errno is set.
2069  */
2070 static int
2071 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
2072                                       struct rte_mtr_error *error)
2073 {
2074         struct mlx5_priv *priv = dev->data->dev_private;
2075         struct mlx5_flow_meter_info *fm;
2076         struct mlx5_flow_meter_policy *policy;
2077         struct mlx5_flow_meter_sub_policy *sub_policy;
2078         struct mlx5_flow_meter_info *next_fm;
2079         struct mlx5_aso_mtr *aso_mtr;
2080         uint32_t mtr_idx = 0;
2081         uint32_t i, policy_idx;
2082         void *entry;
2083
2084         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2085                 return 0;
2086         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2087                 mtr_idx = *(uint32_t *)entry;
2088                 if (!mtr_idx)
2089                         continue;
2090                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2091                 fm = &aso_mtr->fm;
2092                 if (fm->ref_cnt || fm->def_policy)
2093                         continue;
2094                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2095                         return -rte_errno;
2096         }
2097         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2098                 policy_idx = *(uint32_t *)entry;
2099                 sub_policy = mlx5_ipool_get
2100                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2101                                 policy_idx);
2102                 if (!sub_policy)
2103                         return -rte_mtr_error_set(error,
2104                                         EINVAL,
2105                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2106                                         NULL, "Meter policy invalid.");
2107                 policy = sub_policy->main_policy;
2108                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2109                         continue;
2110                 next_fm = mlx5_flow_meter_find(priv,
2111                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2112                                 &mtr_idx);
2113                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2114                                                     error, true))
2115                         return -rte_mtr_error_set(error,
2116                                         EINVAL,
2117                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2118                                         NULL, "Meter policy invalid.");
2119                 mlx5_free(policy);
2120                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2121                         continue;
2122                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2123                                                     mtr_idx, error))
2124                         return -rte_errno;
2125         }
2126         return 0;
2127 }
2128 /**
2129  * Flush meter configuration.
2130  *
2131  * @param[in] dev
2132  *   Pointer to Ethernet device.
2133  * @param[out] error
2134  *   Pointer to rte meter error structure.
2135  *
2136  * @return
2137  *   0 on success, a negative errno value otherwise and rte_errno is set.
2138  */
2139 int
2140 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2141 {
2142         struct mlx5_priv *priv = dev->data->dev_private;
2143         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2144         struct mlx5_flow_meter_profile *fmp;
2145         struct mlx5_legacy_flow_meter *legacy_fm;
2146         struct mlx5_flow_meter_info *fm;
2147         struct mlx5_flow_meter_sub_policy *sub_policy;
2148         void *tmp;
2149         uint32_t i, mtr_idx, policy_idx;
2150         void *entry;
2151         struct mlx5_aso_mtr *aso_mtr;
2152
2153         if (!priv->mtr_en)
2154                 return 0;
2155         if (priv->sh->meter_aso_en) {
2156                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2157                         return -rte_errno;
2158                 if (priv->mtr_idx_tbl) {
2159                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2160                                 mtr_idx = *(uint32_t *)entry;
2161                                 if (mtr_idx) {
2162                                         aso_mtr =
2163                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2164                                         fm = &aso_mtr->fm;
2165                                         (void)mlx5_flow_meter_params_flush(dev,
2166                                                 fm, mtr_idx);
2167                                 }
2168                         }
2169                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2170                         priv->mtr_idx_tbl = NULL;
2171                 }
2172         } else {
2173                 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2174                         fm = &legacy_fm->fm;
2175                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2176                                 return -rte_mtr_error_set(error, EINVAL,
2177                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2178                                 NULL, "MTR object meter profile invalid.");
2179                 }
2180         }
2181         if (priv->policy_idx_tbl) {
2182                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2183                         policy_idx = *(uint32_t *)entry;
2184                         sub_policy = mlx5_ipool_get
2185                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2186                                 policy_idx);
2187                         if (!sub_policy)
2188                                 return -rte_mtr_error_set(error,
2189                                                 EINVAL,
2190                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2191                                                 NULL, "MTR object "
2192                                                 "meter policy invalid.");
2193                         if (__mlx5_flow_meter_policy_delete(dev, i,
2194                                                 sub_policy->main_policy,
2195                                                 error, true))
2196                                 return -rte_mtr_error_set(error,
2197                                                 EINVAL,
2198                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2199                                                 NULL, "MTR object "
2200                                                 "meter policy invalid.");
2201                         mlx5_free(sub_policy->main_policy);
2202                 }
2203                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2204                 priv->policy_idx_tbl = NULL;
2205         }
2206         if (priv->mtr_profile_tbl) {
2207                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2208                         fmp = entry;
2209                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2210                                                            error))
2211                                 return -rte_mtr_error_set(error, EINVAL,
2212                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2213                                                 NULL, "Fail to destroy "
2214                                                 "meter profile.");
2215                 }
2216                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2217                 priv->mtr_profile_tbl = NULL;
2218         }
2219         /* Delete default policy table. */
2220         mlx5_flow_destroy_def_policy(dev);
2221         if (priv->sh->refcnt == 1)
2222                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2223         return 0;
2224 }