drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133         uint32_t ls_factor;
 134         int ret;
 135         uint64_t cir, cbs;
 136         uint64_t eir, ebs;
 137         uint64_t pir, pbs;
 138
 139         /* Profile must not be NULL. */
 140         if (profile == NULL)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 143                                           NULL, "Meter profile is null.");
 144         /* Meter profile ID must be valid. */
 145         if (meter_profile_id == UINT32_MAX)
 146                 return -rte_mtr_error_set(error, EINVAL,
 147                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 148                                           NULL, "Meter profile id not valid.");
 149         /* Meter profile must not exist. */
 150         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 151         if (fmp)
 152                 return -rte_mtr_error_set(error, EEXIST,
 153                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 154                                           NULL,
 155                                           "Meter profile already exists.");
 156         if (!priv->sh->meter_aso_en) {
 157                 /* Old version is even not supported. */
 158                 if (!priv->config.hca_attr.qos.flow_meter_old)
 159                         return -rte_mtr_error_set(error, ENOTSUP,
 160                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 161                                 NULL, "Metering is not supported.");
 162                 /* Old FW metering only supports srTCM. */
 163                 if (profile->alg != RTE_MTR_SRTCM_RFC2697) {
 164                         return -rte_mtr_error_set(error, ENOTSUP,
 165                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 166                                 NULL, "Metering algorithm is not supported.");
 167                 } else if (profile->srtcm_rfc2697.ebs) {
 168                         /* EBS is not supported for old metering. */
 169                         return -rte_mtr_error_set(error, ENOTSUP,
 170                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 171                                 NULL, "EBS is not supported.");
 172                 }
 173                 if (profile->packet_mode)
 174                         return -rte_mtr_error_set(error, ENOTSUP,
 175                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 176                                 "Metering algorithm packet mode is not supported.");
 177         }
 178         ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0;
 179         switch (profile->alg) {
 180         case RTE_MTR_SRTCM_RFC2697:
 181                 cir = profile->srtcm_rfc2697.cir << ls_factor;
 182                 cbs = profile->srtcm_rfc2697.cbs << ls_factor;
 183                 ebs = profile->srtcm_rfc2697.ebs << ls_factor;
 184                 /* EBS could be zero for old metering. */
 185                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 186                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 187                     ebs <= MLX5_SRTCM_XBS_MAX) {
 188                         ret = 0;
 189                 } else {
 190                         ret = -rte_mtr_error_set(error, ENOTSUP,
 191                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 192                                         "Profile values out of range.");
 193                 }
 194                 break;
 195         case RTE_MTR_TRTCM_RFC2698:
 196                 cir = profile->trtcm_rfc2698.cir << ls_factor;
 197                 cbs = profile->trtcm_rfc2698.cbs << ls_factor;
 198                 pir = profile->trtcm_rfc2698.pir << ls_factor;
 199                 pbs = profile->trtcm_rfc2698.pbs << ls_factor;
 200                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 201                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 202                     pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) &&
 203                     pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) {
 204                         ret = 0;
 205                 } else {
 206                         ret = -rte_mtr_error_set(error, ENOTSUP,
 207                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 208                                         "Profile values out of range.");
 209                 }
 210                 break;
 211         case RTE_MTR_TRTCM_RFC4115:
 212                 cir = profile->trtcm_rfc4115.cir << ls_factor;
 213                 cbs = profile->trtcm_rfc4115.cbs << ls_factor;
 214                 eir = profile->trtcm_rfc4115.eir << ls_factor;
 215                 ebs = profile->trtcm_rfc4115.ebs << ls_factor;
 216                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 217                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 218                     eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) {
 219                         ret = 0;
 220                 } else {
 221                         ret = -rte_mtr_error_set(error, ENOTSUP,
 222                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 223                                         "Profile values out of range.");
 224                 }
 225                 break;
 226         default:
 227                 ret = -rte_mtr_error_set(error, ENOTSUP,
 228                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 229                                          "Unknown metering algorithm.");
 230                 break;
 231         }
 232         return ret;
 233 }
 234
 235 /*
 236  * Calculate mantissa and exponent for cir / eir.
 237  *
 238  * @param[in] xir
 239  *   Value to be calculated.
 240  * @param[out] man
 241  *   Pointer to the mantissa.
 242  * @param[out] exp
 243  *   Pointer to the exp.
 244  */
 245 static inline void
 246 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp)
 247 {
 248         int64_t _cir;
 249         int64_t delta = INT64_MAX;
 250         uint8_t _man = 0;
 251         uint8_t _exp = 0;
 252         uint64_t m, e;
 253
 254         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 255                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 256                         _cir = (1000000000ULL * m) >> e;
 257                         if (llabs(xir - _cir) <= delta) {
 258                                 delta = llabs(xir - _cir);
 259                                 _man = m;
 260                                 _exp = e;
 261                         }
 262                 }
 263         }
 264         *man = _man;
 265         *exp = _exp;
 266 }
 267
 268 /*
 269  * Calculate mantissa and exponent for xbs.
 270  *
 271  * @param[in] xbs
 272  *   Value to be calculated.
 273  * @param[out] man
 274  *   Pointer to the mantissa.
 275  * @param[out] exp
 276  *   Pointer to the exp.
 277  */
 278 static inline void
 279 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 280 {
 281         int _exp;
 282         double _man;
 283
 284         /* Special case xbs == 0 ? both exp and matissa are 0. */
 285         if (xbs == 0) {
 286                 *man = 0;
 287                 *exp = 0;
 288                 return;
 289         }
 290         /* xbs = xbs_mantissa * 2^xbs_exponent */
 291         _man = frexp(xbs, &_exp);
 292         _man = _man * pow(2, MLX5_MAN_WIDTH);
 293         _exp = _exp - MLX5_MAN_WIDTH;
 294         *man = (uint8_t)ceil(_man);
 295         *exp = _exp;
 296 }
 297
 298 /**
 299  * Fill the prm meter parameter.
 300  *
 301  * @param[in,out] fmp
 302  *   Pointer to meter profie to be converted.
 303  * @param[out] error
 304  *   Pointer to the error structure.
 305  *
 306  * @return
 307  *   0 on success, a negative errno value otherwise and rte_errno is set.
 308  */
 309 static int
 310 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 311                            struct rte_mtr_error *error)
 312 {
 313         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 314         uint8_t man, exp;
 315         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 316         uint32_t eir_exp, eir_man, ebs_exp, ebs_man;
 317         uint64_t cir, cbs, eir, ebs;
 318
 319         switch (fmp->profile.alg) {
 320         case RTE_MTR_SRTCM_RFC2697:
 321                 cir = fmp->profile.srtcm_rfc2697.cir;
 322                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 323                 eir = 0;
 324                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 325                 break;
 326         case RTE_MTR_TRTCM_RFC2698:
 327                 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir >
 328                             fmp->profile.trtcm_rfc2698.cir &&
 329                             fmp->profile.trtcm_rfc2698.pbs >
 330                             fmp->profile.trtcm_rfc2698.cbs);
 331                 cir = fmp->profile.trtcm_rfc2698.cir;
 332                 cbs = fmp->profile.trtcm_rfc2698.cbs;
 333                 /* EIR / EBS are filled with PIR / PBS. */
 334                 eir = fmp->profile.trtcm_rfc2698.pir;
 335                 ebs = fmp->profile.trtcm_rfc2698.pbs;
 336                 break;
 337         case RTE_MTR_TRTCM_RFC4115:
 338                 cir = fmp->profile.trtcm_rfc4115.cir;
 339                 cbs = fmp->profile.trtcm_rfc4115.cbs;
 340                 eir = fmp->profile.trtcm_rfc4115.eir;
 341                 ebs = fmp->profile.trtcm_rfc4115.ebs;
 342                 break;
 343         default:
 344                 return -rte_mtr_error_set(error, EINVAL,
 345                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 346                                 "Metering algorithm mode is invalid");
 347         }
 348         /* Adjust the values for PPS mode. */
 349         if (fmp->profile.packet_mode) {
 350                 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 351                 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 352                 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 353                 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 354         }
 355         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 356         mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp);
 357         /* Check if cir mantissa is too large. */
 358         if (exp > ASO_DSEG_XIR_EXP_MASK)
 359                 return -rte_mtr_error_set(error, ENOTSUP,
 360                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 361                                           "meter profile parameter cir is not supported.");
 362         cir_man = man;
 363         cir_exp = exp;
 364          /* cbs = cbs_mantissa * 2^cbs_exponent */
 365         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 366         /* Check if cbs mantissa is too large. */
 367         if (exp > ASO_DSEG_EXP_MASK)
 368                 return -rte_mtr_error_set(error, ENOTSUP,
 369                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 370                                           "meter profile parameter cbs is not supported.");
 371         cbs_man = man;
 372         cbs_exp = exp;
 373         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 374                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 375                                           cir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 376                                           cir_man);
 377         mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp);
 378         /* Check if eir mantissa is too large. */
 379         if (exp > ASO_DSEG_XIR_EXP_MASK)
 380                 return -rte_mtr_error_set(error, ENOTSUP,
 381                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 382                                           "meter profile parameter eir is not supported.");
 383         eir_man = man;
 384         eir_exp = exp;
 385         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 386         /* Check if ebs mantissa is too large. */
 387         if (exp > ASO_DSEG_EXP_MASK)
 388                 return -rte_mtr_error_set(error, ENOTSUP,
 389                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 390                                           "meter profile parameter ebs is not supported.");
 391         ebs_man = man;
 392         ebs_exp = exp;
 393         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 394                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET |
 395                                           eir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 396                                           eir_man);
 397         if (srtcm->cbs_cir)
 398                 fmp->g_support = 1;
 399         if (srtcm->ebs_eir)
 400                 fmp->y_support = 1;
 401         return 0;
 402 }
 403
 404 /**
 405  * Callback to get MTR capabilities.
 406  *
 407  * @param[in] dev
 408  *   Pointer to Ethernet device.
 409  * @param[out] cap
 410  *   Pointer to save MTR capabilities.
 411  * @param[out] error
 412  *   Pointer to the error structure.
 413  *
 414  * @return
 415  *   0 on success, a negative errno value otherwise and rte_errno is set.
 416  */
 417 static int
 418 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 419                  struct rte_mtr_capabilities *cap,
 420                  struct rte_mtr_error *error __rte_unused)
 421 {
 422         struct mlx5_priv *priv = dev->data->dev_private;
 423         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 424
 425         if (!priv->mtr_en)
 426                 return -rte_mtr_error_set(error, ENOTSUP,
 427                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 428                                           "Meter is not supported");
 429         memset(cap, 0, sizeof(*cap));
 430         if (priv->sh->meter_aso_en) {
 431                 /* 2 meters per one ASO cache line. */
 432                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 433                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 434         } else {
 435                 cap->n_max = 1 << qattr->log_max_flow_meter;
 436                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 437         }
 438         cap->srtcm_rfc2697_byte_mode_supported = 1;
 439         cap->n_shared_max = cap->n_max;
 440         cap->identical = 1;
 441         cap->shared_identical = 1;
 442         cap->shared_n_flows_per_mtr_max = 4 << 20;
 443         /* 2M flows can share the same meter. */
 444         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 445         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 446         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 447         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 448                           RTE_MTR_STATS_N_PKTS_DROPPED;
 449         return 0;
 450 }
 451
 452 /**
 453  * Callback to add MTR profile.
 454  *
 455  * @param[in] dev
 456  *   Pointer to Ethernet device.
 457  * @param[in] meter_profile_id
 458  *   Meter profile id.
 459  * @param[in] profile
 460  *   Pointer to meter profile detail.
 461  * @param[out] error
 462  *   Pointer to the error structure.
 463  *
 464  * @return
 465  *   0 on success, a negative errno value otherwise and rte_errno is set.
 466  */
 467 static int
 468 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 469                        uint32_t meter_profile_id,
 470                        struct rte_mtr_meter_profile *profile,
 471                        struct rte_mtr_error *error)
 472 {
 473         struct mlx5_priv *priv = dev->data->dev_private;
 474         struct mlx5_flow_meter_profile *fmp;
 475         union mlx5_l3t_data data;
 476         int ret;
 477
 478         if (!priv->mtr_en)
 479                 return -rte_mtr_error_set(error, ENOTSUP,
 480                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 481                                           "Meter is not supported");
 482         /* Check input params. */
 483         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 484                                                profile, error);
 485         if (ret)
 486                 return ret;
 487         /* Meter profile memory allocation. */
 488         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 489                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 490         if (fmp == NULL)
 491                 return -rte_mtr_error_set(error, ENOMEM,
 492                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 493                                           NULL, "Meter profile memory "
 494                                           "alloc failed.");
 495         /* Fill profile info. */
 496         fmp->id = meter_profile_id;
 497         fmp->profile = *profile;
 498         /* Fill the flow meter parameters for the PRM. */
 499         ret = mlx5_flow_meter_param_fill(fmp, error);
 500         if (ret)
 501                 goto error;
 502         data.ptr = fmp;
 503         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 504                                  meter_profile_id, &data);
 505         if (ret)
 506                 return -rte_mtr_error_set(error, ENOTSUP,
 507                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 508                                           NULL, "Meter profile insert fail.");
 509         return 0;
 510 error:
 511         mlx5_free(fmp);
 512         return ret;
 513 }
 514
 515 /**
 516  * Callback to delete MTR profile.
 517  *
 518  * @param[in] dev
 519  *   Pointer to Ethernet device.
 520  * @param[in] meter_profile_id
 521  *   Meter profile id.
 522  * @param[out] error
 523  *   Pointer to the error structure.
 524  *
 525  * @return
 526  *   0 on success, a negative errno value otherwise and rte_errno is set.
 527  */
 528 static int
 529 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 530                           uint32_t meter_profile_id,
 531                           struct rte_mtr_error *error)
 532 {
 533         struct mlx5_priv *priv = dev->data->dev_private;
 534         struct mlx5_flow_meter_profile *fmp;
 535
 536         if (!priv->mtr_en)
 537                 return -rte_mtr_error_set(error, ENOTSUP,
 538                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 539                                           "Meter is not supported");
 540         /* Meter profile must exist. */
 541         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 542         if (fmp == NULL)
 543                 return -rte_mtr_error_set(error, ENOENT,
 544                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 545                                           &meter_profile_id,
 546                                           "Meter profile id is invalid.");
 547         /* Check profile is unused. */
 548         if (fmp->ref_cnt)
 549                 return -rte_mtr_error_set(error, EBUSY,
 550                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 551                                           NULL, "Meter profile is in use.");
 552         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 553                 return -rte_mtr_error_set(error, EBUSY,
 554                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 555                                           NULL, "Meter profile remove fail.");
 556         mlx5_free(fmp);
 557         return 0;
 558 }
 559
 560 /**
 561  * Find policy by id.
 562  *
 563  * @param[in] dev
 564  *   Pointer to Ethernet device.
 565  * @param policy_id
 566  *   Policy id.
 567  *
 568  * @return
 569  *   Pointer to the policy found on success, NULL otherwise.
 570  */
 571 struct mlx5_flow_meter_policy *
 572 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 573                             uint32_t policy_id,
 574                             uint32_t *policy_idx)
 575 {
 576         struct mlx5_priv *priv = dev->data->dev_private;
 577         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 578         union mlx5_l3t_data data;
 579
 580         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 581                 return NULL;
 582         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 583                                 !data.dword)
 584                 return NULL;
 585         if (policy_idx)
 586                 *policy_idx = data.dword;
 587         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 588                                         data.dword);
 589         /* Remove reference taken by the mlx5_l3t_get_entry. */
 590         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 591         if (sub_policy)
 592                 if (sub_policy->main_policy_id)
 593                         return sub_policy->main_policy;
 594         return NULL;
 595 }
 596
 597 /**
 598  * Get the last meter's policy from one meter's policy in hierarchy.
 599  *
 600  * @param[in] dev
 601  *   Pointer to Ethernet device.
 602  * @param[in] policy
 603  *   Pointer to flow meter policy.
 604  *
 605  * @return
 606  *   Pointer to the final meter's policy, or NULL when fail.
 607  */
 608 struct mlx5_flow_meter_policy *
 609 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 610                                         struct mlx5_flow_meter_policy *policy)
 611 {
 612         struct mlx5_priv *priv = dev->data->dev_private;
 613         struct mlx5_flow_meter_info *next_fm;
 614         struct mlx5_flow_meter_policy *next_policy = policy;
 615
 616         while (next_policy->is_hierarchy) {
 617                 next_fm = mlx5_flow_meter_find(priv,
 618                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 619                 if (!next_fm || next_fm->def_policy)
 620                         return NULL;
 621                 next_policy = mlx5_flow_meter_policy_find(dev,
 622                                                 next_fm->policy_id, NULL);
 623                 MLX5_ASSERT(next_policy);
 624         }
 625         return next_policy;
 626 }
 627
 628 /**
 629  * Callback to check MTR policy action validate
 630  *
 631  * @param[in] dev
 632  *   Pointer to Ethernet device.
 633  * @param[in] actions
 634  *   Pointer to meter policy action detail.
 635  * @param[out] error
 636  *   Pointer to the error structure.
 637  *
 638  * @return
 639  *   0 on success, a negative errno value otherwise and rte_errno is set.
 640  */
 641 static int
 642 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 643         struct rte_mtr_meter_policy_params *policy,
 644         struct rte_mtr_error *error)
 645 {
 646         struct mlx5_priv *priv = dev->data->dev_private;
 647         struct rte_flow_attr attr = { .transfer =
 648                         priv->config.dv_esw_en ? 1 : 0};
 649         bool is_rss = false;
 650         uint8_t policy_mode;
 651         uint8_t domain_bitmap;
 652         int ret;
 653
 654         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 655                 return -rte_mtr_error_set(error, ENOTSUP,
 656                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 657                                 NULL, "meter policy unsupported.");
 658         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 659                         &is_rss, &domain_bitmap, &policy_mode, error);
 660         if (ret)
 661                 return ret;
 662         return 0;
 663 }
 664
 665 static int
 666 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 667                         uint32_t policy_id,
 668                         struct mlx5_flow_meter_policy *mtr_policy,
 669                         struct rte_mtr_error *error,
 670                         bool clear_l3t)
 671 {
 672         struct mlx5_priv *priv = dev->data->dev_private;
 673         struct mlx5_flow_meter_sub_policy *sub_policy;
 674         uint32_t i, j;
 675         uint16_t sub_policy_num;
 676
 677         rte_spinlock_lock(&mtr_policy->sl);
 678         if (mtr_policy->ref_cnt) {
 679                 rte_spinlock_unlock(&mtr_policy->sl);
 680                 return -rte_mtr_error_set(error, EBUSY,
 681                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 682                                  NULL,
 683                                 "Meter policy object is being used.");
 684         }
 685         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 686         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 687         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 688                 sub_policy_num = (mtr_policy->sub_policy_num >>
 689                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 690                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 691                 if (sub_policy_num) {
 692                         for (j = 0; j < sub_policy_num; j++) {
 693                                 sub_policy = mtr_policy->sub_policys[i][j];
 694                                 if (sub_policy)
 695                                         mlx5_ipool_free
 696                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 697                                         sub_policy->idx);
 698                         }
 699                 }
 700         }
 701         if (priv->policy_idx_tbl && clear_l3t) {
 702                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 703                         rte_spinlock_unlock(&mtr_policy->sl);
 704                         return -rte_mtr_error_set(error, ENOTSUP,
 705                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 706                                 "Fail to delete policy in index table.");
 707                 }
 708         }
 709         rte_spinlock_unlock(&mtr_policy->sl);
 710         return 0;
 711 }
 712
 713 /**
 714  * Callback to add MTR policy.
 715  *
 716  * @param[in] dev
 717  *   Pointer to Ethernet device.
 718  * @param[out] policy_id
 719  *   Pointer to policy id
 720  * @param[in] actions
 721  *   Pointer to meter policy action detail.
 722  * @param[out] error
 723  *   Pointer to the error structure.
 724  *
 725  * @return
 726  *   0 on success, a negative errno value otherwise and rte_errno is set.
 727  */
 728 static int
 729 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 730                         uint32_t policy_id,
 731                         struct rte_mtr_meter_policy_params *policy,
 732                         struct rte_mtr_error *error)
 733 {
 734         struct mlx5_priv *priv = dev->data->dev_private;
 735         struct rte_flow_attr attr = { .transfer =
 736                         priv->config.dv_esw_en ? 1 : 0};
 737         uint32_t sub_policy_idx = 0;
 738         uint32_t policy_idx = 0;
 739         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 740         struct mlx5_flow_meter_sub_policy *sub_policy;
 741         bool is_rss = false;
 742         uint8_t policy_mode;
 743         uint32_t i;
 744         int ret;
 745         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 746         uint16_t sub_policy_num;
 747         uint8_t domain_bitmap = 0;
 748         union mlx5_l3t_data data;
 749         bool skip_rule = false;
 750
 751         if (!priv->mtr_en)
 752                 return -rte_mtr_error_set(error, ENOTSUP,
 753                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 754                                           NULL, "meter policy unsupported. ");
 755         if (policy_id == MLX5_INVALID_POLICY_ID)
 756                 return -rte_mtr_error_set(error, ENOTSUP,
 757                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 758                                           NULL, "policy ID is invalid. ");
 759         if (policy_id == priv->sh->mtrmng->def_policy_id)
 760                 return -rte_mtr_error_set(error, EEXIST,
 761                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 762                                           NULL, "default policy ID exists. ");
 763         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 764         if (mtr_policy)
 765                 return -rte_mtr_error_set(error, EEXIST,
 766                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 767                                           NULL, "policy ID exists. ");
 768         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 769                                           &is_rss, &domain_bitmap,
 770                                           &policy_mode, error);
 771         if (ret)
 772                 return ret;
 773         if (!domain_bitmap)
 774                 return -rte_mtr_error_set(error, ENOTSUP,
 775                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 776                                           NULL, "fail to find policy domain.");
 777         if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
 778                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 779                         return -rte_mtr_error_set(error, EEXIST,
 780                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 781                                 NULL, "a policy with similar actions "
 782                                 "is already configured");
 783                 if (mlx5_flow_create_def_policy(dev))
 784                         return -rte_mtr_error_set(error, ENOTSUP,
 785                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 786                                 NULL,
 787                                 "fail to create non-terminated policy.");
 788                 priv->sh->mtrmng->def_policy_id = policy_id;
 789                 return 0;
 790         }
 791         if (!priv->sh->meter_aso_en)
 792                 return -rte_mtr_error_set(error, ENOTSUP,
 793                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 794                         "no ASO capability to support the policy ");
 795         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 796                 if (!(domain_bitmap & (1 << i)))
 797                         continue;
 798                 /*
 799                  * If RSS is found, it means that only the ingress domain can
 800                  * be supported. It is invalid to support RSS for one color
 801                  * and egress / transfer domain actions for another. Drop and
 802                  * jump action should have no impact.
 803                  */
 804                 if (is_rss) {
 805                         policy_size +=
 806                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 807                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 808                         break;
 809                 }
 810                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 811         }
 812         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 813                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 814         if (!mtr_policy)
 815                 return -rte_mtr_error_set(error, ENOMEM,
 816                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 817                                 "Memory alloc failed for meter policy.");
 818         if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
 819                 mtr_policy->skip_y = 1;
 820         else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
 821                 mtr_policy->skip_g = 1;
 822         policy_size = sizeof(struct mlx5_flow_meter_policy);
 823         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 824                 if (!(domain_bitmap & (1 << i)))
 825                         continue;
 826                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 827                         mtr_policy->ingress = 1;
 828                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 829                         mtr_policy->egress = 1;
 830                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 831                         mtr_policy->transfer = 1;
 832                 sub_policy = mlx5_ipool_zmalloc
 833                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 834                                  &sub_policy_idx);
 835                 if (!sub_policy ||
 836                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 837                         goto policy_add_err;
 838                 sub_policy->idx = sub_policy_idx;
 839                 sub_policy->main_policy = mtr_policy;
 840                 if (!policy_idx) {
 841                         policy_idx = sub_policy_idx;
 842                         sub_policy->main_policy_id = 1;
 843                 }
 844                 mtr_policy->sub_policys[i] =
 845                         (struct mlx5_flow_meter_sub_policy **)
 846                         ((uint8_t *)mtr_policy + policy_size);
 847                 mtr_policy->sub_policys[i][0] = sub_policy;
 848                 sub_policy_num = (mtr_policy->sub_policy_num >>
 849                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 850                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 851                 sub_policy_num++;
 852                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 853                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 854                 mtr_policy->sub_policy_num |=
 855                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 856                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 857                 /*
 858                  * If RSS is found, it means that only the ingress domain can
 859                  * be supported. It is invalid to support RSS for one color
 860                  * and egress / transfer domain actions for another. Drop and
 861                  * jump action should have no impact.
 862                  */
 863                 if (is_rss) {
 864                         mtr_policy->is_rss = 1;
 865                         break;
 866                 }
 867                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 868         }
 869         rte_spinlock_init(&mtr_policy->sl);
 870         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 871                                         policy->actions, error);
 872         if (ret)
 873                 goto policy_add_err;
 874         if (mtr_policy->is_hierarchy) {
 875                 struct mlx5_flow_meter_policy *final_policy;
 876
 877                 final_policy =
 878                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 879                 if (!final_policy)
 880                         goto policy_add_err;
 881                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 882         }
 883         /*
 884          * If either Green or Yellow has queue / RSS action, all the policy
 885          * rules will be created later in the flow splitting stage.
 886          */
 887         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 888                 /* Create policy rules in HW. */
 889                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 890                 if (ret)
 891                         goto policy_add_err;
 892         }
 893         data.dword = policy_idx;
 894         if (!priv->policy_idx_tbl) {
 895                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 896                 if (!priv->policy_idx_tbl)
 897                         goto policy_add_err;
 898         }
 899         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 900                 goto policy_add_err;
 901         return 0;
 902 policy_add_err:
 903         if (mtr_policy) {
 904                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 905                         mtr_policy, error, false);
 906                 mlx5_free(mtr_policy);
 907                 if (ret)
 908                         return ret;
 909         }
 910         return -rte_mtr_error_set(error, ENOTSUP,
 911                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 912                                   NULL, "Failed to create devx policy.");
 913 }
 914
 915 /**
 916  * Callback to delete MTR policy.
 917  *
 918  * @param[in] dev
 919  *   Pointer to Ethernet device.
 920  * @param[in] policy_id
 921  *   Meter policy id.
 922  * @param[out] error
 923  *   Pointer to the error structure.
 924  *
 925  * @return
 926  *   0 on success, a negative errno value otherwise and rte_errno is set.
 927  */
 928 static int
 929 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 930                           uint32_t policy_id,
 931                           struct rte_mtr_error *error)
 932 {
 933         struct mlx5_priv *priv = dev->data->dev_private;
 934         struct mlx5_flow_meter_policy *mtr_policy;
 935         uint32_t policy_idx;
 936         int ret;
 937
 938         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 939                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 940                         return -rte_mtr_error_set(error, ENOTSUP,
 941                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 942                                 "Meter policy object is being used.");
 943                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 944                 return 0;
 945         }
 946         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 947         if (!mtr_policy)
 948                 return -rte_mtr_error_set(error, ENOTSUP,
 949                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 950                         "Meter policy id is invalid. ");
 951         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 952                                                 error, true);
 953         if (ret)
 954                 return ret;
 955         mlx5_free(mtr_policy);
 956         return 0;
 957 }
 958
 959 /**
 960  * Check meter validation.
 961  *
 962  * @param[in] priv
 963  *   Pointer to mlx5 private data structure.
 964  * @param[in] meter_id
 965  *   Meter id.
 966  * @param[in] params
 967  *   Pointer to rte meter parameters.
 968  * @param[out] error
 969  *   Pointer to rte meter error structure.
 970  *
 971  * @return
 972  *   0 on success, a negative errno value otherwise and rte_errno is set.
 973  */
 974 static int
 975 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 976                          struct rte_mtr_params *params,
 977                          struct rte_mtr_error *error)
 978 {
 979         /* Meter must use global drop action. */
 980         if (!priv->sh->dr_drop_action)
 981                 return -rte_mtr_error_set(error, ENOTSUP,
 982                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 983                                           NULL,
 984                                           "No drop action ready for meter.");
 985         /* Meter params must not be NULL. */
 986         if (params == NULL)
 987                 return -rte_mtr_error_set(error, EINVAL,
 988                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 989                                           NULL, "Meter object params null.");
 990         /* Previous meter color is not supported. */
 991         if (params->use_prev_mtr_color)
 992                 return -rte_mtr_error_set(error, ENOTSUP,
 993                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 994                                           NULL,
 995                                           "Previous meter color "
 996                                           "not supported.");
 997         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 998                 return -rte_mtr_error_set(error, ENOENT,
 999                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1000                                 NULL, "Meter policy id not valid.");
1001         /* Validate meter id. */
1002         if (mlx5_flow_meter_find(priv, meter_id, NULL))
1003                 return -rte_mtr_error_set(error, EEXIST,
1004                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
1005                         "Meter object already exists.");
1006         return 0;
1007 }
1008
1009 /**
1010  * Modify the flow meter action.
1011  *
1012  * @param[in] priv
1013  *   Pointer to mlx5 private data structure.
1014  * @param[in] fm
1015  *   Pointer to flow meter to be modified.
1016  * @param[in] srtcm
1017  *   Pointer to meter srtcm description parameter.
1018  * @param[in] modify_bits
1019  *   The bit in srtcm to be updated.
1020  * @param[in] active_state
1021  *   The state to be updated.
1022  * @return
1023  *   0 on success, o negative value otherwise.
1024  */
1025 static int
1026 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
1027                 struct mlx5_flow_meter_info *fm,
1028                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
1029                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
1030 {
1031 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1032         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
1033         uint32_t *attr;
1034         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
1035         int ret;
1036         struct mlx5_aso_mtr *aso_mtr = NULL;
1037         uint32_t cbs_cir, ebs_eir, val;
1038
1039         if (priv->sh->meter_aso_en) {
1040                 fm->is_enable = !!is_enable;
1041                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1042                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1043                 if (ret)
1044                         return ret;
1045                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
1046                 if (ret)
1047                         return ret;
1048         } else {
1049                 /* Fill command parameters. */
1050                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
1051                 mod_attr.flow_meter_parameter = in;
1052                 mod_attr.flow_meter_parameter_sz =
1053                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
1054                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1055                         mod_attr.active = !!active_state;
1056                 else
1057                         mod_attr.active = 0;
1058                 attr = in;
1059                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
1060                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1061                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1062                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1063                                 ASO_DSEG_EXP_MASK;
1064                         MLX5_SET(flow_meter_parameters, attr,
1065                                 cbs_exponent, val);
1066                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1067                                 ASO_DSEG_MAN_MASK;
1068                         MLX5_SET(flow_meter_parameters, attr,
1069                                 cbs_mantissa, val);
1070                 }
1071                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1072                         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) &
1073                                 ASO_DSEG_EXP_MASK;
1074                         MLX5_SET(flow_meter_parameters, attr,
1075                                 cir_exponent, val);
1076                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1077                         MLX5_SET(flow_meter_parameters, attr,
1078                                 cir_mantissa, val);
1079                 }
1080                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1081                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1082                                 ASO_DSEG_EXP_MASK;
1083                         MLX5_SET(flow_meter_parameters, attr,
1084                                 ebs_exponent, val);
1085                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1086                                 ASO_DSEG_MAN_MASK;
1087                         MLX5_SET(flow_meter_parameters, attr,
1088                                 ebs_mantissa, val);
1089                 }
1090                 /* Apply modifications to meter only if it was created. */
1091                 if (fm->meter_action) {
1092                         ret = mlx5_glue->dv_modify_flow_action_meter
1093                                         (fm->meter_action, &mod_attr,
1094                                         rte_cpu_to_be_64(modify_bits));
1095                         if (ret)
1096                                 return ret;
1097                 }
1098                 /* Update succeedded modify meter parameters. */
1099                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1100                         fm->active_state = !!active_state;
1101         }
1102         return 0;
1103 #else
1104         (void)priv;
1105         (void)fm;
1106         (void)srtcm;
1107         (void)modify_bits;
1108         (void)active_state;
1109         (void)is_enable;
1110         return -ENOTSUP;
1111 #endif
1112 }
1113
1114 static int
1115 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1116                                 struct mlx5_flow_meter_info *fm,
1117                                 uint64_t stats_mask)
1118 {
1119         fm->bytes_dropped =
1120                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1121         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1122         if (fm->bytes_dropped || fm->pkts_dropped) {
1123                 if (!fm->drop_cnt) {
1124                         /* Alloc policer counters. */
1125                         fm->drop_cnt = mlx5_counter_alloc(dev);
1126                         if (!fm->drop_cnt)
1127                                 return -1;
1128                 }
1129         } else {
1130                 if (fm->drop_cnt) {
1131                         mlx5_counter_free(dev, fm->drop_cnt);
1132                         fm->drop_cnt = 0;
1133                 }
1134         }
1135         return 0;
1136 }
1137
1138 /**
1139  * Create meter rules.
1140  *
1141  * @param[in] dev
1142  *   Pointer to Ethernet device.
1143  * @param[in] meter_id
1144  *   Meter id.
1145  * @param[in] params
1146  *   Pointer to rte meter parameters.
1147  * @param[in] shared
1148  *   Meter shared with other flow or not.
1149  * @param[out] error
1150  *   Pointer to rte meter error structure.
1151  *
1152  * @return
1153  *   0 on success, a negative errno value otherwise and rte_errno is set.
1154  */
1155 static int
1156 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1157                        struct rte_mtr_params *params, int shared,
1158                        struct rte_mtr_error *error)
1159 {
1160         struct mlx5_priv *priv = dev->data->dev_private;
1161         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1162         struct mlx5_flow_meter_profile *fmp;
1163         struct mlx5_flow_meter_info *fm;
1164         struct mlx5_legacy_flow_meter *legacy_fm;
1165         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1166         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1167                 .size = 0,
1168                 .trunk_size = 64,
1169                 .need_lock = 1,
1170                 .type = "mlx5_flow_mtr_flow_id_pool",
1171         };
1172         struct mlx5_aso_mtr *aso_mtr;
1173         uint32_t mtr_idx, policy_idx;
1174         union mlx5_l3t_data data;
1175         int ret;
1176         uint8_t domain_bitmap;
1177         uint8_t mtr_id_bits;
1178         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1179                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1180
1181         if (!priv->mtr_en)
1182                 return -rte_mtr_error_set(error, ENOTSUP,
1183                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1184                                         "Meter is not supported");
1185         /* Validate the parameters. */
1186         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1187         if (ret)
1188                 return ret;
1189         /* Meter profile must exist. */
1190         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1191         if (fmp == NULL)
1192                 return -rte_mtr_error_set(error, ENOENT,
1193                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1194                         NULL, "Meter profile id not valid.");
1195         /* Meter policy must exist. */
1196         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1197                 __atomic_add_fetch
1198                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1199                         1, __ATOMIC_RELAXED);
1200                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1201                 if (!priv->config.dv_esw_en)
1202                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1203         } else {
1204                 if (!priv->sh->meter_aso_en)
1205                         return -rte_mtr_error_set(error, ENOTSUP,
1206                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1207                                 "Part of the policies cannot be "
1208                                 "supported without ASO ");
1209                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1210                                 params->meter_policy_id, &policy_idx);
1211                 if (!mtr_policy)
1212                         return -rte_mtr_error_set(error, ENOENT,
1213                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1214                                 NULL, "Meter policy id not valid.");
1215                 domain_bitmap = (mtr_policy->ingress ?
1216                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1217                                 (mtr_policy->egress ?
1218                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1219                                 (mtr_policy->transfer ?
1220                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1221                 if (fmp->g_support && mtr_policy->skip_g)
1222                         return -rte_mtr_error_set(error, ENOTSUP,
1223                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1224                                         NULL, "Meter green policy is empty.");
1225                 if (fmp->y_support && mtr_policy->skip_y)
1226                         return -rte_mtr_error_set(error, ENOTSUP,
1227                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1228                                         NULL, "Meter yellow policy is empty.");
1229         }
1230         /* Allocate the flow meter memory. */
1231         if (priv->sh->meter_aso_en) {
1232                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1233                 if (!mtr_idx)
1234                         return -rte_mtr_error_set(error, ENOMEM,
1235                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1236                                 "Memory alloc failed for meter.");
1237                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1238                 fm = &aso_mtr->fm;
1239         } else {
1240                 if (fmp->y_support)
1241                         return -rte_mtr_error_set(error, ENOMEM,
1242                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1243                                 "Unsupported profile with yellow.");
1244                 legacy_fm = mlx5_ipool_zmalloc
1245                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1246                 if (legacy_fm == NULL)
1247                         return -rte_mtr_error_set(error, ENOMEM,
1248                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1249                                 "Memory alloc failed for meter.");
1250                 legacy_fm->idx = mtr_idx;
1251                 fm = &legacy_fm->fm;
1252         }
1253         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1254         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1255             mtr_reg_bits) {
1256                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1257                 goto error;
1258         }
1259         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1260                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1261         /* Fill the flow meter parameters. */
1262         fm->meter_id = meter_id;
1263         fm->policy_id = params->meter_policy_id;
1264         fm->profile = fmp;
1265         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1266                 goto error;
1267         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1268                 goto error;
1269         /* Add to the flow meter list. */
1270         if (!priv->sh->meter_aso_en)
1271                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1272         /* Add to the flow meter list. */
1273         fm->active_state = 1; /* Config meter starts as active. */
1274         fm->is_enable = 1;
1275         fm->shared = !!shared;
1276         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1277         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1278                 fm->def_policy = 1;
1279                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1280                 if (!fm->flow_ipool)
1281                         goto error;
1282         }
1283         rte_spinlock_init(&fm->sl);
1284         /* If ASO meter supported, update ASO flow meter by wqe. */
1285         if (priv->sh->meter_aso_en) {
1286                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1287                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1288                 if (ret)
1289                         goto error;
1290                 if (!priv->mtr_idx_tbl) {
1291                         priv->mtr_idx_tbl =
1292                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1293                         if (!priv->mtr_idx_tbl)
1294                                 goto error;
1295                 }
1296                 data.dword = mtr_idx;
1297                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1298                         goto error;
1299         }
1300         if (mtr_policy)
1301                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1302         return 0;
1303 error:
1304         mlx5_flow_destroy_mtr_tbls(dev, fm);
1305         /* Free policer counters. */
1306         if (fm->drop_cnt)
1307                 mlx5_counter_free(dev, fm->drop_cnt);
1308         if (priv->sh->meter_aso_en)
1309                 mlx5_flow_mtr_free(dev, mtr_idx);
1310         else
1311                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1312         return -rte_mtr_error_set(error, ENOTSUP,
1313                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1314                 NULL, "Failed to create devx meter.");
1315 }
1316
1317 static int
1318 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1319                         struct mlx5_flow_meter_info *fm,
1320                         uint32_t mtr_idx)
1321 {
1322         struct mlx5_priv *priv = dev->data->dev_private;
1323         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1324         struct mlx5_flow_meter_profile *fmp;
1325         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1326         struct mlx5_flow_meter_policy *mtr_policy;
1327
1328         /* Meter object must not have any owner. */
1329         MLX5_ASSERT(!fm->ref_cnt);
1330         /* Get meter profile. */
1331         fmp = fm->profile;
1332         if (fmp == NULL)
1333                 return -1;
1334         /* Update dependencies. */
1335         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1336         fm->profile = NULL;
1337         /* Remove from list. */
1338         if (!priv->sh->meter_aso_en) {
1339                 legacy_fm = container_of(fm,
1340                         struct mlx5_legacy_flow_meter, fm);
1341                 TAILQ_REMOVE(fms, legacy_fm, next);
1342         }
1343         /* Free drop counters. */
1344         if (fm->drop_cnt)
1345                 mlx5_counter_free(dev, fm->drop_cnt);
1346         /* Free meter flow table. */
1347         if (fm->flow_ipool) {
1348                 mlx5_ipool_destroy(fm->flow_ipool);
1349                 fm->flow_ipool = 0;
1350         }
1351         mlx5_flow_destroy_mtr_tbls(dev, fm);
1352         if (fm->def_policy)
1353                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1354                                 1, __ATOMIC_RELAXED);
1355         if (priv->sh->meter_aso_en) {
1356                 if (!fm->def_policy) {
1357                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1358                                                 fm->policy_id, NULL);
1359                         if (mtr_policy)
1360                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1361                                                 1, __ATOMIC_RELAXED);
1362                         fm->policy_id = 0;
1363                 }
1364                 fm->def_policy = 0;
1365                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1366                         return -1;
1367                 mlx5_flow_mtr_free(dev, mtr_idx);
1368         } else {
1369                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1370                                         legacy_fm->idx);
1371         }
1372         return 0;
1373 }
1374
1375 /**
1376  * Destroy meter rules.
1377  *
1378  * @param[in] dev
1379  *   Pointer to Ethernet device.
1380  * @param[in] meter_id
1381  *   Meter id.
1382  * @param[out] error
1383  *   Pointer to rte meter error structure.
1384  *
1385  * @return
1386  *   0 on success, a negative errno value otherwise and rte_errno is set.
1387  */
1388 static int
1389 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1390                         struct rte_mtr_error *error)
1391 {
1392         struct mlx5_priv *priv = dev->data->dev_private;
1393         struct mlx5_flow_meter_info *fm;
1394         uint32_t mtr_idx = 0;
1395
1396         if (!priv->mtr_en)
1397                 return -rte_mtr_error_set(error, ENOTSUP,
1398                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1399                                           NULL,
1400                                           "Meter is not supported");
1401         /* Meter object must exist. */
1402         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1403         if (fm == NULL)
1404                 return -rte_mtr_error_set(error, ENOENT,
1405                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1406                                           NULL,
1407                                           "Meter object id not valid.");
1408         /* Meter object must not have any owner. */
1409         if (fm->ref_cnt > 0)
1410                 return -rte_mtr_error_set(error, EBUSY,
1411                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1412                                           NULL,
1413                                           "Meter object is being used.");
1414         /* Destroy the meter profile. */
1415         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1416                 return -rte_mtr_error_set(error, EINVAL,
1417                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1418                                         NULL,
1419                                         "MTR object meter profile invalid.");
1420         return 0;
1421 }
1422
1423 /**
1424  * Modify meter state.
1425  *
1426  * @param[in] priv
1427  *   Pointer to mlx5 private data structure.
1428  * @param[in] fm
1429  *   Pointer to flow meter.
1430  * @param[in] new_state
1431  *   New state to update.
1432  * @param[out] error
1433  *   Pointer to rte meter error structure.
1434  *
1435  * @return
1436  *   0 on success, a negative errno value otherwise and rte_errno is set.
1437  */
1438 static int
1439 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1440                              struct mlx5_flow_meter_info *fm,
1441                              uint32_t new_state,
1442                              struct rte_mtr_error *error)
1443 {
1444         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1445                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1446                 .ebs_eir = 0,
1447         };
1448         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1449                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1450         int ret;
1451
1452         if (new_state == MLX5_FLOW_METER_DISABLE)
1453                 ret = mlx5_flow_meter_action_modify(priv, fm,
1454                                 &srtcm, modify_bits, 0, 0);
1455         else
1456                 ret = mlx5_flow_meter_action_modify(priv, fm,
1457                                                     &fm->profile->srtcm_prm,
1458                                                     modify_bits, 0, 1);
1459         if (ret)
1460                 return -rte_mtr_error_set(error, -ret,
1461                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1462                                           NULL,
1463                                           new_state ?
1464                                           "Failed to enable meter." :
1465                                           "Failed to disable meter.");
1466         return 0;
1467 }
1468
1469 /**
1470  * Callback to enable flow meter.
1471  *
1472  * @param[in] dev
1473  *   Pointer to Ethernet device.
1474  * @param[in] meter_id
1475  *   Meter id.
1476  * @param[out] error
1477  *   Pointer to rte meter error structure.
1478  *
1479  * @return
1480  *   0 on success, a negative errno value otherwise and rte_errno is set.
1481  */
1482 static int
1483 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1484                        uint32_t meter_id,
1485                        struct rte_mtr_error *error)
1486 {
1487         struct mlx5_priv *priv = dev->data->dev_private;
1488         struct mlx5_flow_meter_info *fm;
1489         int ret;
1490
1491         if (!priv->mtr_en)
1492                 return -rte_mtr_error_set(error, ENOTSUP,
1493                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1494                                           "Meter is not supported");
1495         /* Meter object must exist. */
1496         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1497         if (fm == NULL)
1498                 return -rte_mtr_error_set(error, ENOENT,
1499                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1500                                           NULL, "Meter not found.");
1501         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1502                 return 0;
1503         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1504                                            error);
1505         if (!ret)
1506                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1507         return ret;
1508 }
1509
1510 /**
1511  * Callback to disable flow meter.
1512  *
1513  * @param[in] dev
1514  *   Pointer to Ethernet device.
1515  * @param[in] meter_id
1516  *   Meter id.
1517  * @param[out] error
1518  *   Pointer to rte meter error structure.
1519  *
1520  * @return
1521  *   0 on success, a negative errno value otherwise and rte_errno is set.
1522  */
1523 static int
1524 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1525                         uint32_t meter_id,
1526                         struct rte_mtr_error *error)
1527 {
1528         struct mlx5_priv *priv = dev->data->dev_private;
1529         struct mlx5_flow_meter_info *fm;
1530         int ret;
1531
1532         if (!priv->mtr_en)
1533                 return -rte_mtr_error_set(error, ENOTSUP,
1534                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1535                                           "Meter is not supported");
1536         /* Meter object must exist. */
1537         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1538         if (fm == NULL)
1539                 return -rte_mtr_error_set(error, ENOENT,
1540                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1541                                           NULL, "Meter not found.");
1542         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1543                 return 0;
1544         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1545                                            error);
1546         if (!ret)
1547                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1548         return ret;
1549 }
1550
1551 /**
1552  * Callback to update meter profile.
1553  *
1554  * @param[in] dev
1555  *   Pointer to Ethernet device.
1556  * @param[in] meter_id
1557  *   Meter id.
1558  * @param[in] meter_profile_id
1559  *   To be updated meter profile id.
1560  * @param[out] error
1561  *   Pointer to rte meter error structure.
1562  *
1563  * @return
1564  *   0 on success, a negative errno value otherwise and rte_errno is set.
1565  */
1566 static int
1567 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1568                                uint32_t meter_id,
1569                                uint32_t meter_profile_id,
1570                                struct rte_mtr_error *error)
1571 {
1572         struct mlx5_priv *priv = dev->data->dev_private;
1573         struct mlx5_flow_meter_profile *fmp;
1574         struct mlx5_flow_meter_profile *old_fmp;
1575         struct mlx5_flow_meter_info *fm;
1576         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1577                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1578         int ret;
1579
1580         if (!priv->mtr_en)
1581                 return -rte_mtr_error_set(error, ENOTSUP,
1582                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1583                                           "Meter is not supported");
1584         /* Meter profile must exist. */
1585         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1586         if (fmp == NULL)
1587                 return -rte_mtr_error_set(error, ENOENT,
1588                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1589                                           NULL, "Meter profile not found.");
1590         /* Meter object must exist. */
1591         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1592         if (fm == NULL)
1593                 return -rte_mtr_error_set(error, ENOENT,
1594                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1595                                           NULL, "Meter not found.");
1596         /* MTR object already set to meter profile id. */
1597         old_fmp = fm->profile;
1598         if (fmp == old_fmp)
1599                 return 0;
1600         /* Update the profile. */
1601         fm->profile = fmp;
1602         /* Update meter params in HW (if not disabled). */
1603         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1604                 return 0;
1605         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1606                                               modify_bits, fm->active_state, 1);
1607         if (ret) {
1608                 fm->profile = old_fmp;
1609                 return -rte_mtr_error_set(error, -ret,
1610                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1611                                           NULL, "Failed to update meter"
1612                                           " parmeters in hardware.");
1613         }
1614         old_fmp->ref_cnt--;
1615         fmp->ref_cnt++;
1616         return 0;
1617 }
1618
1619 /**
1620  * Callback to update meter stats mask.
1621  *
1622  * @param[in] dev
1623  *   Pointer to Ethernet device.
1624  * @param[in] meter_id
1625  *   Meter id.
1626  * @param[in] stats_mask
1627  *   To be updated stats_mask.
1628  * @param[out] error
1629  *   Pointer to rte meter error structure.
1630  *
1631  * @return
1632  *   0 on success, a negative errno value otherwise and rte_errno is set.
1633  */
1634 static int
1635 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1636                              uint32_t meter_id,
1637                              uint64_t stats_mask,
1638                              struct rte_mtr_error *error)
1639 {
1640         struct mlx5_priv *priv = dev->data->dev_private;
1641         struct mlx5_flow_meter_info *fm;
1642
1643         if (!priv->mtr_en)
1644                 return -rte_mtr_error_set(error, ENOTSUP,
1645                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1646                                           "Meter is not supported");
1647         /* Meter object must exist. */
1648         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1649         if (fm == NULL)
1650                 return -rte_mtr_error_set(error, ENOENT,
1651                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1652                                           NULL, "Meter object id not valid.");
1653         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1654                 return -rte_mtr_error_set(error, ENOENT,
1655                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1656                                           NULL, "Fail to allocate "
1657                                           "counter for meter.");
1658         return 0;
1659 }
1660
1661 /**
1662  * Callback to read meter statistics.
1663  *
1664  * @param[in] dev
1665  *   Pointer to Ethernet device.
1666  * @param[in] meter_id
1667  *   Meter id.
1668  * @param[out] stats
1669  *   Pointer to store the statistics.
1670  * @param[out] stats_mask
1671  *   Pointer to store the stats_mask.
1672  * @param[in] clear
1673  *   Statistic to be cleared after read or not.
1674  * @param[out] error
1675  *   Pointer to rte meter error structure.
1676  *
1677  * @return
1678  *   0 on success, a negative errno value otherwise and rte_errno is set.
1679  */
1680 static int
1681 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1682                            uint32_t meter_id,
1683                            struct rte_mtr_stats *stats,
1684                            uint64_t *stats_mask,
1685                            int clear,
1686                            struct rte_mtr_error *error)
1687 {
1688         struct mlx5_priv *priv = dev->data->dev_private;
1689         struct mlx5_flow_meter_info *fm;
1690         uint64_t pkts;
1691         uint64_t bytes;
1692         int ret = 0;
1693
1694         if (!priv->mtr_en)
1695                 return -rte_mtr_error_set(error, ENOTSUP,
1696                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1697                                           "Meter is not supported");
1698         /* Meter object must exist. */
1699         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1700         if (fm == NULL)
1701                 return -rte_mtr_error_set(error, ENOENT,
1702                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1703                                           NULL, "Meter object id not valid.");
1704         *stats_mask = 0;
1705         if (fm->bytes_dropped)
1706                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1707         if (fm->pkts_dropped)
1708                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1709         memset(stats, 0, sizeof(*stats));
1710         if (fm->drop_cnt) {
1711                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1712                                                  &bytes);
1713                 if (ret)
1714                         goto error;
1715                 /* If need to read the packets, set it. */
1716                 if (fm->pkts_dropped)
1717                         stats->n_pkts_dropped = pkts;
1718                 /* If need to read the bytes, set it. */
1719                 if (fm->bytes_dropped)
1720                         stats->n_bytes_dropped = bytes;
1721         }
1722         return 0;
1723 error:
1724         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1725                                  "Failed to read meter drop counters.");
1726 }
1727
1728 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1729         .capabilities_get = mlx5_flow_mtr_cap_get,
1730         .meter_profile_add = mlx5_flow_meter_profile_add,
1731         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1732         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1733         .meter_policy_add = mlx5_flow_meter_policy_add,
1734         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1735         .create = mlx5_flow_meter_create,
1736         .destroy = mlx5_flow_meter_destroy,
1737         .meter_enable = mlx5_flow_meter_enable,
1738         .meter_disable = mlx5_flow_meter_disable,
1739         .meter_profile_update = mlx5_flow_meter_profile_update,
1740         .meter_dscp_table_update = NULL,
1741         .stats_update = mlx5_flow_meter_stats_update,
1742         .stats_read = mlx5_flow_meter_stats_read,
1743 };
1744
1745 /**
1746  * Get meter operations.
1747  *
1748  * @param dev
1749  *   Pointer to Ethernet device structure.
1750  * @param arg
1751  *   Pointer to set the mtr operations.
1752  *
1753  * @return
1754  *   Always 0.
1755  */
1756 int
1757 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1758 {
1759         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1760         return 0;
1761 }
1762
1763 /**
1764  * Find meter by id.
1765  *
1766  * @param priv
1767  *   Pointer to mlx5_priv.
1768  * @param meter_id
1769  *   Meter id.
1770  * @param mtr_idx
1771  *   Pointer to Meter index.
1772  *
1773  * @return
1774  *   Pointer to the meter info found on success, NULL otherwise.
1775  */
1776 struct mlx5_flow_meter_info *
1777 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1778                 uint32_t *mtr_idx)
1779 {
1780         struct mlx5_legacy_flow_meter *legacy_fm;
1781         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1782         struct mlx5_aso_mtr *aso_mtr;
1783         struct mlx5_aso_mtr_pools_mng *pools_mng =
1784                                 &priv->sh->mtrmng->pools_mng;
1785         union mlx5_l3t_data data;
1786
1787         if (priv->sh->meter_aso_en) {
1788                 rte_spinlock_lock(&pools_mng->mtrsl);
1789                 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1790                         rte_spinlock_unlock(&pools_mng->mtrsl);
1791                         return NULL;
1792                 }
1793                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1794                         !data.dword) {
1795                         rte_spinlock_unlock(&pools_mng->mtrsl);
1796                         return NULL;
1797                 }
1798                 if (mtr_idx)
1799                         *mtr_idx = data.dword;
1800                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1801                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1802                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1803                 rte_spinlock_unlock(&pools_mng->mtrsl);
1804                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1805                         return NULL;
1806                 return &aso_mtr->fm;
1807         }
1808         TAILQ_FOREACH(legacy_fm, fms, next)
1809                 if (meter_id == legacy_fm->fm.meter_id) {
1810                         if (mtr_idx)
1811                                 *mtr_idx = legacy_fm->idx;
1812                         return &legacy_fm->fm;
1813                 }
1814         return NULL;
1815 }
1816
1817 /**
1818  * Find meter by index.
1819  *
1820  * @param priv
1821  *   Pointer to mlx5_priv.
1822  * @param idx
1823  *   Meter index.
1824  *
1825  * @return
1826  *   Pointer to the meter info found on success, NULL otherwise.
1827  */
1828 struct mlx5_flow_meter_info *
1829 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1830 {
1831         struct mlx5_aso_mtr *aso_mtr;
1832
1833         if (priv->sh->meter_aso_en) {
1834                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1835                 if (!aso_mtr)
1836                         return NULL;
1837                 return &aso_mtr->fm;
1838         } else {
1839                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1840         }
1841 }
1842
1843 /**
1844  * Attach meter to flow.
1845  * Unidirectional Meter creation can only be done
1846  * when flow direction is known, i.e. when calling meter_attach.
1847  *
1848  * @param [in] priv
1849  *  Pointer to mlx5 private data.
1850  * @param[in] fm
1851  *   Pointer to flow meter.
1852  * @param [in] attr
1853  *  Pointer to flow attributes.
1854  * @param [out] error
1855  *  Pointer to error structure.
1856  *
1857  * @return
1858  *   0 on success, a negative errno value otherwise and rte_errno is set.
1859  */
1860 int
1861 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1862                        struct mlx5_flow_meter_info *fm,
1863                        const struct rte_flow_attr *attr,
1864                        struct rte_flow_error *error)
1865 {
1866         int ret = 0;
1867
1868         if (priv->sh->meter_aso_en) {
1869                 struct mlx5_aso_mtr *aso_mtr;
1870
1871                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1872                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1873                         return rte_flow_error_set(error, ENOENT,
1874                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1875                                         NULL,
1876                                         "Timeout in meter configuration");
1877                 }
1878                 rte_spinlock_lock(&fm->sl);
1879                 if (fm->shared || !fm->ref_cnt) {
1880                         fm->ref_cnt++;
1881                 } else {
1882                         rte_flow_error_set(error, EINVAL,
1883                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1884                                    "Meter cannot be shared");
1885                         ret = -1;
1886                 }
1887                 rte_spinlock_unlock(&fm->sl);
1888         } else {
1889                 rte_spinlock_lock(&fm->sl);
1890                 if (fm->meter_action) {
1891                         if (fm->shared &&
1892                             attr->transfer == fm->transfer &&
1893                             attr->ingress == fm->ingress &&
1894                             attr->egress == fm->egress) {
1895                                 fm->ref_cnt++;
1896                         } else {
1897                                 rte_flow_error_set(error, EINVAL,
1898                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1899                                         fm->shared ?
1900                                         "Meter attr not match." :
1901                                         "Meter cannot be shared.");
1902                                 ret = -1;
1903                         }
1904                 } else {
1905                         fm->ingress = attr->ingress;
1906                         fm->egress = attr->egress;
1907                         fm->transfer = attr->transfer;
1908                         fm->ref_cnt = 1;
1909                         /* This also creates the meter object. */
1910                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1911                                                                          fm);
1912                         if (!fm->meter_action) {
1913                                 fm->ref_cnt = 0;
1914                                 fm->ingress = 0;
1915                                 fm->egress = 0;
1916                                 fm->transfer = 0;
1917                                 rte_flow_error_set(error, EINVAL,
1918                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1919                                         "Meter action create failed.");
1920                                 ret = -1;
1921                         }
1922                 }
1923                 rte_spinlock_unlock(&fm->sl);
1924         }
1925         return ret ? -rte_errno : 0;
1926 }
1927
1928 /**
1929  * Detach meter from flow.
1930  *
1931  * @param [in] priv
1932  *  Pointer to mlx5 private data.
1933  * @param [in] fm
1934  *  Pointer to flow meter.
1935  */
1936 void
1937 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1938                        struct mlx5_flow_meter_info *fm)
1939 {
1940 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1941         rte_spinlock_lock(&fm->sl);
1942         MLX5_ASSERT(fm->ref_cnt);
1943         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1944                 mlx5_glue->destroy_flow_action(fm->meter_action);
1945                 fm->meter_action = NULL;
1946                 fm->ingress = 0;
1947                 fm->egress = 0;
1948                 fm->transfer = 0;
1949         }
1950         rte_spinlock_unlock(&fm->sl);
1951 #else
1952         (void)priv;
1953         (void)fm;
1954 #endif
1955 }
1956
1957 /**
1958  * Flush meter with Rx queue configuration.
1959  *
1960  * @param[in] dev
1961  *   Pointer to Ethernet device.
1962  */
1963 void
1964 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1965 {
1966         struct mlx5_priv *priv = dev->data->dev_private;
1967         struct mlx5_flow_meter_sub_policy *sub_policy;
1968         struct mlx5_flow_meter_policy *mtr_policy;
1969         void *entry;
1970         uint32_t i, policy_idx;
1971
1972         if (!priv->mtr_en)
1973                 return;
1974         if (priv->policy_idx_tbl) {
1975                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1976                         policy_idx = *(uint32_t *)entry;
1977                         sub_policy = mlx5_ipool_get
1978                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1979                                 policy_idx);
1980                         if (!sub_policy || !sub_policy->main_policy)
1981                                 continue;
1982                         mtr_policy = sub_policy->main_policy;
1983                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1984                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1985                                         mtr_policy);
1986                 }
1987         }
1988 }
1989
1990 /**
1991  * Iterate a meter hierarchy and flush all meters and policies if possible.
1992  *
1993  * @param[in] dev
1994  *   Pointer to Ethernet device.
1995  * @param[in] fm
1996  *   Pointer to flow meter.
1997  * @param[in] mtr_idx
1998  *   .Meter's index
1999  * @param[out] error
2000  *   Pointer to rte meter error structure.
2001  *
2002  * @return
2003  *   0 on success, a negative errno value otherwise and rte_errno is set.
2004  */
2005 static int
2006 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
2007                                 struct mlx5_flow_meter_info *fm,
2008                                 uint32_t mtr_idx,
2009                                 struct rte_mtr_error *error)
2010 {
2011         struct mlx5_priv *priv = dev->data->dev_private;
2012         struct mlx5_flow_meter_policy *policy;
2013         uint32_t policy_id;
2014         struct mlx5_flow_meter_info *next_fm;
2015         uint32_t next_mtr_idx;
2016         struct mlx5_flow_meter_policy *next_policy = NULL;
2017
2018         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
2019         MLX5_ASSERT(policy);
2020         while (!fm->ref_cnt && policy->is_hierarchy) {
2021                 policy_id = fm->policy_id;
2022                 next_fm = mlx5_flow_meter_find(priv,
2023                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2024                                 &next_mtr_idx);
2025                 if (next_fm) {
2026                         next_policy = mlx5_flow_meter_policy_find(dev,
2027                                                         next_fm->policy_id,
2028                                                         NULL);
2029                         MLX5_ASSERT(next_policy);
2030                 }
2031                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
2032                         return -rte_mtr_error_set(error, ENOTSUP,
2033                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
2034                                                 NULL,
2035                                                 "Failed to flush meter.");
2036                 if (policy->ref_cnt)
2037                         break;
2038                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
2039                                                 policy, error, true))
2040                         return -rte_errno;
2041                 mlx5_free(policy);
2042                 if (!next_fm || !next_policy)
2043                         break;
2044                 fm = next_fm;
2045                 mtr_idx = next_mtr_idx;
2046                 policy = next_policy;
2047         }
2048         return 0;
2049 }
2050
2051 /**
2052  * Flush all the hierarchy meters and their policies.
2053  *
2054  * @param[in] dev
2055  *   Pointer to Ethernet device.
2056  * @param[out] error
2057  *   Pointer to rte meter error structure.
2058  *
2059  * @return
2060  *   0 on success, a negative errno value otherwise and rte_errno is set.
2061  */
2062 static int
2063 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
2064                                       struct rte_mtr_error *error)
2065 {
2066         struct mlx5_priv *priv = dev->data->dev_private;
2067         struct mlx5_flow_meter_info *fm;
2068         struct mlx5_flow_meter_policy *policy;
2069         struct mlx5_flow_meter_sub_policy *sub_policy;
2070         struct mlx5_flow_meter_info *next_fm;
2071         struct mlx5_aso_mtr *aso_mtr;
2072         uint32_t mtr_idx = 0;
2073         uint32_t i, policy_idx;
2074         void *entry;
2075
2076         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2077                 return 0;
2078         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2079                 mtr_idx = *(uint32_t *)entry;
2080                 if (!mtr_idx)
2081                         continue;
2082                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2083                 fm = &aso_mtr->fm;
2084                 if (fm->ref_cnt || fm->def_policy)
2085                         continue;
2086                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2087                         return -rte_errno;
2088         }
2089         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2090                 policy_idx = *(uint32_t *)entry;
2091                 sub_policy = mlx5_ipool_get
2092                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2093                                 policy_idx);
2094                 if (!sub_policy)
2095                         return -rte_mtr_error_set(error,
2096                                         EINVAL,
2097                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2098                                         NULL, "Meter policy invalid.");
2099                 policy = sub_policy->main_policy;
2100                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2101                         continue;
2102                 next_fm = mlx5_flow_meter_find(priv,
2103                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2104                                 &mtr_idx);
2105                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2106                                                     error, true))
2107                         return -rte_mtr_error_set(error,
2108                                         EINVAL,
2109                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2110                                         NULL, "Meter policy invalid.");
2111                 mlx5_free(policy);
2112                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2113                         continue;
2114                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2115                                                     mtr_idx, error))
2116                         return -rte_errno;
2117         }
2118         return 0;
2119 }
2120 /**
2121  * Flush meter configuration.
2122  *
2123  * @param[in] dev
2124  *   Pointer to Ethernet device.
2125  * @param[out] error
2126  *   Pointer to rte meter error structure.
2127  *
2128  * @return
2129  *   0 on success, a negative errno value otherwise and rte_errno is set.
2130  */
2131 int
2132 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2133 {
2134         struct mlx5_priv *priv = dev->data->dev_private;
2135         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2136         struct mlx5_flow_meter_profile *fmp;
2137         struct mlx5_legacy_flow_meter *legacy_fm;
2138         struct mlx5_flow_meter_info *fm;
2139         struct mlx5_flow_meter_sub_policy *sub_policy;
2140         void *tmp;
2141         uint32_t i, mtr_idx, policy_idx;
2142         void *entry;
2143         struct mlx5_aso_mtr *aso_mtr;
2144
2145         if (!priv->mtr_en)
2146                 return 0;
2147         if (priv->sh->meter_aso_en) {
2148                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2149                         return -rte_errno;
2150                 if (priv->mtr_idx_tbl) {
2151                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2152                                 mtr_idx = *(uint32_t *)entry;
2153                                 if (mtr_idx) {
2154                                         aso_mtr =
2155                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2156                                         fm = &aso_mtr->fm;
2157                                         (void)mlx5_flow_meter_params_flush(dev,
2158                                                 fm, mtr_idx);
2159                                 }
2160                         }
2161                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2162                         priv->mtr_idx_tbl = NULL;
2163                 }
2164         } else {
2165                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2166                         fm = &legacy_fm->fm;
2167                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2168                                 return -rte_mtr_error_set(error, EINVAL,
2169                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2170                                 NULL, "MTR object meter profile invalid.");
2171                 }
2172         }
2173         if (priv->policy_idx_tbl) {
2174                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2175                         policy_idx = *(uint32_t *)entry;
2176                         sub_policy = mlx5_ipool_get
2177                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2178                                 policy_idx);
2179                         if (!sub_policy)
2180                                 return -rte_mtr_error_set(error,
2181                                                 EINVAL,
2182                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2183                                                 NULL, "MTR object "
2184                                                 "meter policy invalid.");
2185                         if (__mlx5_flow_meter_policy_delete(dev, i,
2186                                                 sub_policy->main_policy,
2187                                                 error, true))
2188                                 return -rte_mtr_error_set(error,
2189                                                 EINVAL,
2190                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2191                                                 NULL, "MTR object "
2192                                                 "meter policy invalid.");
2193                         mlx5_free(sub_policy->main_policy);
2194                 }
2195                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2196                 priv->policy_idx_tbl = NULL;
2197         }
2198         if (priv->mtr_profile_tbl) {
2199                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2200                         fmp = entry;
2201                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2202                                                            error))
2203                                 return -rte_mtr_error_set(error, EINVAL,
2204                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2205                                                 NULL, "Fail to destroy "
2206                                                 "meter profile.");
2207                 }
2208                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2209                 priv->mtr_profile_tbl = NULL;
2210         }
2211         /* Delete default policy table. */
2212         mlx5_flow_destroy_def_policy(dev);
2213         if (priv->sh->refcnt == 1)
2214                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2215         return 0;
2216 }