drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev,
  19                 uint32_t meter_id, struct rte_mtr_error *error);
  20
  21 /**
  22  * Create the meter action.
  23  *
  24  * @param priv
  25  *   Pointer to mlx5_priv.
  26  * @param[in] fm
  27  *   Pointer to flow meter to be converted.
  28  *
  29  * @return
  30  *   Pointer to the meter action on success, NULL otherwise.
  31  */
  32 static void *
  33 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  34                               struct mlx5_flow_meter_info *fm)
  35 {
  36 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  37         struct mlx5dv_dr_flow_meter_attr mtr_init;
  38         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  39         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  40                                                      &fm->profile->srtcm_prm;
  41         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  42         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  43         uint32_t val;
  44         enum mlx5_meter_domain domain =
  45                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  46                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  47                                 MLX5_MTR_DOMAIN_INGRESS;
  48         struct mlx5_flow_meter_def_policy *def_policy =
  49                 priv->sh->mtrmng->def_policy[domain];
  50
  51         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  52         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  53         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  54         MLX5_SET(flow_meter_parameters, fmp,
  55                 start_color, MLX5_FLOW_COLOR_GREEN);
  56         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  57         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  58         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  59         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  60         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  61         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  62         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  63         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  64         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  65         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  66         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  67         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  68         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  69         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  70         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  71         mtr_init.flow_meter_parameter = fmp;
  72         mtr_init.flow_meter_parameter_sz =
  73                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  74         mtr_init.active = fm->active_state;
  75         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  76 #else
  77         (void)priv;
  78         (void)fm;
  79         return NULL;
  80 #endif
  81 }
  82
  83 /**
  84  * Find meter profile by id.
  85  *
  86  * @param priv
  87  *   Pointer to mlx5_priv.
  88  * @param meter_profile_id
  89  *   Meter profile id.
  90  *
  91  * @return
  92  *   Pointer to the profile found on success, NULL otherwise.
  93  */
  94 static struct mlx5_flow_meter_profile *
  95 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  96 {
  97         struct mlx5_flow_meter_profile *fmp;
  98         union mlx5_l3t_data data;
  99         int32_t ret;
 100
 101         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
 102                                meter_profile_id, &data) || !data.ptr)
 103                 return NULL;
 104         fmp = data.ptr;
 105         /* Remove reference taken by the mlx5_l3t_get_entry. */
 106         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 107                                    meter_profile_id);
 108         if (!ret || ret == -1)
 109                 return NULL;
 110         return fmp;
 111 }
 112
 113 /**
 114  * Validate the MTR profile.
 115  *
 116  * @param[in] dev
 117  *   Pointer to Ethernet device.
 118  * @param[in] meter_profile_id
 119  *   Meter profile id.
 120  * @param[in] profile
 121  *   Pointer to meter profile detail.
 122  * @param[out] error
 123  *   Pointer to the error structure.
 124  *
 125  * @return
 126  *   0 on success, a negative errno value otherwise and rte_errno is set.
 127  */
 128 static int
 129 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 130                                  uint32_t meter_profile_id,
 131                                  struct rte_mtr_meter_profile *profile,
 132                                  struct rte_mtr_error *error)
 133 {
 134         struct mlx5_priv *priv = dev->data->dev_private;
 135         struct mlx5_flow_meter_profile *fmp;
 136         uint32_t ls_factor;
 137         int ret;
 138         uint64_t cir, cbs;
 139         uint64_t eir, ebs;
 140         uint64_t pir, pbs;
 141
 142         /* Profile must not be NULL. */
 143         if (profile == NULL)
 144                 return -rte_mtr_error_set(error, EINVAL,
 145                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 146                                           NULL, "Meter profile is null.");
 147         /* Meter profile ID must be valid. */
 148         if (meter_profile_id == UINT32_MAX)
 149                 return -rte_mtr_error_set(error, EINVAL,
 150                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 151                                           NULL, "Meter profile id not valid.");
 152         /* Meter profile must not exist. */
 153         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 154         if (fmp)
 155                 return -rte_mtr_error_set(error, EEXIST,
 156                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 157                                           NULL,
 158                                           "Meter profile already exists.");
 159         if (!priv->sh->meter_aso_en) {
 160                 /* Old version is even not supported. */
 161                 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old)
 162                         return -rte_mtr_error_set(error, ENOTSUP,
 163                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 164                                 NULL, "Metering is not supported.");
 165                 /* Old FW metering only supports srTCM. */
 166                 if (profile->alg != RTE_MTR_SRTCM_RFC2697) {
 167                         return -rte_mtr_error_set(error, ENOTSUP,
 168                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 169                                 NULL, "Metering algorithm is not supported.");
 170                 } else if (profile->srtcm_rfc2697.ebs) {
 171                         /* EBS is not supported for old metering. */
 172                         return -rte_mtr_error_set(error, ENOTSUP,
 173                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 174                                 NULL, "EBS is not supported.");
 175                 }
 176                 if (profile->packet_mode)
 177                         return -rte_mtr_error_set(error, ENOTSUP,
 178                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 179                                 "Metering algorithm packet mode is not supported.");
 180         }
 181         ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0;
 182         switch (profile->alg) {
 183         case RTE_MTR_SRTCM_RFC2697:
 184                 cir = profile->srtcm_rfc2697.cir << ls_factor;
 185                 cbs = profile->srtcm_rfc2697.cbs << ls_factor;
 186                 ebs = profile->srtcm_rfc2697.ebs << ls_factor;
 187                 /* EBS could be zero for old metering. */
 188                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 189                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 190                     ebs <= MLX5_SRTCM_XBS_MAX) {
 191                         ret = 0;
 192                 } else {
 193                         ret = -rte_mtr_error_set(error, ENOTSUP,
 194                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 195                                         "Profile values out of range.");
 196                 }
 197                 break;
 198         case RTE_MTR_TRTCM_RFC2698:
 199                 cir = profile->trtcm_rfc2698.cir << ls_factor;
 200                 cbs = profile->trtcm_rfc2698.cbs << ls_factor;
 201                 pir = profile->trtcm_rfc2698.pir << ls_factor;
 202                 pbs = profile->trtcm_rfc2698.pbs << ls_factor;
 203                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 204                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 205                     pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) &&
 206                     pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) {
 207                         ret = 0;
 208                 } else {
 209                         ret = -rte_mtr_error_set(error, ENOTSUP,
 210                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 211                                         "Profile values out of range.");
 212                 }
 213                 break;
 214         case RTE_MTR_TRTCM_RFC4115:
 215                 cir = profile->trtcm_rfc4115.cir << ls_factor;
 216                 cbs = profile->trtcm_rfc4115.cbs << ls_factor;
 217                 eir = profile->trtcm_rfc4115.eir << ls_factor;
 218                 ebs = profile->trtcm_rfc4115.ebs << ls_factor;
 219                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 220                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 221                     eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) {
 222                         ret = 0;
 223                 } else {
 224                         ret = -rte_mtr_error_set(error, ENOTSUP,
 225                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 226                                         "Profile values out of range.");
 227                 }
 228                 break;
 229         default:
 230                 ret = -rte_mtr_error_set(error, ENOTSUP,
 231                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 232                                          "Unknown metering algorithm.");
 233                 break;
 234         }
 235         return ret;
 236 }
 237
 238 /*
 239  * Calculate mantissa and exponent for cir / eir.
 240  *
 241  * @param[in] xir
 242  *   Value to be calculated.
 243  * @param[out] man
 244  *   Pointer to the mantissa.
 245  * @param[out] exp
 246  *   Pointer to the exp.
 247  */
 248 static inline void
 249 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp)
 250 {
 251         int64_t _xir;
 252         int64_t delta = INT64_MAX;
 253         uint8_t _man = 0;
 254         uint8_t _exp = 0;
 255         uint64_t m, e;
 256
 257         /* Special case xir == 0 ? both exp and mantissa are 0. */
 258         if (xir == 0) {
 259                 *man = 0;
 260                 *exp = 0;
 261                 return;
 262         }
 263         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 264                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 265                         _xir = (1000000000ULL * m) >> e;
 266                         if (llabs(xir - _xir) <= delta) {
 267                                 delta = llabs(xir - _xir);
 268                                 _man = m;
 269                                 _exp = e;
 270                         }
 271                 }
 272         }
 273         *man = _man;
 274         *exp = _exp;
 275 }
 276
 277 /*
 278  * Calculate mantissa and exponent for xbs.
 279  *
 280  * @param[in] xbs
 281  *   Value to be calculated.
 282  * @param[out] man
 283  *   Pointer to the mantissa.
 284  * @param[out] exp
 285  *   Pointer to the exp.
 286  */
 287 static inline void
 288 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 289 {
 290         int _exp;
 291         double _man;
 292
 293         /* Special case xbs == 0 ? both exp and mantissa are 0. */
 294         if (xbs == 0) {
 295                 *man = 0;
 296                 *exp = 0;
 297                 return;
 298         }
 299         /* xbs = xbs_mantissa * 2^xbs_exponent */
 300         _man = frexp(xbs, &_exp);
 301         if (_exp >= MLX5_MAN_WIDTH) {
 302                 _man = _man * pow(2, MLX5_MAN_WIDTH);
 303                 _exp = _exp - MLX5_MAN_WIDTH;
 304         }
 305         *man = (uint8_t)ceil(_man);
 306         *exp = _exp;
 307 }
 308
 309 /**
 310  * Fill the prm meter parameter.
 311  *
 312  * @param[in,out] fmp
 313  *   Pointer to meter profile to be converted.
 314  * @param[out] error
 315  *   Pointer to the error structure.
 316  *
 317  * @return
 318  *   0 on success, a negative errno value otherwise and rte_errno is set.
 319  */
 320 static int
 321 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 322                            struct rte_mtr_error *error)
 323 {
 324         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 325         uint8_t man, exp;
 326         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 327         uint32_t eir_exp, eir_man, ebs_exp, ebs_man;
 328         uint64_t cir, cbs, eir, ebs;
 329
 330         switch (fmp->profile.alg) {
 331         case RTE_MTR_SRTCM_RFC2697:
 332                 cir = fmp->profile.srtcm_rfc2697.cir;
 333                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 334                 eir = 0;
 335                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 336                 break;
 337         case RTE_MTR_TRTCM_RFC2698:
 338                 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir >
 339                             fmp->profile.trtcm_rfc2698.cir &&
 340                             fmp->profile.trtcm_rfc2698.pbs >
 341                             fmp->profile.trtcm_rfc2698.cbs);
 342                 cir = fmp->profile.trtcm_rfc2698.cir;
 343                 cbs = fmp->profile.trtcm_rfc2698.cbs;
 344                 /* EIR / EBS are filled with PIR / PBS. */
 345                 eir = fmp->profile.trtcm_rfc2698.pir;
 346                 ebs = fmp->profile.trtcm_rfc2698.pbs;
 347                 break;
 348         case RTE_MTR_TRTCM_RFC4115:
 349                 cir = fmp->profile.trtcm_rfc4115.cir;
 350                 cbs = fmp->profile.trtcm_rfc4115.cbs;
 351                 eir = fmp->profile.trtcm_rfc4115.eir;
 352                 ebs = fmp->profile.trtcm_rfc4115.ebs;
 353                 break;
 354         default:
 355                 return -rte_mtr_error_set(error, EINVAL,
 356                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 357                                 "Metering algorithm mode is invalid");
 358         }
 359         /* Adjust the values for PPS mode. */
 360         if (fmp->profile.packet_mode) {
 361                 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 362                 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 363                 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 364                 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 365         }
 366         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 367         mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp);
 368         /* Check if cir mantissa is too large. */
 369         if (exp > ASO_DSEG_XIR_EXP_MASK)
 370                 return -rte_mtr_error_set(error, ENOTSUP,
 371                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 372                                           "meter profile parameter cir is not supported.");
 373         cir_man = man;
 374         cir_exp = exp;
 375          /* cbs = cbs_mantissa * 2^cbs_exponent */
 376         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 377         /* Check if cbs mantissa is too large. */
 378         if (exp > ASO_DSEG_EXP_MASK)
 379                 return -rte_mtr_error_set(error, ENOTSUP,
 380                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 381                                           "meter profile parameter cbs is not supported.");
 382         cbs_man = man;
 383         cbs_exp = exp;
 384         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 385                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 386                                           cir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 387                                           cir_man);
 388         mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp);
 389         /* Check if eir mantissa is too large. */
 390         if (exp > ASO_DSEG_XIR_EXP_MASK)
 391                 return -rte_mtr_error_set(error, ENOTSUP,
 392                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 393                                           "meter profile parameter eir is not supported.");
 394         eir_man = man;
 395         eir_exp = exp;
 396         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 397         /* Check if ebs mantissa is too large. */
 398         if (exp > ASO_DSEG_EXP_MASK)
 399                 return -rte_mtr_error_set(error, ENOTSUP,
 400                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 401                                           "meter profile parameter ebs is not supported.");
 402         ebs_man = man;
 403         ebs_exp = exp;
 404         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 405                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET |
 406                                           eir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 407                                           eir_man);
 408         if (srtcm->cbs_cir)
 409                 fmp->g_support = 1;
 410         if (srtcm->ebs_eir)
 411                 fmp->y_support = 1;
 412         return 0;
 413 }
 414
 415 /**
 416  * Callback to get MTR capabilities.
 417  *
 418  * @param[in] dev
 419  *   Pointer to Ethernet device.
 420  * @param[out] cap
 421  *   Pointer to save MTR capabilities.
 422  * @param[out] error
 423  *   Pointer to the error structure.
 424  *
 425  * @return
 426  *   0 on success, a negative errno value otherwise and rte_errno is set.
 427  */
 428 static int
 429 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 430                  struct rte_mtr_capabilities *cap,
 431                  struct rte_mtr_error *error __rte_unused)
 432 {
 433         struct mlx5_priv *priv = dev->data->dev_private;
 434         struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos;
 435
 436         if (!priv->mtr_en)
 437                 return -rte_mtr_error_set(error, ENOTSUP,
 438                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 439                                           "Meter is not supported");
 440         memset(cap, 0, sizeof(*cap));
 441         if (priv->sh->meter_aso_en) {
 442                 /* 2 meters per one ASO cache line. */
 443                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 444                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 445                 cap->trtcm_rfc2698_packet_mode_supported = 1;
 446                 cap->trtcm_rfc4115_packet_mode_supported = 1;
 447         } else {
 448                 cap->n_max = 1 << qattr->log_max_flow_meter;
 449         }
 450         cap->srtcm_rfc2697_byte_mode_supported = 1;
 451         cap->trtcm_rfc2698_byte_mode_supported = 1;
 452         cap->trtcm_rfc4115_byte_mode_supported = 1;
 453         cap->n_shared_max = cap->n_max;
 454         cap->identical = 1;
 455         cap->shared_identical = 1;
 456         cap->shared_n_flows_per_mtr_max = 4 << 20;
 457         /* 2M flows can share the same meter. */
 458         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 459         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 460         cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 461         cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 462         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 463         cap->meter_policy_n_max = cap->n_max;
 464         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 465                           RTE_MTR_STATS_N_PKTS_DROPPED;
 466         return 0;
 467 }
 468
 469 /**
 470  * Callback to add MTR profile.
 471  *
 472  * @param[in] dev
 473  *   Pointer to Ethernet device.
 474  * @param[in] meter_profile_id
 475  *   Meter profile id.
 476  * @param[in] profile
 477  *   Pointer to meter profile detail.
 478  * @param[out] error
 479  *   Pointer to the error structure.
 480  *
 481  * @return
 482  *   0 on success, a negative errno value otherwise and rte_errno is set.
 483  */
 484 static int
 485 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 486                        uint32_t meter_profile_id,
 487                        struct rte_mtr_meter_profile *profile,
 488                        struct rte_mtr_error *error)
 489 {
 490         struct mlx5_priv *priv = dev->data->dev_private;
 491         struct mlx5_flow_meter_profile *fmp;
 492         union mlx5_l3t_data data;
 493         int ret;
 494
 495         if (!priv->mtr_en)
 496                 return -rte_mtr_error_set(error, ENOTSUP,
 497                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 498                                           "Meter is not supported");
 499         /* Check input params. */
 500         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 501                                                profile, error);
 502         if (ret)
 503                 return ret;
 504         /* Meter profile memory allocation. */
 505         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 506                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 507         if (fmp == NULL)
 508                 return -rte_mtr_error_set(error, ENOMEM,
 509                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 510                                           NULL, "Meter profile memory "
 511                                           "alloc failed.");
 512         /* Fill profile info. */
 513         fmp->id = meter_profile_id;
 514         fmp->profile = *profile;
 515         /* Fill the flow meter parameters for the PRM. */
 516         ret = mlx5_flow_meter_param_fill(fmp, error);
 517         if (ret)
 518                 goto error;
 519         data.ptr = fmp;
 520         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 521                                  meter_profile_id, &data);
 522         if (ret)
 523                 return -rte_mtr_error_set(error, ENOTSUP,
 524                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 525                                           NULL, "Meter profile insert fail.");
 526         return 0;
 527 error:
 528         mlx5_free(fmp);
 529         return ret;
 530 }
 531
 532 /**
 533  * Callback to delete MTR profile.
 534  *
 535  * @param[in] dev
 536  *   Pointer to Ethernet device.
 537  * @param[in] meter_profile_id
 538  *   Meter profile id.
 539  * @param[out] error
 540  *   Pointer to the error structure.
 541  *
 542  * @return
 543  *   0 on success, a negative errno value otherwise and rte_errno is set.
 544  */
 545 static int
 546 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 547                           uint32_t meter_profile_id,
 548                           struct rte_mtr_error *error)
 549 {
 550         struct mlx5_priv *priv = dev->data->dev_private;
 551         struct mlx5_flow_meter_profile *fmp;
 552
 553         if (!priv->mtr_en)
 554                 return -rte_mtr_error_set(error, ENOTSUP,
 555                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 556                                           "Meter is not supported");
 557         /* Meter profile must exist. */
 558         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 559         if (fmp == NULL)
 560                 return -rte_mtr_error_set(error, ENOENT,
 561                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 562                                           &meter_profile_id,
 563                                           "Meter profile id is invalid.");
 564         /* Check profile is unused. */
 565         if (fmp->ref_cnt)
 566                 return -rte_mtr_error_set(error, EBUSY,
 567                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 568                                           NULL, "Meter profile is in use.");
 569         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 570                 return -rte_mtr_error_set(error, EBUSY,
 571                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 572                                           NULL, "Meter profile remove fail.");
 573         mlx5_free(fmp);
 574         return 0;
 575 }
 576
 577 /**
 578  * Find policy by id.
 579  *
 580  * @param[in] dev
 581  *   Pointer to Ethernet device.
 582  * @param policy_id
 583  *   Policy id.
 584  *
 585  * @return
 586  *   Pointer to the policy found on success, NULL otherwise.
 587  */
 588 struct mlx5_flow_meter_policy *
 589 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 590                             uint32_t policy_id,
 591                             uint32_t *policy_idx)
 592 {
 593         struct mlx5_priv *priv = dev->data->dev_private;
 594         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 595         union mlx5_l3t_data data;
 596
 597         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 598                 return NULL;
 599         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 600                                 !data.dword)
 601                 return NULL;
 602         if (policy_idx)
 603                 *policy_idx = data.dword;
 604         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 605                                         data.dword);
 606         /* Remove reference taken by the mlx5_l3t_get_entry. */
 607         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 608         if (sub_policy)
 609                 if (sub_policy->main_policy_id)
 610                         return sub_policy->main_policy;
 611         return NULL;
 612 }
 613
 614 /**
 615  * Get the next meter from one meter's policy in hierarchy chain.
 616  * Lock free, mutex should be acquired by caller.
 617  *
 618  * @param[in] priv
 619  *   Pointer to mlx5_priv.
 620  * @param[in] policy
 621  *   Pointer to flow meter policy.
 622  * @param[out] mtr_idx
 623  *   Pointer to Meter index.
 624  *
 625  * @return
 626  *   Pointer to the next meter, or NULL when fail.
 627  */
 628 struct mlx5_flow_meter_info *
 629 mlx5_flow_meter_hierarchy_next_meter(struct mlx5_priv *priv,
 630                                      struct mlx5_flow_meter_policy *policy,
 631                                      uint32_t *mtr_idx)
 632 {
 633         int i;
 634
 635         for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
 636                 if (policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
 637                         return mlx5_flow_meter_find(priv,
 638                                                     policy->act_cnt[i].next_mtr_id,
 639                                                     mtr_idx);
 640         }
 641         return NULL;
 642 }
 643
 644 /**
 645  * Get the last meter's policy from one meter's policy in hierarchy.
 646  *
 647  * @param[in] dev
 648  *   Pointer to Ethernet device.
 649  * @param[in] policy
 650  *   Pointer to flow meter policy.
 651  *
 652  * @return
 653  *   Pointer to the final meter's policy, or NULL when fail.
 654  */
 655 struct mlx5_flow_meter_policy *
 656 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 657                                         struct mlx5_flow_meter_policy *policy)
 658 {
 659         struct mlx5_priv *priv = dev->data->dev_private;
 660         struct mlx5_flow_meter_info *next_fm;
 661         struct mlx5_flow_meter_policy *next_policy = policy;
 662
 663         while (next_policy->is_hierarchy) {
 664                 rte_spinlock_lock(&next_policy->sl);
 665                 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, next_policy, NULL);
 666                 rte_spinlock_unlock(&next_policy->sl);
 667                 if (!next_fm || next_fm->def_policy)
 668                         return NULL;
 669                 next_policy = mlx5_flow_meter_policy_find(dev,
 670                                                 next_fm->policy_id, NULL);
 671                 MLX5_ASSERT(next_policy);
 672         }
 673         return next_policy;
 674 }
 675
 676 /**
 677  * Callback to check MTR policy action validate
 678  *
 679  * @param[in] dev
 680  *   Pointer to Ethernet device.
 681  * @param[in] actions
 682  *   Pointer to meter policy action detail.
 683  * @param[out] error
 684  *   Pointer to the error structure.
 685  *
 686  * @return
 687  *   0 on success, a negative errno value otherwise and rte_errno is set.
 688  */
 689 static int
 690 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 691         struct rte_mtr_meter_policy_params *policy,
 692         struct rte_mtr_error *error)
 693 {
 694         struct mlx5_priv *priv = dev->data->dev_private;
 695         struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ?
 696                                                   1 : 0 };
 697         bool is_rss = false;
 698         uint8_t policy_mode;
 699         uint8_t domain_bitmap;
 700         int ret;
 701
 702         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 703                 return -rte_mtr_error_set(error, ENOTSUP,
 704                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 705                                 NULL, "meter policy unsupported.");
 706         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 707                         &is_rss, &domain_bitmap, &policy_mode, error);
 708         if (ret)
 709                 return ret;
 710         return 0;
 711 }
 712
 713 static int
 714 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 715                         uint32_t policy_id,
 716                         struct mlx5_flow_meter_policy *mtr_policy,
 717                         struct rte_mtr_error *error,
 718                         bool clear_l3t)
 719 {
 720         struct mlx5_priv *priv = dev->data->dev_private;
 721         struct mlx5_flow_meter_sub_policy *sub_policy;
 722         uint32_t i, j;
 723         uint16_t sub_policy_num;
 724
 725         rte_spinlock_lock(&mtr_policy->sl);
 726         if (mtr_policy->ref_cnt) {
 727                 rte_spinlock_unlock(&mtr_policy->sl);
 728                 return -rte_mtr_error_set(error, EBUSY,
 729                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 730                                  NULL,
 731                                 "Meter policy object is being used.");
 732         }
 733         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 734         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 735         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 736                 sub_policy_num = (mtr_policy->sub_policy_num >>
 737                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 738                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 739                 if (sub_policy_num) {
 740                         for (j = 0; j < sub_policy_num; j++) {
 741                                 sub_policy = mtr_policy->sub_policys[i][j];
 742                                 if (sub_policy)
 743                                         mlx5_ipool_free
 744                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 745                                         sub_policy->idx);
 746                         }
 747                 }
 748         }
 749         if (priv->policy_idx_tbl && clear_l3t) {
 750                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 751                         rte_spinlock_unlock(&mtr_policy->sl);
 752                         return -rte_mtr_error_set(error, ENOTSUP,
 753                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 754                                 "Fail to delete policy in index table.");
 755                 }
 756         }
 757         rte_spinlock_unlock(&mtr_policy->sl);
 758         return 0;
 759 }
 760
 761 /**
 762  * Callback to add MTR policy.
 763  *
 764  * @param[in] dev
 765  *   Pointer to Ethernet device.
 766  * @param[out] policy_id
 767  *   Pointer to policy id
 768  * @param[in] actions
 769  *   Pointer to meter policy action detail.
 770  * @param[out] error
 771  *   Pointer to the error structure.
 772  *
 773  * @return
 774  *   0 on success, a negative errno value otherwise and rte_errno is set.
 775  */
 776 static int
 777 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 778                         uint32_t policy_id,
 779                         struct rte_mtr_meter_policy_params *policy,
 780                         struct rte_mtr_error *error)
 781 {
 782         struct mlx5_priv *priv = dev->data->dev_private;
 783         struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ?
 784                                                   1 : 0 };
 785         uint32_t sub_policy_idx = 0;
 786         uint32_t policy_idx = 0;
 787         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 788         struct mlx5_flow_meter_sub_policy *sub_policy;
 789         bool is_rss = false;
 790         uint8_t policy_mode;
 791         uint32_t i;
 792         int ret;
 793         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 794         uint16_t sub_policy_num;
 795         uint8_t domain_bitmap = 0;
 796         union mlx5_l3t_data data;
 797         bool skip_rule = false;
 798
 799         if (!priv->mtr_en)
 800                 return -rte_mtr_error_set(error, ENOTSUP,
 801                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 802                                           NULL, "meter policy unsupported. ");
 803         if (policy_id == MLX5_INVALID_POLICY_ID)
 804                 return -rte_mtr_error_set(error, ENOTSUP,
 805                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 806                                           NULL, "policy ID is invalid. ");
 807         if (policy_id == priv->sh->mtrmng->def_policy_id)
 808                 return -rte_mtr_error_set(error, EEXIST,
 809                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 810                                           NULL, "default policy ID exists. ");
 811         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 812         if (mtr_policy)
 813                 return -rte_mtr_error_set(error, EEXIST,
 814                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 815                                           NULL, "policy ID exists. ");
 816         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 817                                           &is_rss, &domain_bitmap,
 818                                           &policy_mode, error);
 819         if (ret)
 820                 return ret;
 821         if (!domain_bitmap)
 822                 return -rte_mtr_error_set(error, ENOTSUP,
 823                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 824                                           NULL, "fail to find policy domain.");
 825         if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
 826                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 827                         return -rte_mtr_error_set(error, EEXIST,
 828                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 829                                 NULL, "a policy with similar actions "
 830                                 "is already configured");
 831                 if (mlx5_flow_create_def_policy(dev))
 832                         return -rte_mtr_error_set(error, ENOTSUP,
 833                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 834                                 NULL,
 835                                 "fail to create non-terminated policy.");
 836                 priv->sh->mtrmng->def_policy_id = policy_id;
 837                 return 0;
 838         }
 839         if (!priv->sh->meter_aso_en)
 840                 return -rte_mtr_error_set(error, ENOTSUP,
 841                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 842                         "no ASO capability to support the policy ");
 843         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 844                 if (!(domain_bitmap & (1 << i)))
 845                         continue;
 846                 /*
 847                  * If RSS is found, it means that only the ingress domain can
 848                  * be supported. It is invalid to support RSS for one color
 849                  * and egress / transfer domain actions for another. Drop and
 850                  * jump action should have no impact.
 851                  */
 852                 if (is_rss) {
 853                         policy_size +=
 854                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 855                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 856                         break;
 857                 }
 858                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 859         }
 860         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 861                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 862         if (!mtr_policy)
 863                 return -rte_mtr_error_set(error, ENOMEM,
 864                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 865                                 "Memory alloc failed for meter policy.");
 866         if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
 867                 mtr_policy->skip_y = 1;
 868         else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
 869                 mtr_policy->skip_g = 1;
 870         policy_size = sizeof(struct mlx5_flow_meter_policy);
 871         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 872                 if (!(domain_bitmap & (1 << i)))
 873                         continue;
 874                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 875                         mtr_policy->ingress = 1;
 876                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 877                         mtr_policy->egress = 1;
 878                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 879                         mtr_policy->transfer = 1;
 880                 sub_policy = mlx5_ipool_zmalloc
 881                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 882                                  &sub_policy_idx);
 883                 if (!sub_policy ||
 884                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 885                         goto policy_add_err;
 886                 sub_policy->idx = sub_policy_idx;
 887                 sub_policy->main_policy = mtr_policy;
 888                 if (!policy_idx) {
 889                         policy_idx = sub_policy_idx;
 890                         sub_policy->main_policy_id = 1;
 891                 }
 892                 mtr_policy->sub_policys[i] =
 893                         (struct mlx5_flow_meter_sub_policy **)
 894                         ((uint8_t *)mtr_policy + policy_size);
 895                 mtr_policy->sub_policys[i][0] = sub_policy;
 896                 sub_policy_num = (mtr_policy->sub_policy_num >>
 897                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 898                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 899                 sub_policy_num++;
 900                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 901                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 902                 mtr_policy->sub_policy_num |=
 903                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 904                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 905                 /*
 906                  * If RSS is found, it means that only the ingress domain can
 907                  * be supported. It is invalid to support RSS for one color
 908                  * and egress / transfer domain actions for another. Drop and
 909                  * jump action should have no impact.
 910                  */
 911                 if (is_rss) {
 912                         mtr_policy->is_rss = 1;
 913                         break;
 914                 }
 915                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 916         }
 917         rte_spinlock_init(&mtr_policy->sl);
 918         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 919                                         policy->actions, error);
 920         if (ret)
 921                 goto policy_add_err;
 922         if (mtr_policy->is_hierarchy) {
 923                 struct mlx5_flow_meter_policy *final_policy;
 924
 925                 final_policy =
 926                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 927                 if (!final_policy)
 928                         goto policy_add_err;
 929                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 930         }
 931         /*
 932          * If either Green or Yellow has queue / RSS action, all the policy
 933          * rules will be created later in the flow splitting stage.
 934          */
 935         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 936                 /* Create policy rules in HW. */
 937                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 938                 if (ret)
 939                         goto policy_add_err;
 940         }
 941         data.dword = policy_idx;
 942         if (!priv->policy_idx_tbl) {
 943                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 944                 if (!priv->policy_idx_tbl)
 945                         goto policy_add_err;
 946         }
 947         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 948                 goto policy_add_err;
 949         return 0;
 950 policy_add_err:
 951         if (mtr_policy) {
 952                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 953                         mtr_policy, error, false);
 954                 mlx5_free(mtr_policy);
 955                 if (ret)
 956                         return ret;
 957         }
 958         return -rte_mtr_error_set(error, ENOTSUP,
 959                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 960                                   NULL, "Failed to create devx policy.");
 961 }
 962
 963 /**
 964  * Callback to delete MTR policy.
 965  *
 966  * @param[in] dev
 967  *   Pointer to Ethernet device.
 968  * @param[in] policy_id
 969  *   Meter policy id.
 970  * @param[out] error
 971  *   Pointer to the error structure.
 972  *
 973  * @return
 974  *   0 on success, a negative errno value otherwise and rte_errno is set.
 975  */
 976 static int
 977 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 978                           uint32_t policy_id,
 979                           struct rte_mtr_error *error)
 980 {
 981         struct mlx5_priv *priv = dev->data->dev_private;
 982         struct mlx5_flow_meter_policy *mtr_policy;
 983         uint32_t policy_idx;
 984         int ret;
 985
 986         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 987                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 988                         return -rte_mtr_error_set(error, ENOTSUP,
 989                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 990                                 "Meter policy object is being used.");
 991                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 992                 return 0;
 993         }
 994         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 995         if (!mtr_policy)
 996                 return -rte_mtr_error_set(error, ENOTSUP,
 997                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 998                         "Meter policy id is invalid. ");
 999         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
1000                                                 error, true);
1001         if (ret)
1002                 return ret;
1003         mlx5_free(mtr_policy);
1004         return 0;
1005 }
1006
1007 /**
1008  * Check meter validation.
1009  *
1010  * @param[in] priv
1011  *   Pointer to mlx5 private data structure.
1012  * @param[in] meter_id
1013  *   Meter id.
1014  * @param[in] params
1015  *   Pointer to rte meter parameters.
1016  * @param[out] error
1017  *   Pointer to rte meter error structure.
1018  *
1019  * @return
1020  *   0 on success, a negative errno value otherwise and rte_errno is set.
1021  */
1022 static int
1023 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
1024                          struct rte_mtr_params *params,
1025                          struct rte_mtr_error *error)
1026 {
1027         /* Meter must use global drop action. */
1028         if (!priv->sh->dr_drop_action)
1029                 return -rte_mtr_error_set(error, ENOTSUP,
1030                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1031                                           NULL,
1032                                           "No drop action ready for meter.");
1033         /* Meter params must not be NULL. */
1034         if (params == NULL)
1035                 return -rte_mtr_error_set(error, EINVAL,
1036                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1037                                           NULL, "Meter object params null.");
1038         /* Previous meter color is not supported. */
1039         if (params->use_prev_mtr_color && !priv->sh->meter_aso_en)
1040                 return -rte_mtr_error_set(error, ENOTSUP,
1041                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1042                                           NULL,
1043                                           "Previous meter color "
1044                                           "not supported.");
1045         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
1046                 return -rte_mtr_error_set(error, ENOENT,
1047                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1048                                 NULL, "Meter policy id not valid.");
1049         /* Validate meter id. */
1050         if (mlx5_flow_meter_find(priv, meter_id, NULL))
1051                 return -rte_mtr_error_set(error, EEXIST,
1052                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
1053                         "Meter object already exists.");
1054         return 0;
1055 }
1056
1057 /**
1058  * Modify the flow meter action.
1059  *
1060  * @param[in] priv
1061  *   Pointer to mlx5 private data structure.
1062  * @param[in] fm
1063  *   Pointer to flow meter to be modified.
1064  * @param[in] srtcm
1065  *   Pointer to meter srtcm description parameter.
1066  * @param[in] modify_bits
1067  *   The bit in srtcm to be updated.
1068  * @param[in] active_state
1069  *   The state to be updated.
1070  * @return
1071  *   0 on success, o negative value otherwise.
1072  */
1073 static int
1074 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
1075                 struct mlx5_flow_meter_info *fm,
1076                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
1077                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
1078 {
1079 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1080         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
1081         uint32_t *attr;
1082         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
1083         int ret;
1084         struct mlx5_aso_mtr *aso_mtr = NULL;
1085         uint32_t cbs_cir, ebs_eir, val;
1086
1087         if (priv->sh->meter_aso_en) {
1088                 fm->is_enable = !!is_enable;
1089                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1090                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1091                 if (ret)
1092                         return ret;
1093                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
1094                 if (ret)
1095                         return ret;
1096         } else {
1097                 /* Fill command parameters. */
1098                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
1099                 mod_attr.flow_meter_parameter = in;
1100                 mod_attr.flow_meter_parameter_sz =
1101                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
1102                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1103                         mod_attr.active = !!active_state;
1104                 else
1105                         mod_attr.active = 0;
1106                 attr = in;
1107                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
1108                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1109                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1110                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1111                                 ASO_DSEG_EXP_MASK;
1112                         MLX5_SET(flow_meter_parameters, attr,
1113                                 cbs_exponent, val);
1114                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1115                                 ASO_DSEG_MAN_MASK;
1116                         MLX5_SET(flow_meter_parameters, attr,
1117                                 cbs_mantissa, val);
1118                 }
1119                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1120                         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) &
1121                                 ASO_DSEG_EXP_MASK;
1122                         MLX5_SET(flow_meter_parameters, attr,
1123                                 cir_exponent, val);
1124                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1125                         MLX5_SET(flow_meter_parameters, attr,
1126                                 cir_mantissa, val);
1127                 }
1128                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1129                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1130                                 ASO_DSEG_EXP_MASK;
1131                         MLX5_SET(flow_meter_parameters, attr,
1132                                 ebs_exponent, val);
1133                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1134                                 ASO_DSEG_MAN_MASK;
1135                         MLX5_SET(flow_meter_parameters, attr,
1136                                 ebs_mantissa, val);
1137                 }
1138                 /* Apply modifications to meter only if it was created. */
1139                 if (fm->meter_action_g) {
1140                         ret = mlx5_glue->dv_modify_flow_action_meter
1141                                         (fm->meter_action_g, &mod_attr,
1142                                         rte_cpu_to_be_64(modify_bits));
1143                         if (ret)
1144                                 return ret;
1145                 }
1146                 /* Update succeeded modify meter parameters. */
1147                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1148                         fm->active_state = !!active_state;
1149         }
1150         return 0;
1151 #else
1152         (void)priv;
1153         (void)fm;
1154         (void)srtcm;
1155         (void)modify_bits;
1156         (void)active_state;
1157         (void)is_enable;
1158         return -ENOTSUP;
1159 #endif
1160 }
1161
1162 static int
1163 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1164                                 struct mlx5_flow_meter_info *fm,
1165                                 uint64_t stats_mask)
1166 {
1167         fm->bytes_dropped =
1168                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1169         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1170         if (fm->bytes_dropped || fm->pkts_dropped) {
1171                 if (!fm->drop_cnt) {
1172                         /* Alloc policer counters. */
1173                         fm->drop_cnt = mlx5_counter_alloc(dev);
1174                         if (!fm->drop_cnt)
1175                                 return -1;
1176                 }
1177         } else {
1178                 if (fm->drop_cnt) {
1179                         mlx5_counter_free(dev, fm->drop_cnt);
1180                         fm->drop_cnt = 0;
1181                 }
1182         }
1183         return 0;
1184 }
1185
1186 /**
1187  * Create meter rules.
1188  *
1189  * @param[in] dev
1190  *   Pointer to Ethernet device.
1191  * @param[in] meter_id
1192  *   Meter id.
1193  * @param[in] params
1194  *   Pointer to rte meter parameters.
1195  * @param[in] shared
1196  *   Meter shared with other flow or not.
1197  * @param[out] error
1198  *   Pointer to rte meter error structure.
1199  *
1200  * @return
1201  *   0 on success, a negative errno value otherwise and rte_errno is set.
1202  */
1203 static int
1204 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1205                        struct rte_mtr_params *params, int shared,
1206                        struct rte_mtr_error *error)
1207 {
1208         struct mlx5_priv *priv = dev->data->dev_private;
1209         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1210         struct mlx5_flow_meter_profile *fmp;
1211         struct mlx5_flow_meter_info *fm;
1212         /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */
1213         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1214         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1215         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1216                 .size = 0,
1217                 .trunk_size = 64,
1218                 .need_lock = 1,
1219                 .type = "mlx5_flow_mtr_flow_id_pool",
1220         };
1221         struct mlx5_aso_mtr *aso_mtr;
1222         uint32_t mtr_idx, policy_idx;
1223         union mlx5_l3t_data data;
1224         int ret;
1225         uint8_t domain_bitmap;
1226         uint8_t mtr_id_bits;
1227         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1228                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1229
1230         if (!priv->mtr_en)
1231                 return -rte_mtr_error_set(error, ENOTSUP,
1232                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1233                                         "Meter is not supported");
1234         /* Validate the parameters. */
1235         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1236         if (ret)
1237                 return ret;
1238         /* Meter profile must exist. */
1239         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1240         if (fmp == NULL)
1241                 return -rte_mtr_error_set(error, ENOENT,
1242                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1243                         NULL, "Meter profile id not valid.");
1244         /* Meter policy must exist. */
1245         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1246                 __atomic_add_fetch
1247                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1248                         1, __ATOMIC_RELAXED);
1249                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1250                 if (!priv->sh->config.dv_esw_en)
1251                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1252         } else {
1253                 if (!priv->sh->meter_aso_en)
1254                         return -rte_mtr_error_set(error, ENOTSUP,
1255                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1256                                 "Part of the policies cannot be "
1257                                 "supported without ASO ");
1258                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1259                                 params->meter_policy_id, &policy_idx);
1260                 if (!mtr_policy)
1261                         return -rte_mtr_error_set(error, ENOENT,
1262                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1263                                 NULL, "Meter policy id not valid.");
1264                 domain_bitmap = (mtr_policy->ingress ?
1265                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1266                                 (mtr_policy->egress ?
1267                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1268                                 (mtr_policy->transfer ?
1269                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1270                 if (fmp->g_support && mtr_policy->skip_g)
1271                         return -rte_mtr_error_set(error, ENOTSUP,
1272                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1273                                         NULL, "Meter green policy is empty.");
1274                 if (fmp->y_support && mtr_policy->skip_y)
1275                         return -rte_mtr_error_set(error, ENOTSUP,
1276                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1277                                         NULL, "Meter yellow policy is empty.");
1278         }
1279         /* Allocate the flow meter memory. */
1280         if (priv->sh->meter_aso_en) {
1281                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1282                 if (!mtr_idx)
1283                         return -rte_mtr_error_set(error, ENOMEM,
1284                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1285                                 "Memory alloc failed for meter.");
1286                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1287                 fm = &aso_mtr->fm;
1288         } else {
1289                 if (fmp->y_support)
1290                         return -rte_mtr_error_set(error, ENOMEM,
1291                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1292                                 "Unsupported profile with yellow.");
1293                 legacy_fm = mlx5_ipool_zmalloc
1294                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1295                 if (legacy_fm == NULL)
1296                         return -rte_mtr_error_set(error, ENOMEM,
1297                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1298                                 "Memory alloc failed for meter.");
1299                 legacy_fm->idx = mtr_idx;
1300                 fm = &legacy_fm->fm;
1301         }
1302         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1303         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1304             mtr_reg_bits) {
1305                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1306                 goto error;
1307         }
1308         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1309                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1310         /* Fill the flow meter parameters. */
1311         fm->meter_id = meter_id;
1312         fm->policy_id = params->meter_policy_id;
1313         fm->profile = fmp;
1314         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1315                 goto error;
1316         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1317                 goto error;
1318         /* Add to the flow meter list. */
1319         if (!priv->sh->meter_aso_en) {
1320                 MLX5_ASSERT(legacy_fm != NULL);
1321                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1322         }
1323         /* Add to the flow meter list. */
1324         fm->active_state = 1; /* Config meter starts as active. */
1325         fm->is_enable = params->meter_enable;
1326         fm->shared = !!shared;
1327         fm->color_aware = !!params->use_prev_mtr_color;
1328         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1329         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1330                 fm->def_policy = 1;
1331                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1332                 if (!fm->flow_ipool)
1333                         goto error;
1334         }
1335         rte_spinlock_init(&fm->sl);
1336         /* If ASO meter supported, update ASO flow meter by wqe. */
1337         if (priv->sh->meter_aso_en) {
1338                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1339                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1340                 if (ret)
1341                         goto error;
1342                 if (!priv->mtr_idx_tbl) {
1343                         priv->mtr_idx_tbl =
1344                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1345                         if (!priv->mtr_idx_tbl)
1346                                 goto error;
1347                 }
1348                 data.dword = mtr_idx;
1349                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1350                         goto error;
1351         } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) {
1352                 goto error;
1353         }
1354         fm->active_state = params->meter_enable;
1355         if (mtr_policy)
1356                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1357         return 0;
1358 error:
1359         mlx5_flow_destroy_mtr_tbls(dev, fm);
1360         /* Free policer counters. */
1361         if (fm->drop_cnt)
1362                 mlx5_counter_free(dev, fm->drop_cnt);
1363         if (priv->sh->meter_aso_en)
1364                 mlx5_flow_mtr_free(dev, mtr_idx);
1365         else
1366                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1367         return -rte_mtr_error_set(error, ENOTSUP,
1368                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1369                 NULL, "Failed to create devx meter.");
1370 }
1371
1372 static int
1373 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1374                         struct mlx5_flow_meter_info *fm,
1375                         uint32_t mtr_idx)
1376 {
1377         struct mlx5_priv *priv = dev->data->dev_private;
1378         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1379         struct mlx5_flow_meter_profile *fmp;
1380         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1381         struct mlx5_flow_meter_policy *mtr_policy;
1382
1383         /* Meter object must not have any owner. */
1384         MLX5_ASSERT(!fm->ref_cnt);
1385         /* Get meter profile. */
1386         fmp = fm->profile;
1387         if (fmp == NULL)
1388                 return -1;
1389         /* Update dependencies. */
1390         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1391         fm->profile = NULL;
1392         /* Remove from list. */
1393         if (!priv->sh->meter_aso_en) {
1394                 legacy_fm = container_of(fm,
1395                         struct mlx5_legacy_flow_meter, fm);
1396                 TAILQ_REMOVE(fms, legacy_fm, next);
1397         }
1398         /* Free drop counters. */
1399         if (fm->drop_cnt)
1400                 mlx5_counter_free(dev, fm->drop_cnt);
1401         /* Free meter flow table. */
1402         if (fm->flow_ipool) {
1403                 mlx5_ipool_destroy(fm->flow_ipool);
1404                 fm->flow_ipool = 0;
1405         }
1406         mlx5_flow_destroy_mtr_tbls(dev, fm);
1407         if (fm->def_policy)
1408                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1409                                 1, __ATOMIC_RELAXED);
1410         if (priv->sh->meter_aso_en) {
1411                 if (!fm->def_policy) {
1412                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1413                                                 fm->policy_id, NULL);
1414                         if (mtr_policy)
1415                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1416                                                 1, __ATOMIC_RELAXED);
1417                         fm->policy_id = 0;
1418                 }
1419                 fm->def_policy = 0;
1420                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1421                         return -1;
1422                 mlx5_flow_mtr_free(dev, mtr_idx);
1423         } else {
1424                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1425                                         legacy_fm->idx);
1426         }
1427         return 0;
1428 }
1429
1430 /**
1431  * Destroy meter rules.
1432  *
1433  * @param[in] dev
1434  *   Pointer to Ethernet device.
1435  * @param[in] meter_id
1436  *   Meter id.
1437  * @param[out] error
1438  *   Pointer to rte meter error structure.
1439  *
1440  * @return
1441  *   0 on success, a negative errno value otherwise and rte_errno is set.
1442  */
1443 static int
1444 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1445                         struct rte_mtr_error *error)
1446 {
1447         struct mlx5_priv *priv = dev->data->dev_private;
1448         struct mlx5_flow_meter_info *fm;
1449         uint32_t mtr_idx = 0;
1450
1451         if (!priv->mtr_en)
1452                 return -rte_mtr_error_set(error, ENOTSUP,
1453                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1454                                           NULL,
1455                                           "Meter is not supported");
1456         /* Meter object must exist. */
1457         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1458         if (fm == NULL)
1459                 return -rte_mtr_error_set(error, ENOENT,
1460                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1461                                           NULL,
1462                                           "Meter object id not valid.");
1463         /* Meter object must not have any owner. */
1464         if (fm->ref_cnt > 0)
1465                 return -rte_mtr_error_set(error, EBUSY,
1466                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1467                                           NULL,
1468                                           "Meter object is being used.");
1469         /* Destroy the meter profile. */
1470         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1471                 return -rte_mtr_error_set(error, EINVAL,
1472                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1473                                         NULL,
1474                                         "MTR object meter profile invalid.");
1475         return 0;
1476 }
1477
1478 /**
1479  * Modify meter state.
1480  *
1481  * @param[in] priv
1482  *   Pointer to mlx5 private data structure.
1483  * @param[in] fm
1484  *   Pointer to flow meter.
1485  * @param[in] new_state
1486  *   New state to update.
1487  * @param[out] error
1488  *   Pointer to rte meter error structure.
1489  *
1490  * @return
1491  *   0 on success, a negative errno value otherwise and rte_errno is set.
1492  */
1493 static int
1494 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1495                              struct mlx5_flow_meter_info *fm,
1496                              uint32_t new_state,
1497                              struct rte_mtr_error *error)
1498 {
1499         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1500                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1501                 .ebs_eir = 0,
1502         };
1503         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1504                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1505         int ret;
1506
1507         if (new_state == MLX5_FLOW_METER_DISABLE)
1508                 ret = mlx5_flow_meter_action_modify(priv, fm,
1509                                 &srtcm, modify_bits, 0, 0);
1510         else
1511                 ret = mlx5_flow_meter_action_modify(priv, fm,
1512                                                     &fm->profile->srtcm_prm,
1513                                                     modify_bits, 0, 1);
1514         if (ret)
1515                 return -rte_mtr_error_set(error, -ret,
1516                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1517                                           NULL,
1518                                           new_state ?
1519                                           "Failed to enable meter." :
1520                                           "Failed to disable meter.");
1521         return 0;
1522 }
1523
1524 /**
1525  * Callback to enable flow meter.
1526  *
1527  * @param[in] dev
1528  *   Pointer to Ethernet device.
1529  * @param[in] meter_id
1530  *   Meter id.
1531  * @param[out] error
1532  *   Pointer to rte meter error structure.
1533  *
1534  * @return
1535  *   0 on success, a negative errno value otherwise and rte_errno is set.
1536  */
1537 static int
1538 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1539                        uint32_t meter_id,
1540                        struct rte_mtr_error *error)
1541 {
1542         struct mlx5_priv *priv = dev->data->dev_private;
1543         struct mlx5_flow_meter_info *fm;
1544         int ret;
1545
1546         if (!priv->mtr_en)
1547                 return -rte_mtr_error_set(error, ENOTSUP,
1548                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1549                                           "Meter is not supported");
1550         /* Meter object must exist. */
1551         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1552         if (fm == NULL)
1553                 return -rte_mtr_error_set(error, ENOENT,
1554                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1555                                           NULL, "Meter not found.");
1556         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1557                 return 0;
1558         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1559                                            error);
1560         if (!ret)
1561                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1562         return ret;
1563 }
1564
1565 /**
1566  * Callback to disable flow meter.
1567  *
1568  * @param[in] dev
1569  *   Pointer to Ethernet device.
1570  * @param[in] meter_id
1571  *   Meter id.
1572  * @param[out] error
1573  *   Pointer to rte meter error structure.
1574  *
1575  * @return
1576  *   0 on success, a negative errno value otherwise and rte_errno is set.
1577  */
1578 static int
1579 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1580                         uint32_t meter_id,
1581                         struct rte_mtr_error *error)
1582 {
1583         struct mlx5_priv *priv = dev->data->dev_private;
1584         struct mlx5_flow_meter_info *fm;
1585         int ret;
1586
1587         if (!priv->mtr_en)
1588                 return -rte_mtr_error_set(error, ENOTSUP,
1589                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1590                                           "Meter is not supported");
1591         /* Meter object must exist. */
1592         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1593         if (fm == NULL)
1594                 return -rte_mtr_error_set(error, ENOENT,
1595                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1596                                           NULL, "Meter not found.");
1597         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1598                 return 0;
1599         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1600                                            error);
1601         if (!ret)
1602                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1603         return ret;
1604 }
1605
1606 /**
1607  * Callback to update meter profile.
1608  *
1609  * @param[in] dev
1610  *   Pointer to Ethernet device.
1611  * @param[in] meter_id
1612  *   Meter id.
1613  * @param[in] meter_profile_id
1614  *   To be updated meter profile id.
1615  * @param[out] error
1616  *   Pointer to rte meter error structure.
1617  *
1618  * @return
1619  *   0 on success, a negative errno value otherwise and rte_errno is set.
1620  */
1621 static int
1622 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1623                                uint32_t meter_id,
1624                                uint32_t meter_profile_id,
1625                                struct rte_mtr_error *error)
1626 {
1627         struct mlx5_priv *priv = dev->data->dev_private;
1628         struct mlx5_flow_meter_profile *fmp;
1629         struct mlx5_flow_meter_profile *old_fmp;
1630         struct mlx5_flow_meter_info *fm;
1631         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1632                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1633         int ret;
1634
1635         if (!priv->mtr_en)
1636                 return -rte_mtr_error_set(error, ENOTSUP,
1637                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1638                                           "Meter is not supported");
1639         /* Meter profile must exist. */
1640         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1641         if (fmp == NULL)
1642                 return -rte_mtr_error_set(error, ENOENT,
1643                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1644                                           NULL, "Meter profile not found.");
1645         /* Meter object must exist. */
1646         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1647         if (fm == NULL)
1648                 return -rte_mtr_error_set(error, ENOENT,
1649                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1650                                           NULL, "Meter not found.");
1651         /* MTR object already set to meter profile id. */
1652         old_fmp = fm->profile;
1653         if (fmp == old_fmp)
1654                 return 0;
1655         /* Update the profile. */
1656         fm->profile = fmp;
1657         /* Update meter params in HW (if not disabled). */
1658         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1659                 return 0;
1660         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1661                                               modify_bits, fm->active_state, 1);
1662         if (ret) {
1663                 fm->profile = old_fmp;
1664                 return -rte_mtr_error_set(error, -ret,
1665                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1666                                           NULL, "Failed to update meter"
1667                                           " parameters in hardware.");
1668         }
1669         old_fmp->ref_cnt--;
1670         fmp->ref_cnt++;
1671         return 0;
1672 }
1673
1674 /**
1675  * Callback to update meter stats mask.
1676  *
1677  * @param[in] dev
1678  *   Pointer to Ethernet device.
1679  * @param[in] meter_id
1680  *   Meter id.
1681  * @param[in] stats_mask
1682  *   To be updated stats_mask.
1683  * @param[out] error
1684  *   Pointer to rte meter error structure.
1685  *
1686  * @return
1687  *   0 on success, a negative errno value otherwise and rte_errno is set.
1688  */
1689 static int
1690 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1691                              uint32_t meter_id,
1692                              uint64_t stats_mask,
1693                              struct rte_mtr_error *error)
1694 {
1695         struct mlx5_priv *priv = dev->data->dev_private;
1696         struct mlx5_flow_meter_info *fm;
1697
1698         if (!priv->mtr_en)
1699                 return -rte_mtr_error_set(error, ENOTSUP,
1700                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1701                                           "Meter is not supported");
1702         /* Meter object must exist. */
1703         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1704         if (fm == NULL)
1705                 return -rte_mtr_error_set(error, ENOENT,
1706                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1707                                           NULL, "Meter object id not valid.");
1708         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1709                 return -rte_mtr_error_set(error, ENOENT,
1710                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1711                                           NULL, "Fail to allocate "
1712                                           "counter for meter.");
1713         return 0;
1714 }
1715
1716 /**
1717  * Callback to read meter statistics.
1718  *
1719  * @param[in] dev
1720  *   Pointer to Ethernet device.
1721  * @param[in] meter_id
1722  *   Meter id.
1723  * @param[out] stats
1724  *   Pointer to store the statistics.
1725  * @param[out] stats_mask
1726  *   Pointer to store the stats_mask.
1727  * @param[in] clear
1728  *   Statistic to be cleared after read or not.
1729  * @param[out] error
1730  *   Pointer to rte meter error structure.
1731  *
1732  * @return
1733  *   0 on success, a negative errno value otherwise and rte_errno is set.
1734  */
1735 static int
1736 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1737                            uint32_t meter_id,
1738                            struct rte_mtr_stats *stats,
1739                            uint64_t *stats_mask,
1740                            int clear,
1741                            struct rte_mtr_error *error)
1742 {
1743         struct mlx5_priv *priv = dev->data->dev_private;
1744         struct mlx5_flow_meter_info *fm;
1745         uint64_t pkts;
1746         uint64_t bytes;
1747         int ret = 0;
1748
1749         if (!priv->mtr_en)
1750                 return -rte_mtr_error_set(error, ENOTSUP,
1751                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1752                                           "Meter is not supported");
1753         /* Meter object must exist. */
1754         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1755         if (fm == NULL)
1756                 return -rte_mtr_error_set(error, ENOENT,
1757                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1758                                           NULL, "Meter object id not valid.");
1759         *stats_mask = 0;
1760         if (fm->bytes_dropped)
1761                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1762         if (fm->pkts_dropped)
1763                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1764         memset(stats, 0, sizeof(*stats));
1765         if (fm->drop_cnt) {
1766                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1767                                                  &bytes, NULL);
1768                 if (ret)
1769                         goto error;
1770                 /* If need to read the packets, set it. */
1771                 if (fm->pkts_dropped)
1772                         stats->n_pkts_dropped = pkts;
1773                 /* If need to read the bytes, set it. */
1774                 if (fm->bytes_dropped)
1775                         stats->n_bytes_dropped = bytes;
1776         }
1777         return 0;
1778 error:
1779         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1780                                  "Failed to read meter drop counters.");
1781 }
1782
1783 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1784         .capabilities_get = mlx5_flow_mtr_cap_get,
1785         .meter_profile_add = mlx5_flow_meter_profile_add,
1786         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1787         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1788         .meter_policy_add = mlx5_flow_meter_policy_add,
1789         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1790         .create = mlx5_flow_meter_create,
1791         .destroy = mlx5_flow_meter_destroy,
1792         .meter_enable = mlx5_flow_meter_enable,
1793         .meter_disable = mlx5_flow_meter_disable,
1794         .meter_profile_update = mlx5_flow_meter_profile_update,
1795         .meter_dscp_table_update = NULL,
1796         .stats_update = mlx5_flow_meter_stats_update,
1797         .stats_read = mlx5_flow_meter_stats_read,
1798 };
1799
1800 /**
1801  * Get meter operations.
1802  *
1803  * @param dev
1804  *   Pointer to Ethernet device structure.
1805  * @param arg
1806  *   Pointer to set the mtr operations.
1807  *
1808  * @return
1809  *   Always 0.
1810  */
1811 int
1812 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1813 {
1814         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1815         return 0;
1816 }
1817
1818 /**
1819  * Find meter by id.
1820  *
1821  * @param priv
1822  *   Pointer to mlx5_priv.
1823  * @param meter_id
1824  *   Meter id.
1825  * @param mtr_idx
1826  *   Pointer to Meter index.
1827  *
1828  * @return
1829  *   Pointer to the meter info found on success, NULL otherwise.
1830  */
1831 struct mlx5_flow_meter_info *
1832 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1833                 uint32_t *mtr_idx)
1834 {
1835         struct mlx5_legacy_flow_meter *legacy_fm;
1836         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1837         struct mlx5_aso_mtr *aso_mtr;
1838         struct mlx5_aso_mtr_pools_mng *pools_mng =
1839                                 &priv->sh->mtrmng->pools_mng;
1840         union mlx5_l3t_data data;
1841         uint16_t n_valid;
1842
1843         if (priv->sh->meter_aso_en) {
1844                 rte_rwlock_read_lock(&pools_mng->resize_mtrwl);
1845                 n_valid = pools_mng->n_valid;
1846                 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl);
1847                 if (!n_valid || !priv->mtr_idx_tbl ||
1848                     (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1849                     !data.dword))
1850                         return NULL;
1851                 if (mtr_idx)
1852                         *mtr_idx = data.dword;
1853                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1854                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1855                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1856                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1857                         return NULL;
1858                 return &aso_mtr->fm;
1859         }
1860         TAILQ_FOREACH(legacy_fm, fms, next)
1861                 if (meter_id == legacy_fm->fm.meter_id) {
1862                         if (mtr_idx)
1863                                 *mtr_idx = legacy_fm->idx;
1864                         return &legacy_fm->fm;
1865                 }
1866         return NULL;
1867 }
1868
1869 /**
1870  * Find meter by index.
1871  *
1872  * @param priv
1873  *   Pointer to mlx5_priv.
1874  * @param idx
1875  *   Meter index.
1876  *
1877  * @return
1878  *   Pointer to the meter info found on success, NULL otherwise.
1879  */
1880 struct mlx5_flow_meter_info *
1881 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1882 {
1883         struct mlx5_aso_mtr *aso_mtr;
1884
1885         if (priv->sh->meter_aso_en) {
1886                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1887                 if (!aso_mtr)
1888                         return NULL;
1889                 return &aso_mtr->fm;
1890         } else {
1891                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1892         }
1893 }
1894
1895 /**
1896  * Attach meter to flow.
1897  * Unidirectional Meter creation can only be done
1898  * when flow direction is known, i.e. when calling meter_attach.
1899  *
1900  * @param [in] priv
1901  *  Pointer to mlx5 private data.
1902  * @param[in] fm
1903  *   Pointer to flow meter.
1904  * @param [in] attr
1905  *  Pointer to flow attributes.
1906  * @param [out] error
1907  *  Pointer to error structure.
1908  *
1909  * @return
1910  *   0 on success, a negative errno value otherwise and rte_errno is set.
1911  */
1912 int
1913 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1914                        struct mlx5_flow_meter_info *fm,
1915                        const struct rte_flow_attr *attr,
1916                        struct rte_flow_error *error)
1917 {
1918         int ret = 0;
1919
1920         if (priv->sh->meter_aso_en) {
1921                 struct mlx5_aso_mtr *aso_mtr;
1922
1923                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1924                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1925                         return rte_flow_error_set(error, ENOENT,
1926                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1927                                         NULL,
1928                                         "Timeout in meter configuration");
1929                 }
1930                 rte_spinlock_lock(&fm->sl);
1931                 if (fm->shared || !fm->ref_cnt) {
1932                         fm->ref_cnt++;
1933                 } else {
1934                         rte_flow_error_set(error, EINVAL,
1935                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1936                                    "Meter cannot be shared");
1937                         ret = -1;
1938                 }
1939                 rte_spinlock_unlock(&fm->sl);
1940         } else {
1941                 rte_spinlock_lock(&fm->sl);
1942                 if (fm->meter_action_g) {
1943                         if (fm->shared &&
1944                             attr->transfer == fm->transfer &&
1945                             attr->ingress == fm->ingress &&
1946                             attr->egress == fm->egress) {
1947                                 fm->ref_cnt++;
1948                         } else {
1949                                 rte_flow_error_set(error, EINVAL,
1950                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1951                                         fm->shared ?
1952                                         "Meter attr not match." :
1953                                         "Meter cannot be shared.");
1954                                 ret = -1;
1955                         }
1956                 } else {
1957                         fm->ingress = attr->ingress;
1958                         fm->egress = attr->egress;
1959                         fm->transfer = attr->transfer;
1960                         fm->ref_cnt = 1;
1961                         /* This also creates the meter object. */
1962                         fm->meter_action_g = mlx5_flow_meter_action_create(priv,
1963                                                                          fm);
1964                         if (!fm->meter_action_g) {
1965                                 fm->ref_cnt = 0;
1966                                 fm->ingress = 0;
1967                                 fm->egress = 0;
1968                                 fm->transfer = 0;
1969                                 rte_flow_error_set(error, EINVAL,
1970                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1971                                         "Meter action create failed.");
1972                                 ret = -1;
1973                         }
1974                 }
1975                 rte_spinlock_unlock(&fm->sl);
1976         }
1977         return ret ? -rte_errno : 0;
1978 }
1979
1980 /**
1981  * Detach meter from flow.
1982  *
1983  * @param [in] priv
1984  *  Pointer to mlx5 private data.
1985  * @param [in] fm
1986  *  Pointer to flow meter.
1987  */
1988 void
1989 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1990                        struct mlx5_flow_meter_info *fm)
1991 {
1992 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1993         rte_spinlock_lock(&fm->sl);
1994         MLX5_ASSERT(fm->ref_cnt);
1995         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1996                 mlx5_glue->destroy_flow_action(fm->meter_action_g);
1997                 fm->meter_action_g = NULL;
1998                 fm->ingress = 0;
1999                 fm->egress = 0;
2000                 fm->transfer = 0;
2001         }
2002         rte_spinlock_unlock(&fm->sl);
2003 #else
2004         (void)priv;
2005         (void)fm;
2006 #endif
2007 }
2008
2009 /**
2010  * Flush meter with Rx queue configuration.
2011  *
2012  * @param[in] dev
2013  *   Pointer to Ethernet device.
2014  */
2015 void
2016 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
2017 {
2018         struct mlx5_priv *priv = dev->data->dev_private;
2019         struct mlx5_flow_meter_sub_policy *sub_policy;
2020         struct mlx5_flow_meter_policy *mtr_policy;
2021         void *entry;
2022         uint32_t i, policy_idx;
2023
2024         if (!priv->mtr_en)
2025                 return;
2026         if (priv->policy_idx_tbl) {
2027                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2028                         policy_idx = *(uint32_t *)entry;
2029                         sub_policy = mlx5_ipool_get
2030                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2031                                 policy_idx);
2032                         if (!sub_policy || !sub_policy->main_policy)
2033                                 continue;
2034                         mtr_policy = sub_policy->main_policy;
2035                         if (mtr_policy->is_queue || mtr_policy->is_rss)
2036                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
2037                                         mtr_policy);
2038                 }
2039         }
2040 }
2041
2042 /**
2043  * Iterate a meter hierarchy and flush all meters and policies if possible.
2044  *
2045  * @param[in] dev
2046  *   Pointer to Ethernet device.
2047  * @param[in] fm
2048  *   Pointer to flow meter.
2049  * @param[in] mtr_idx
2050  *   .Meter's index
2051  * @param[out] error
2052  *   Pointer to rte meter error structure.
2053  *
2054  * @return
2055  *   0 on success, a negative errno value otherwise and rte_errno is set.
2056  */
2057 static int
2058 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
2059                                 struct mlx5_flow_meter_info *fm,
2060                                 uint32_t mtr_idx,
2061                                 struct rte_mtr_error *error)
2062 {
2063         struct mlx5_priv *priv = dev->data->dev_private;
2064         struct mlx5_flow_meter_policy *policy;
2065         uint32_t policy_id;
2066         struct mlx5_flow_meter_info *next_fm;
2067         uint32_t next_mtr_idx;
2068         struct mlx5_flow_meter_policy *next_policy = NULL;
2069
2070         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
2071         MLX5_ASSERT(policy);
2072         while (!fm->ref_cnt && policy->is_hierarchy) {
2073                 policy_id = fm->policy_id;
2074                 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &next_mtr_idx);
2075                 if (next_fm) {
2076                         next_policy = mlx5_flow_meter_policy_find(dev,
2077                                                         next_fm->policy_id,
2078                                                         NULL);
2079                         MLX5_ASSERT(next_policy);
2080                 }
2081                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
2082                         return -rte_mtr_error_set(error, ENOTSUP,
2083                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
2084                                                 NULL,
2085                                                 "Failed to flush meter.");
2086                 if (policy->ref_cnt)
2087                         break;
2088                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
2089                                                 policy, error, true))
2090                         return -rte_errno;
2091                 mlx5_free(policy);
2092                 if (!next_fm || !next_policy)
2093                         break;
2094                 fm = next_fm;
2095                 mtr_idx = next_mtr_idx;
2096                 policy = next_policy;
2097         }
2098         return 0;
2099 }
2100
2101 /**
2102  * Flush all the hierarchy meters and their policies.
2103  *
2104  * @param[in] dev
2105  *   Pointer to Ethernet device.
2106  * @param[out] error
2107  *   Pointer to rte meter error structure.
2108  *
2109  * @return
2110  *   0 on success, a negative errno value otherwise and rte_errno is set.
2111  */
2112 static int
2113 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
2114                                       struct rte_mtr_error *error)
2115 {
2116         struct mlx5_priv *priv = dev->data->dev_private;
2117         struct mlx5_flow_meter_info *fm;
2118         struct mlx5_flow_meter_policy *policy;
2119         struct mlx5_flow_meter_sub_policy *sub_policy;
2120         struct mlx5_flow_meter_info *next_fm;
2121         struct mlx5_aso_mtr *aso_mtr;
2122         uint32_t mtr_idx = 0;
2123         uint32_t i, policy_idx;
2124         void *entry;
2125
2126         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2127                 return 0;
2128         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2129                 mtr_idx = *(uint32_t *)entry;
2130                 if (!mtr_idx)
2131                         continue;
2132                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2133                 fm = &aso_mtr->fm;
2134                 if (fm->ref_cnt || fm->def_policy)
2135                         continue;
2136                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2137                         return -rte_errno;
2138         }
2139         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2140                 policy_idx = *(uint32_t *)entry;
2141                 sub_policy = mlx5_ipool_get
2142                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2143                                 policy_idx);
2144                 if (!sub_policy)
2145                         return -rte_mtr_error_set(error,
2146                                         EINVAL,
2147                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2148                                         NULL, "Meter policy invalid.");
2149                 policy = sub_policy->main_policy;
2150                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2151                         continue;
2152                 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &mtr_idx);
2153                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2154                                                     error, true))
2155                         return -rte_mtr_error_set(error,
2156                                         EINVAL,
2157                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2158                                         NULL, "Meter policy invalid.");
2159                 mlx5_free(policy);
2160                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2161                         continue;
2162                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2163                                                     mtr_idx, error))
2164                         return -rte_errno;
2165         }
2166         return 0;
2167 }
2168 /**
2169  * Flush meter configuration.
2170  *
2171  * @param[in] dev
2172  *   Pointer to Ethernet device.
2173  * @param[out] error
2174  *   Pointer to rte meter error structure.
2175  *
2176  * @return
2177  *   0 on success, a negative errno value otherwise and rte_errno is set.
2178  */
2179 int
2180 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2181 {
2182         struct mlx5_priv *priv = dev->data->dev_private;
2183         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2184         struct mlx5_flow_meter_profile *fmp;
2185         struct mlx5_legacy_flow_meter *legacy_fm;
2186         struct mlx5_flow_meter_info *fm;
2187         struct mlx5_flow_meter_sub_policy *sub_policy;
2188         void *tmp;
2189         uint32_t i, mtr_idx, policy_idx;
2190         void *entry;
2191         struct mlx5_aso_mtr *aso_mtr;
2192
2193         if (!priv->mtr_en)
2194                 return 0;
2195         if (priv->sh->meter_aso_en) {
2196                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2197                         return -rte_errno;
2198                 if (priv->mtr_idx_tbl) {
2199                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2200                                 mtr_idx = *(uint32_t *)entry;
2201                                 if (mtr_idx) {
2202                                         aso_mtr =
2203                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2204                                         fm = &aso_mtr->fm;
2205                                         (void)mlx5_flow_meter_params_flush(dev,
2206                                                 fm, mtr_idx);
2207                                 }
2208                         }
2209                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2210                         priv->mtr_idx_tbl = NULL;
2211                 }
2212         } else {
2213                 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2214                         fm = &legacy_fm->fm;
2215                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2216                                 return -rte_mtr_error_set(error, EINVAL,
2217                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2218                                 NULL, "MTR object meter profile invalid.");
2219                 }
2220         }
2221         if (priv->policy_idx_tbl) {
2222                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2223                         policy_idx = *(uint32_t *)entry;
2224                         sub_policy = mlx5_ipool_get
2225                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2226                                 policy_idx);
2227                         if (!sub_policy)
2228                                 return -rte_mtr_error_set(error,
2229                                                 EINVAL,
2230                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2231                                                 NULL, "MTR object "
2232                                                 "meter policy invalid.");
2233                         if (__mlx5_flow_meter_policy_delete(dev, i,
2234                                                 sub_policy->main_policy,
2235                                                 error, true))
2236                                 return -rte_mtr_error_set(error,
2237                                                 EINVAL,
2238                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2239                                                 NULL, "MTR object "
2240                                                 "meter policy invalid.");
2241                         mlx5_free(sub_policy->main_policy);
2242                 }
2243                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2244                 priv->policy_idx_tbl = NULL;
2245         }
2246         if (priv->mtr_profile_tbl) {
2247                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2248                         fmp = entry;
2249                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2250                                                            error))
2251                                 return -rte_mtr_error_set(error, EINVAL,
2252                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2253                                                 NULL, "Fail to destroy "
2254                                                 "meter profile.");
2255                 }
2256                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2257                 priv->mtr_profile_tbl = NULL;
2258         }
2259         /* Delete default policy table. */
2260         mlx5_flow_destroy_def_policy(dev);
2261         if (priv->sh->refcnt == 1)
2262                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2263         return 0;
2264 }