drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev,
  19                 uint32_t meter_id, struct rte_mtr_error *error);
  20
  21 /**
  22  * Create the meter action.
  23  *
  24  * @param priv
  25  *   Pointer to mlx5_priv.
  26  * @param[in] fm
  27  *   Pointer to flow meter to be converted.
  28  *
  29  * @return
  30  *   Pointer to the meter action on success, NULL otherwise.
  31  */
  32 static void *
  33 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  34                               struct mlx5_flow_meter_info *fm)
  35 {
  36 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  37         struct mlx5dv_dr_flow_meter_attr mtr_init;
  38         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  39         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  40                                                      &fm->profile->srtcm_prm;
  41         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  42         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  43         uint32_t val;
  44         enum mlx5_meter_domain domain =
  45                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  46                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  47                                 MLX5_MTR_DOMAIN_INGRESS;
  48         struct mlx5_flow_meter_def_policy *def_policy =
  49                 priv->sh->mtrmng->def_policy[domain];
  50
  51         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  52         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  53         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  54         MLX5_SET(flow_meter_parameters, fmp,
  55                 start_color, MLX5_FLOW_COLOR_GREEN);
  56         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  57         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  58         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  59         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  60         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  61         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  62         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  63         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  64         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  65         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  66         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  67         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  68         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  69         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  70         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  71         mtr_init.flow_meter_parameter = fmp;
  72         mtr_init.flow_meter_parameter_sz =
  73                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  74         mtr_init.active = fm->active_state;
  75         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  76 #else
  77         (void)priv;
  78         (void)fm;
  79         return NULL;
  80 #endif
  81 }
  82
  83 /**
  84  * Find meter profile by id.
  85  *
  86  * @param priv
  87  *   Pointer to mlx5_priv.
  88  * @param meter_profile_id
  89  *   Meter profile id.
  90  *
  91  * @return
  92  *   Pointer to the profile found on success, NULL otherwise.
  93  */
  94 static struct mlx5_flow_meter_profile *
  95 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  96 {
  97         struct mlx5_flow_meter_profile *fmp;
  98         union mlx5_l3t_data data;
  99         int32_t ret;
 100
 101         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
 102                                meter_profile_id, &data) || !data.ptr)
 103                 return NULL;
 104         fmp = data.ptr;
 105         /* Remove reference taken by the mlx5_l3t_get_entry. */
 106         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 107                                    meter_profile_id);
 108         if (!ret || ret == -1)
 109                 return NULL;
 110         return fmp;
 111 }
 112
 113 /**
 114  * Validate the MTR profile.
 115  *
 116  * @param[in] dev
 117  *   Pointer to Ethernet device.
 118  * @param[in] meter_profile_id
 119  *   Meter profile id.
 120  * @param[in] profile
 121  *   Pointer to meter profile detail.
 122  * @param[out] error
 123  *   Pointer to the error structure.
 124  *
 125  * @return
 126  *   0 on success, a negative errno value otherwise and rte_errno is set.
 127  */
 128 static int
 129 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 130                                  uint32_t meter_profile_id,
 131                                  struct rte_mtr_meter_profile *profile,
 132                                  struct rte_mtr_error *error)
 133 {
 134         struct mlx5_priv *priv = dev->data->dev_private;
 135         struct mlx5_flow_meter_profile *fmp;
 136         uint32_t ls_factor;
 137         int ret;
 138         uint64_t cir, cbs;
 139         uint64_t eir, ebs;
 140         uint64_t pir, pbs;
 141
 142         /* Profile must not be NULL. */
 143         if (profile == NULL)
 144                 return -rte_mtr_error_set(error, EINVAL,
 145                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 146                                           NULL, "Meter profile is null.");
 147         /* Meter profile ID must be valid. */
 148         if (meter_profile_id == UINT32_MAX)
 149                 return -rte_mtr_error_set(error, EINVAL,
 150                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 151                                           NULL, "Meter profile id not valid.");
 152         /* Meter profile must not exist. */
 153         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 154         if (fmp)
 155                 return -rte_mtr_error_set(error, EEXIST,
 156                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 157                                           NULL,
 158                                           "Meter profile already exists.");
 159         if (!priv->sh->meter_aso_en) {
 160                 /* Old version is even not supported. */
 161                 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old)
 162                         return -rte_mtr_error_set(error, ENOTSUP,
 163                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 164                                 NULL, "Metering is not supported.");
 165                 /* Old FW metering only supports srTCM. */
 166                 if (profile->alg != RTE_MTR_SRTCM_RFC2697) {
 167                         return -rte_mtr_error_set(error, ENOTSUP,
 168                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 169                                 NULL, "Metering algorithm is not supported.");
 170                 } else if (profile->srtcm_rfc2697.ebs) {
 171                         /* EBS is not supported for old metering. */
 172                         return -rte_mtr_error_set(error, ENOTSUP,
 173                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 174                                 NULL, "EBS is not supported.");
 175                 }
 176                 if (profile->packet_mode)
 177                         return -rte_mtr_error_set(error, ENOTSUP,
 178                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 179                                 "Metering algorithm packet mode is not supported.");
 180         }
 181         ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0;
 182         switch (profile->alg) {
 183         case RTE_MTR_SRTCM_RFC2697:
 184                 cir = profile->srtcm_rfc2697.cir << ls_factor;
 185                 cbs = profile->srtcm_rfc2697.cbs << ls_factor;
 186                 ebs = profile->srtcm_rfc2697.ebs << ls_factor;
 187                 /* EBS could be zero for old metering. */
 188                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 189                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 190                     ebs <= MLX5_SRTCM_XBS_MAX) {
 191                         ret = 0;
 192                 } else {
 193                         ret = -rte_mtr_error_set(error, ENOTSUP,
 194                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 195                                         "Profile values out of range.");
 196                 }
 197                 break;
 198         case RTE_MTR_TRTCM_RFC2698:
 199                 cir = profile->trtcm_rfc2698.cir << ls_factor;
 200                 cbs = profile->trtcm_rfc2698.cbs << ls_factor;
 201                 pir = profile->trtcm_rfc2698.pir << ls_factor;
 202                 pbs = profile->trtcm_rfc2698.pbs << ls_factor;
 203                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 204                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 205                     pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) &&
 206                     pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) {
 207                         ret = 0;
 208                 } else {
 209                         ret = -rte_mtr_error_set(error, ENOTSUP,
 210                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 211                                         "Profile values out of range.");
 212                 }
 213                 break;
 214         case RTE_MTR_TRTCM_RFC4115:
 215                 cir = profile->trtcm_rfc4115.cir << ls_factor;
 216                 cbs = profile->trtcm_rfc4115.cbs << ls_factor;
 217                 eir = profile->trtcm_rfc4115.eir << ls_factor;
 218                 ebs = profile->trtcm_rfc4115.ebs << ls_factor;
 219                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 220                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 221                     eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) {
 222                         ret = 0;
 223                 } else {
 224                         ret = -rte_mtr_error_set(error, ENOTSUP,
 225                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 226                                         "Profile values out of range.");
 227                 }
 228                 break;
 229         default:
 230                 ret = -rte_mtr_error_set(error, ENOTSUP,
 231                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 232                                          "Unknown metering algorithm.");
 233                 break;
 234         }
 235         return ret;
 236 }
 237
 238 /*
 239  * Calculate mantissa and exponent for cir / eir.
 240  *
 241  * @param[in] xir
 242  *   Value to be calculated.
 243  * @param[out] man
 244  *   Pointer to the mantissa.
 245  * @param[out] exp
 246  *   Pointer to the exp.
 247  */
 248 static inline void
 249 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp)
 250 {
 251         int64_t _xir;
 252         int64_t delta = INT64_MAX;
 253         uint8_t _man = 0;
 254         uint8_t _exp = 0;
 255         uint64_t m, e;
 256
 257         /* Special case xir == 0 ? both exp and mantissa are 0. */
 258         if (xir == 0) {
 259                 *man = 0;
 260                 *exp = 0;
 261                 return;
 262         }
 263         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 264                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 265                         _xir = (1000000000ULL * m) >> e;
 266                         if (llabs(xir - _xir) <= delta) {
 267                                 delta = llabs(xir - _xir);
 268                                 _man = m;
 269                                 _exp = e;
 270                         }
 271                 }
 272         }
 273         *man = _man;
 274         *exp = _exp;
 275 }
 276
 277 /*
 278  * Calculate mantissa and exponent for xbs.
 279  *
 280  * @param[in] xbs
 281  *   Value to be calculated.
 282  * @param[out] man
 283  *   Pointer to the mantissa.
 284  * @param[out] exp
 285  *   Pointer to the exp.
 286  */
 287 static inline void
 288 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 289 {
 290         int _exp;
 291         double _man;
 292
 293         /* Special case xbs == 0 ? both exp and mantissa are 0. */
 294         if (xbs == 0) {
 295                 *man = 0;
 296                 *exp = 0;
 297                 return;
 298         }
 299         /* xbs = xbs_mantissa * 2^xbs_exponent */
 300         _man = frexp(xbs, &_exp);
 301         if (_exp >= MLX5_MAN_WIDTH) {
 302                 _man = _man * pow(2, MLX5_MAN_WIDTH);
 303                 _exp = _exp - MLX5_MAN_WIDTH;
 304         }
 305         *man = (uint8_t)ceil(_man);
 306         *exp = _exp;
 307 }
 308
 309 /**
 310  * Fill the prm meter parameter.
 311  *
 312  * @param[in,out] fmp
 313  *   Pointer to meter profile to be converted.
 314  * @param[out] error
 315  *   Pointer to the error structure.
 316  *
 317  * @return
 318  *   0 on success, a negative errno value otherwise and rte_errno is set.
 319  */
 320 static int
 321 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 322                            struct rte_mtr_error *error)
 323 {
 324         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 325         uint8_t man, exp;
 326         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 327         uint32_t eir_exp, eir_man, ebs_exp, ebs_man;
 328         uint64_t cir, cbs, eir, ebs;
 329
 330         switch (fmp->profile.alg) {
 331         case RTE_MTR_SRTCM_RFC2697:
 332                 cir = fmp->profile.srtcm_rfc2697.cir;
 333                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 334                 eir = 0;
 335                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 336                 break;
 337         case RTE_MTR_TRTCM_RFC2698:
 338                 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir >
 339                             fmp->profile.trtcm_rfc2698.cir &&
 340                             fmp->profile.trtcm_rfc2698.pbs >
 341                             fmp->profile.trtcm_rfc2698.cbs);
 342                 cir = fmp->profile.trtcm_rfc2698.cir;
 343                 cbs = fmp->profile.trtcm_rfc2698.cbs;
 344                 /* EIR / EBS are filled with PIR / PBS. */
 345                 eir = fmp->profile.trtcm_rfc2698.pir;
 346                 ebs = fmp->profile.trtcm_rfc2698.pbs;
 347                 break;
 348         case RTE_MTR_TRTCM_RFC4115:
 349                 cir = fmp->profile.trtcm_rfc4115.cir;
 350                 cbs = fmp->profile.trtcm_rfc4115.cbs;
 351                 eir = fmp->profile.trtcm_rfc4115.eir;
 352                 ebs = fmp->profile.trtcm_rfc4115.ebs;
 353                 break;
 354         default:
 355                 return -rte_mtr_error_set(error, EINVAL,
 356                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 357                                 "Metering algorithm mode is invalid");
 358         }
 359         /* Adjust the values for PPS mode. */
 360         if (fmp->profile.packet_mode) {
 361                 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 362                 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 363                 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 364                 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 365         }
 366         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 367         mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp);
 368         /* Check if cir mantissa is too large. */
 369         if (exp > ASO_DSEG_XIR_EXP_MASK)
 370                 return -rte_mtr_error_set(error, ENOTSUP,
 371                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 372                                           "meter profile parameter cir is not supported.");
 373         cir_man = man;
 374         cir_exp = exp;
 375          /* cbs = cbs_mantissa * 2^cbs_exponent */
 376         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 377         /* Check if cbs mantissa is too large. */
 378         if (exp > ASO_DSEG_EXP_MASK)
 379                 return -rte_mtr_error_set(error, ENOTSUP,
 380                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 381                                           "meter profile parameter cbs is not supported.");
 382         cbs_man = man;
 383         cbs_exp = exp;
 384         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 385                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 386                                           cir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 387                                           cir_man);
 388         mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp);
 389         /* Check if eir mantissa is too large. */
 390         if (exp > ASO_DSEG_XIR_EXP_MASK)
 391                 return -rte_mtr_error_set(error, ENOTSUP,
 392                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 393                                           "meter profile parameter eir is not supported.");
 394         eir_man = man;
 395         eir_exp = exp;
 396         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 397         /* Check if ebs mantissa is too large. */
 398         if (exp > ASO_DSEG_EXP_MASK)
 399                 return -rte_mtr_error_set(error, ENOTSUP,
 400                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 401                                           "meter profile parameter ebs is not supported.");
 402         ebs_man = man;
 403         ebs_exp = exp;
 404         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 405                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET |
 406                                           eir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 407                                           eir_man);
 408         if (srtcm->cbs_cir)
 409                 fmp->g_support = 1;
 410         if (srtcm->ebs_eir)
 411                 fmp->y_support = 1;
 412         return 0;
 413 }
 414
 415 /**
 416  * Callback to get MTR capabilities.
 417  *
 418  * @param[in] dev
 419  *   Pointer to Ethernet device.
 420  * @param[out] cap
 421  *   Pointer to save MTR capabilities.
 422  * @param[out] error
 423  *   Pointer to the error structure.
 424  *
 425  * @return
 426  *   0 on success, a negative errno value otherwise and rte_errno is set.
 427  */
 428 static int
 429 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 430                  struct rte_mtr_capabilities *cap,
 431                  struct rte_mtr_error *error __rte_unused)
 432 {
 433         struct mlx5_priv *priv = dev->data->dev_private;
 434         struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos;
 435
 436         if (!priv->mtr_en)
 437                 return -rte_mtr_error_set(error, ENOTSUP,
 438                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 439                                           "Meter is not supported");
 440         memset(cap, 0, sizeof(*cap));
 441         if (priv->sh->meter_aso_en) {
 442                 /* 2 meters per one ASO cache line. */
 443                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 444                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 445                 cap->trtcm_rfc2698_packet_mode_supported = 1;
 446                 cap->trtcm_rfc4115_packet_mode_supported = 1;
 447         } else {
 448                 cap->n_max = 1 << qattr->log_max_flow_meter;
 449         }
 450         cap->srtcm_rfc2697_byte_mode_supported = 1;
 451         cap->trtcm_rfc2698_byte_mode_supported = 1;
 452         cap->trtcm_rfc4115_byte_mode_supported = 1;
 453         cap->n_shared_max = cap->n_max;
 454         cap->identical = 1;
 455         cap->shared_identical = 1;
 456         cap->shared_n_flows_per_mtr_max = 4 << 20;
 457         /* 2M flows can share the same meter. */
 458         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 459         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 460         cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 461         cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 462         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 463         cap->meter_policy_n_max = cap->n_max;
 464         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 465                           RTE_MTR_STATS_N_PKTS_DROPPED;
 466         return 0;
 467 }
 468
 469 /**
 470  * Callback to add MTR profile.
 471  *
 472  * @param[in] dev
 473  *   Pointer to Ethernet device.
 474  * @param[in] meter_profile_id
 475  *   Meter profile id.
 476  * @param[in] profile
 477  *   Pointer to meter profile detail.
 478  * @param[out] error
 479  *   Pointer to the error structure.
 480  *
 481  * @return
 482  *   0 on success, a negative errno value otherwise and rte_errno is set.
 483  */
 484 static int
 485 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 486                        uint32_t meter_profile_id,
 487                        struct rte_mtr_meter_profile *profile,
 488                        struct rte_mtr_error *error)
 489 {
 490         struct mlx5_priv *priv = dev->data->dev_private;
 491         struct mlx5_flow_meter_profile *fmp;
 492         union mlx5_l3t_data data;
 493         int ret;
 494
 495         if (!priv->mtr_en)
 496                 return -rte_mtr_error_set(error, ENOTSUP,
 497                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 498                                           "Meter is not supported");
 499         /* Check input params. */
 500         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 501                                                profile, error);
 502         if (ret)
 503                 return ret;
 504         /* Meter profile memory allocation. */
 505         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 506                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 507         if (fmp == NULL)
 508                 return -rte_mtr_error_set(error, ENOMEM,
 509                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 510                                           NULL, "Meter profile memory "
 511                                           "alloc failed.");
 512         /* Fill profile info. */
 513         fmp->id = meter_profile_id;
 514         fmp->profile = *profile;
 515         /* Fill the flow meter parameters for the PRM. */
 516         ret = mlx5_flow_meter_param_fill(fmp, error);
 517         if (ret)
 518                 goto error;
 519         data.ptr = fmp;
 520         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 521                                  meter_profile_id, &data);
 522         if (ret)
 523                 return -rte_mtr_error_set(error, ENOTSUP,
 524                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 525                                           NULL, "Meter profile insert fail.");
 526         return 0;
 527 error:
 528         mlx5_free(fmp);
 529         return ret;
 530 }
 531
 532 /**
 533  * Callback to delete MTR profile.
 534  *
 535  * @param[in] dev
 536  *   Pointer to Ethernet device.
 537  * @param[in] meter_profile_id
 538  *   Meter profile id.
 539  * @param[out] error
 540  *   Pointer to the error structure.
 541  *
 542  * @return
 543  *   0 on success, a negative errno value otherwise and rte_errno is set.
 544  */
 545 static int
 546 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 547                           uint32_t meter_profile_id,
 548                           struct rte_mtr_error *error)
 549 {
 550         struct mlx5_priv *priv = dev->data->dev_private;
 551         struct mlx5_flow_meter_profile *fmp;
 552
 553         if (!priv->mtr_en)
 554                 return -rte_mtr_error_set(error, ENOTSUP,
 555                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 556                                           "Meter is not supported");
 557         /* Meter profile must exist. */
 558         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 559         if (fmp == NULL)
 560                 return -rte_mtr_error_set(error, ENOENT,
 561                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 562                                           &meter_profile_id,
 563                                           "Meter profile id is invalid.");
 564         /* Check profile is unused. */
 565         if (fmp->ref_cnt)
 566                 return -rte_mtr_error_set(error, EBUSY,
 567                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 568                                           NULL, "Meter profile is in use.");
 569         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 570                 return -rte_mtr_error_set(error, EBUSY,
 571                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 572                                           NULL, "Meter profile remove fail.");
 573         mlx5_free(fmp);
 574         return 0;
 575 }
 576
 577 /**
 578  * Find policy by id.
 579  *
 580  * @param[in] dev
 581  *   Pointer to Ethernet device.
 582  * @param policy_id
 583  *   Policy id.
 584  *
 585  * @return
 586  *   Pointer to the policy found on success, NULL otherwise.
 587  */
 588 struct mlx5_flow_meter_policy *
 589 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 590                             uint32_t policy_id,
 591                             uint32_t *policy_idx)
 592 {
 593         struct mlx5_priv *priv = dev->data->dev_private;
 594         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 595         union mlx5_l3t_data data;
 596
 597         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 598                 return NULL;
 599         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 600                                 !data.dword)
 601                 return NULL;
 602         if (policy_idx)
 603                 *policy_idx = data.dword;
 604         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 605                                         data.dword);
 606         /* Remove reference taken by the mlx5_l3t_get_entry. */
 607         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 608         if (sub_policy)
 609                 if (sub_policy->main_policy_id)
 610                         return sub_policy->main_policy;
 611         return NULL;
 612 }
 613
 614 /**
 615  * Get the last meter's policy from one meter's policy in hierarchy.
 616  *
 617  * @param[in] dev
 618  *   Pointer to Ethernet device.
 619  * @param[in] policy
 620  *   Pointer to flow meter policy.
 621  *
 622  * @return
 623  *   Pointer to the final meter's policy, or NULL when fail.
 624  */
 625 struct mlx5_flow_meter_policy *
 626 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 627                                         struct mlx5_flow_meter_policy *policy)
 628 {
 629         struct mlx5_priv *priv = dev->data->dev_private;
 630         struct mlx5_flow_meter_info *next_fm;
 631         struct mlx5_flow_meter_policy *next_policy = policy;
 632
 633         while (next_policy->is_hierarchy) {
 634                 next_fm = mlx5_flow_meter_find(priv,
 635                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 636                 if (!next_fm || next_fm->def_policy)
 637                         return NULL;
 638                 next_policy = mlx5_flow_meter_policy_find(dev,
 639                                                 next_fm->policy_id, NULL);
 640                 MLX5_ASSERT(next_policy);
 641         }
 642         return next_policy;
 643 }
 644
 645 /**
 646  * Callback to check MTR policy action validate
 647  *
 648  * @param[in] dev
 649  *   Pointer to Ethernet device.
 650  * @param[in] actions
 651  *   Pointer to meter policy action detail.
 652  * @param[out] error
 653  *   Pointer to the error structure.
 654  *
 655  * @return
 656  *   0 on success, a negative errno value otherwise and rte_errno is set.
 657  */
 658 static int
 659 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 660         struct rte_mtr_meter_policy_params *policy,
 661         struct rte_mtr_error *error)
 662 {
 663         struct mlx5_priv *priv = dev->data->dev_private;
 664         struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ?
 665                                                   1 : 0 };
 666         bool is_rss = false;
 667         uint8_t policy_mode;
 668         uint8_t domain_bitmap;
 669         int ret;
 670
 671         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 672                 return -rte_mtr_error_set(error, ENOTSUP,
 673                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 674                                 NULL, "meter policy unsupported.");
 675         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 676                         &is_rss, &domain_bitmap, &policy_mode, error);
 677         if (ret)
 678                 return ret;
 679         return 0;
 680 }
 681
 682 static int
 683 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 684                         uint32_t policy_id,
 685                         struct mlx5_flow_meter_policy *mtr_policy,
 686                         struct rte_mtr_error *error,
 687                         bool clear_l3t)
 688 {
 689         struct mlx5_priv *priv = dev->data->dev_private;
 690         struct mlx5_flow_meter_sub_policy *sub_policy;
 691         uint32_t i, j;
 692         uint16_t sub_policy_num;
 693
 694         rte_spinlock_lock(&mtr_policy->sl);
 695         if (mtr_policy->ref_cnt) {
 696                 rte_spinlock_unlock(&mtr_policy->sl);
 697                 return -rte_mtr_error_set(error, EBUSY,
 698                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 699                                  NULL,
 700                                 "Meter policy object is being used.");
 701         }
 702         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 703         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 704         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 705                 sub_policy_num = (mtr_policy->sub_policy_num >>
 706                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 707                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 708                 if (sub_policy_num) {
 709                         for (j = 0; j < sub_policy_num; j++) {
 710                                 sub_policy = mtr_policy->sub_policys[i][j];
 711                                 if (sub_policy)
 712                                         mlx5_ipool_free
 713                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 714                                         sub_policy->idx);
 715                         }
 716                 }
 717         }
 718         if (priv->policy_idx_tbl && clear_l3t) {
 719                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 720                         rte_spinlock_unlock(&mtr_policy->sl);
 721                         return -rte_mtr_error_set(error, ENOTSUP,
 722                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 723                                 "Fail to delete policy in index table.");
 724                 }
 725         }
 726         rte_spinlock_unlock(&mtr_policy->sl);
 727         return 0;
 728 }
 729
 730 /**
 731  * Callback to add MTR policy.
 732  *
 733  * @param[in] dev
 734  *   Pointer to Ethernet device.
 735  * @param[out] policy_id
 736  *   Pointer to policy id
 737  * @param[in] actions
 738  *   Pointer to meter policy action detail.
 739  * @param[out] error
 740  *   Pointer to the error structure.
 741  *
 742  * @return
 743  *   0 on success, a negative errno value otherwise and rte_errno is set.
 744  */
 745 static int
 746 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 747                         uint32_t policy_id,
 748                         struct rte_mtr_meter_policy_params *policy,
 749                         struct rte_mtr_error *error)
 750 {
 751         struct mlx5_priv *priv = dev->data->dev_private;
 752         struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ?
 753                                                   1 : 0 };
 754         uint32_t sub_policy_idx = 0;
 755         uint32_t policy_idx = 0;
 756         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 757         struct mlx5_flow_meter_sub_policy *sub_policy;
 758         bool is_rss = false;
 759         uint8_t policy_mode;
 760         uint32_t i;
 761         int ret;
 762         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 763         uint16_t sub_policy_num;
 764         uint8_t domain_bitmap = 0;
 765         union mlx5_l3t_data data;
 766         bool skip_rule = false;
 767
 768         if (!priv->mtr_en)
 769                 return -rte_mtr_error_set(error, ENOTSUP,
 770                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 771                                           NULL, "meter policy unsupported. ");
 772         if (policy_id == MLX5_INVALID_POLICY_ID)
 773                 return -rte_mtr_error_set(error, ENOTSUP,
 774                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 775                                           NULL, "policy ID is invalid. ");
 776         if (policy_id == priv->sh->mtrmng->def_policy_id)
 777                 return -rte_mtr_error_set(error, EEXIST,
 778                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 779                                           NULL, "default policy ID exists. ");
 780         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 781         if (mtr_policy)
 782                 return -rte_mtr_error_set(error, EEXIST,
 783                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 784                                           NULL, "policy ID exists. ");
 785         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 786                                           &is_rss, &domain_bitmap,
 787                                           &policy_mode, error);
 788         if (ret)
 789                 return ret;
 790         if (!domain_bitmap)
 791                 return -rte_mtr_error_set(error, ENOTSUP,
 792                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 793                                           NULL, "fail to find policy domain.");
 794         if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
 795                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 796                         return -rte_mtr_error_set(error, EEXIST,
 797                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 798                                 NULL, "a policy with similar actions "
 799                                 "is already configured");
 800                 if (mlx5_flow_create_def_policy(dev))
 801                         return -rte_mtr_error_set(error, ENOTSUP,
 802                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 803                                 NULL,
 804                                 "fail to create non-terminated policy.");
 805                 priv->sh->mtrmng->def_policy_id = policy_id;
 806                 return 0;
 807         }
 808         if (!priv->sh->meter_aso_en)
 809                 return -rte_mtr_error_set(error, ENOTSUP,
 810                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 811                         "no ASO capability to support the policy ");
 812         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 813                 if (!(domain_bitmap & (1 << i)))
 814                         continue;
 815                 /*
 816                  * If RSS is found, it means that only the ingress domain can
 817                  * be supported. It is invalid to support RSS for one color
 818                  * and egress / transfer domain actions for another. Drop and
 819                  * jump action should have no impact.
 820                  */
 821                 if (is_rss) {
 822                         policy_size +=
 823                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 824                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 825                         break;
 826                 }
 827                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 828         }
 829         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 830                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 831         if (!mtr_policy)
 832                 return -rte_mtr_error_set(error, ENOMEM,
 833                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 834                                 "Memory alloc failed for meter policy.");
 835         if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
 836                 mtr_policy->skip_y = 1;
 837         else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
 838                 mtr_policy->skip_g = 1;
 839         policy_size = sizeof(struct mlx5_flow_meter_policy);
 840         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 841                 if (!(domain_bitmap & (1 << i)))
 842                         continue;
 843                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 844                         mtr_policy->ingress = 1;
 845                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 846                         mtr_policy->egress = 1;
 847                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 848                         mtr_policy->transfer = 1;
 849                 sub_policy = mlx5_ipool_zmalloc
 850                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 851                                  &sub_policy_idx);
 852                 if (!sub_policy ||
 853                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 854                         goto policy_add_err;
 855                 sub_policy->idx = sub_policy_idx;
 856                 sub_policy->main_policy = mtr_policy;
 857                 if (!policy_idx) {
 858                         policy_idx = sub_policy_idx;
 859                         sub_policy->main_policy_id = 1;
 860                 }
 861                 mtr_policy->sub_policys[i] =
 862                         (struct mlx5_flow_meter_sub_policy **)
 863                         ((uint8_t *)mtr_policy + policy_size);
 864                 mtr_policy->sub_policys[i][0] = sub_policy;
 865                 sub_policy_num = (mtr_policy->sub_policy_num >>
 866                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 867                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 868                 sub_policy_num++;
 869                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 870                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 871                 mtr_policy->sub_policy_num |=
 872                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 873                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 874                 /*
 875                  * If RSS is found, it means that only the ingress domain can
 876                  * be supported. It is invalid to support RSS for one color
 877                  * and egress / transfer domain actions for another. Drop and
 878                  * jump action should have no impact.
 879                  */
 880                 if (is_rss) {
 881                         mtr_policy->is_rss = 1;
 882                         break;
 883                 }
 884                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 885         }
 886         rte_spinlock_init(&mtr_policy->sl);
 887         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 888                                         policy->actions, error);
 889         if (ret)
 890                 goto policy_add_err;
 891         if (mtr_policy->is_hierarchy) {
 892                 struct mlx5_flow_meter_policy *final_policy;
 893
 894                 final_policy =
 895                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 896                 if (!final_policy)
 897                         goto policy_add_err;
 898                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 899         }
 900         /*
 901          * If either Green or Yellow has queue / RSS action, all the policy
 902          * rules will be created later in the flow splitting stage.
 903          */
 904         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 905                 /* Create policy rules in HW. */
 906                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 907                 if (ret)
 908                         goto policy_add_err;
 909         }
 910         data.dword = policy_idx;
 911         if (!priv->policy_idx_tbl) {
 912                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 913                 if (!priv->policy_idx_tbl)
 914                         goto policy_add_err;
 915         }
 916         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 917                 goto policy_add_err;
 918         return 0;
 919 policy_add_err:
 920         if (mtr_policy) {
 921                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 922                         mtr_policy, error, false);
 923                 mlx5_free(mtr_policy);
 924                 if (ret)
 925                         return ret;
 926         }
 927         return -rte_mtr_error_set(error, ENOTSUP,
 928                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 929                                   NULL, "Failed to create devx policy.");
 930 }
 931
 932 /**
 933  * Callback to delete MTR policy.
 934  *
 935  * @param[in] dev
 936  *   Pointer to Ethernet device.
 937  * @param[in] policy_id
 938  *   Meter policy id.
 939  * @param[out] error
 940  *   Pointer to the error structure.
 941  *
 942  * @return
 943  *   0 on success, a negative errno value otherwise and rte_errno is set.
 944  */
 945 static int
 946 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 947                           uint32_t policy_id,
 948                           struct rte_mtr_error *error)
 949 {
 950         struct mlx5_priv *priv = dev->data->dev_private;
 951         struct mlx5_flow_meter_policy *mtr_policy;
 952         uint32_t policy_idx;
 953         int ret;
 954
 955         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 956                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 957                         return -rte_mtr_error_set(error, ENOTSUP,
 958                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 959                                 "Meter policy object is being used.");
 960                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 961                 return 0;
 962         }
 963         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 964         if (!mtr_policy)
 965                 return -rte_mtr_error_set(error, ENOTSUP,
 966                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 967                         "Meter policy id is invalid. ");
 968         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 969                                                 error, true);
 970         if (ret)
 971                 return ret;
 972         mlx5_free(mtr_policy);
 973         return 0;
 974 }
 975
 976 /**
 977  * Check meter validation.
 978  *
 979  * @param[in] priv
 980  *   Pointer to mlx5 private data structure.
 981  * @param[in] meter_id
 982  *   Meter id.
 983  * @param[in] params
 984  *   Pointer to rte meter parameters.
 985  * @param[out] error
 986  *   Pointer to rte meter error structure.
 987  *
 988  * @return
 989  *   0 on success, a negative errno value otherwise and rte_errno is set.
 990  */
 991 static int
 992 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 993                          struct rte_mtr_params *params,
 994                          struct rte_mtr_error *error)
 995 {
 996         /* Meter must use global drop action. */
 997         if (!priv->sh->dr_drop_action)
 998                 return -rte_mtr_error_set(error, ENOTSUP,
 999                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1000                                           NULL,
1001                                           "No drop action ready for meter.");
1002         /* Meter params must not be NULL. */
1003         if (params == NULL)
1004                 return -rte_mtr_error_set(error, EINVAL,
1005                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1006                                           NULL, "Meter object params null.");
1007         /* Previous meter color is not supported. */
1008         if (params->use_prev_mtr_color)
1009                 return -rte_mtr_error_set(error, ENOTSUP,
1010                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1011                                           NULL,
1012                                           "Previous meter color "
1013                                           "not supported.");
1014         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
1015                 return -rte_mtr_error_set(error, ENOENT,
1016                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1017                                 NULL, "Meter policy id not valid.");
1018         /* Validate meter id. */
1019         if (mlx5_flow_meter_find(priv, meter_id, NULL))
1020                 return -rte_mtr_error_set(error, EEXIST,
1021                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
1022                         "Meter object already exists.");
1023         return 0;
1024 }
1025
1026 /**
1027  * Modify the flow meter action.
1028  *
1029  * @param[in] priv
1030  *   Pointer to mlx5 private data structure.
1031  * @param[in] fm
1032  *   Pointer to flow meter to be modified.
1033  * @param[in] srtcm
1034  *   Pointer to meter srtcm description parameter.
1035  * @param[in] modify_bits
1036  *   The bit in srtcm to be updated.
1037  * @param[in] active_state
1038  *   The state to be updated.
1039  * @return
1040  *   0 on success, o negative value otherwise.
1041  */
1042 static int
1043 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
1044                 struct mlx5_flow_meter_info *fm,
1045                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
1046                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
1047 {
1048 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1049         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
1050         uint32_t *attr;
1051         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
1052         int ret;
1053         struct mlx5_aso_mtr *aso_mtr = NULL;
1054         uint32_t cbs_cir, ebs_eir, val;
1055
1056         if (priv->sh->meter_aso_en) {
1057                 fm->is_enable = !!is_enable;
1058                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1059                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1060                 if (ret)
1061                         return ret;
1062                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
1063                 if (ret)
1064                         return ret;
1065         } else {
1066                 /* Fill command parameters. */
1067                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
1068                 mod_attr.flow_meter_parameter = in;
1069                 mod_attr.flow_meter_parameter_sz =
1070                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
1071                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1072                         mod_attr.active = !!active_state;
1073                 else
1074                         mod_attr.active = 0;
1075                 attr = in;
1076                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
1077                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1078                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1079                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1080                                 ASO_DSEG_EXP_MASK;
1081                         MLX5_SET(flow_meter_parameters, attr,
1082                                 cbs_exponent, val);
1083                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1084                                 ASO_DSEG_MAN_MASK;
1085                         MLX5_SET(flow_meter_parameters, attr,
1086                                 cbs_mantissa, val);
1087                 }
1088                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1089                         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) &
1090                                 ASO_DSEG_EXP_MASK;
1091                         MLX5_SET(flow_meter_parameters, attr,
1092                                 cir_exponent, val);
1093                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1094                         MLX5_SET(flow_meter_parameters, attr,
1095                                 cir_mantissa, val);
1096                 }
1097                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1098                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1099                                 ASO_DSEG_EXP_MASK;
1100                         MLX5_SET(flow_meter_parameters, attr,
1101                                 ebs_exponent, val);
1102                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1103                                 ASO_DSEG_MAN_MASK;
1104                         MLX5_SET(flow_meter_parameters, attr,
1105                                 ebs_mantissa, val);
1106                 }
1107                 /* Apply modifications to meter only if it was created. */
1108                 if (fm->meter_action) {
1109                         ret = mlx5_glue->dv_modify_flow_action_meter
1110                                         (fm->meter_action, &mod_attr,
1111                                         rte_cpu_to_be_64(modify_bits));
1112                         if (ret)
1113                                 return ret;
1114                 }
1115                 /* Update succeeded modify meter parameters. */
1116                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1117                         fm->active_state = !!active_state;
1118         }
1119         return 0;
1120 #else
1121         (void)priv;
1122         (void)fm;
1123         (void)srtcm;
1124         (void)modify_bits;
1125         (void)active_state;
1126         (void)is_enable;
1127         return -ENOTSUP;
1128 #endif
1129 }
1130
1131 static int
1132 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1133                                 struct mlx5_flow_meter_info *fm,
1134                                 uint64_t stats_mask)
1135 {
1136         fm->bytes_dropped =
1137                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1138         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1139         if (fm->bytes_dropped || fm->pkts_dropped) {
1140                 if (!fm->drop_cnt) {
1141                         /* Alloc policer counters. */
1142                         fm->drop_cnt = mlx5_counter_alloc(dev);
1143                         if (!fm->drop_cnt)
1144                                 return -1;
1145                 }
1146         } else {
1147                 if (fm->drop_cnt) {
1148                         mlx5_counter_free(dev, fm->drop_cnt);
1149                         fm->drop_cnt = 0;
1150                 }
1151         }
1152         return 0;
1153 }
1154
1155 /**
1156  * Create meter rules.
1157  *
1158  * @param[in] dev
1159  *   Pointer to Ethernet device.
1160  * @param[in] meter_id
1161  *   Meter id.
1162  * @param[in] params
1163  *   Pointer to rte meter parameters.
1164  * @param[in] shared
1165  *   Meter shared with other flow or not.
1166  * @param[out] error
1167  *   Pointer to rte meter error structure.
1168  *
1169  * @return
1170  *   0 on success, a negative errno value otherwise and rte_errno is set.
1171  */
1172 static int
1173 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1174                        struct rte_mtr_params *params, int shared,
1175                        struct rte_mtr_error *error)
1176 {
1177         struct mlx5_priv *priv = dev->data->dev_private;
1178         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1179         struct mlx5_flow_meter_profile *fmp;
1180         struct mlx5_flow_meter_info *fm;
1181         /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */
1182         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1183         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1184         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1185                 .size = 0,
1186                 .trunk_size = 64,
1187                 .need_lock = 1,
1188                 .type = "mlx5_flow_mtr_flow_id_pool",
1189         };
1190         struct mlx5_aso_mtr *aso_mtr;
1191         uint32_t mtr_idx, policy_idx;
1192         union mlx5_l3t_data data;
1193         int ret;
1194         uint8_t domain_bitmap;
1195         uint8_t mtr_id_bits;
1196         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1197                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1198
1199         if (!priv->mtr_en)
1200                 return -rte_mtr_error_set(error, ENOTSUP,
1201                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1202                                         "Meter is not supported");
1203         /* Validate the parameters. */
1204         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1205         if (ret)
1206                 return ret;
1207         /* Meter profile must exist. */
1208         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1209         if (fmp == NULL)
1210                 return -rte_mtr_error_set(error, ENOENT,
1211                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1212                         NULL, "Meter profile id not valid.");
1213         /* Meter policy must exist. */
1214         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1215                 __atomic_add_fetch
1216                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1217                         1, __ATOMIC_RELAXED);
1218                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1219                 if (!priv->sh->config.dv_esw_en)
1220                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1221         } else {
1222                 if (!priv->sh->meter_aso_en)
1223                         return -rte_mtr_error_set(error, ENOTSUP,
1224                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1225                                 "Part of the policies cannot be "
1226                                 "supported without ASO ");
1227                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1228                                 params->meter_policy_id, &policy_idx);
1229                 if (!mtr_policy)
1230                         return -rte_mtr_error_set(error, ENOENT,
1231                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1232                                 NULL, "Meter policy id not valid.");
1233                 domain_bitmap = (mtr_policy->ingress ?
1234                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1235                                 (mtr_policy->egress ?
1236                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1237                                 (mtr_policy->transfer ?
1238                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1239                 if (fmp->g_support && mtr_policy->skip_g)
1240                         return -rte_mtr_error_set(error, ENOTSUP,
1241                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1242                                         NULL, "Meter green policy is empty.");
1243                 if (fmp->y_support && mtr_policy->skip_y)
1244                         return -rte_mtr_error_set(error, ENOTSUP,
1245                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1246                                         NULL, "Meter yellow policy is empty.");
1247         }
1248         /* Allocate the flow meter memory. */
1249         if (priv->sh->meter_aso_en) {
1250                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1251                 if (!mtr_idx)
1252                         return -rte_mtr_error_set(error, ENOMEM,
1253                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1254                                 "Memory alloc failed for meter.");
1255                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1256                 fm = &aso_mtr->fm;
1257         } else {
1258                 if (fmp->y_support)
1259                         return -rte_mtr_error_set(error, ENOMEM,
1260                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1261                                 "Unsupported profile with yellow.");
1262                 legacy_fm = mlx5_ipool_zmalloc
1263                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1264                 if (legacy_fm == NULL)
1265                         return -rte_mtr_error_set(error, ENOMEM,
1266                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1267                                 "Memory alloc failed for meter.");
1268                 legacy_fm->idx = mtr_idx;
1269                 fm = &legacy_fm->fm;
1270         }
1271         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1272         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1273             mtr_reg_bits) {
1274                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1275                 goto error;
1276         }
1277         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1278                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1279         /* Fill the flow meter parameters. */
1280         fm->meter_id = meter_id;
1281         fm->policy_id = params->meter_policy_id;
1282         fm->profile = fmp;
1283         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1284                 goto error;
1285         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1286                 goto error;
1287         /* Add to the flow meter list. */
1288         if (!priv->sh->meter_aso_en) {
1289                 MLX5_ASSERT(legacy_fm != NULL);
1290                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1291         }
1292         /* Add to the flow meter list. */
1293         fm->active_state = 1; /* Config meter starts as active. */
1294         fm->is_enable = params->meter_enable;
1295         fm->shared = !!shared;
1296         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1297         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1298                 fm->def_policy = 1;
1299                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1300                 if (!fm->flow_ipool)
1301                         goto error;
1302         }
1303         rte_spinlock_init(&fm->sl);
1304         /* If ASO meter supported, update ASO flow meter by wqe. */
1305         if (priv->sh->meter_aso_en) {
1306                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1307                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1308                 if (ret)
1309                         goto error;
1310                 if (!priv->mtr_idx_tbl) {
1311                         priv->mtr_idx_tbl =
1312                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1313                         if (!priv->mtr_idx_tbl)
1314                                 goto error;
1315                 }
1316                 data.dword = mtr_idx;
1317                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1318                         goto error;
1319         } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) {
1320                 goto error;
1321         }
1322         fm->active_state = params->meter_enable;
1323         if (mtr_policy)
1324                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1325         return 0;
1326 error:
1327         mlx5_flow_destroy_mtr_tbls(dev, fm);
1328         /* Free policer counters. */
1329         if (fm->drop_cnt)
1330                 mlx5_counter_free(dev, fm->drop_cnt);
1331         if (priv->sh->meter_aso_en)
1332                 mlx5_flow_mtr_free(dev, mtr_idx);
1333         else
1334                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1335         return -rte_mtr_error_set(error, ENOTSUP,
1336                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1337                 NULL, "Failed to create devx meter.");
1338 }
1339
1340 static int
1341 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1342                         struct mlx5_flow_meter_info *fm,
1343                         uint32_t mtr_idx)
1344 {
1345         struct mlx5_priv *priv = dev->data->dev_private;
1346         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1347         struct mlx5_flow_meter_profile *fmp;
1348         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1349         struct mlx5_flow_meter_policy *mtr_policy;
1350
1351         /* Meter object must not have any owner. */
1352         MLX5_ASSERT(!fm->ref_cnt);
1353         /* Get meter profile. */
1354         fmp = fm->profile;
1355         if (fmp == NULL)
1356                 return -1;
1357         /* Update dependencies. */
1358         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1359         fm->profile = NULL;
1360         /* Remove from list. */
1361         if (!priv->sh->meter_aso_en) {
1362                 legacy_fm = container_of(fm,
1363                         struct mlx5_legacy_flow_meter, fm);
1364                 TAILQ_REMOVE(fms, legacy_fm, next);
1365         }
1366         /* Free drop counters. */
1367         if (fm->drop_cnt)
1368                 mlx5_counter_free(dev, fm->drop_cnt);
1369         /* Free meter flow table. */
1370         if (fm->flow_ipool) {
1371                 mlx5_ipool_destroy(fm->flow_ipool);
1372                 fm->flow_ipool = 0;
1373         }
1374         mlx5_flow_destroy_mtr_tbls(dev, fm);
1375         if (fm->def_policy)
1376                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1377                                 1, __ATOMIC_RELAXED);
1378         if (priv->sh->meter_aso_en) {
1379                 if (!fm->def_policy) {
1380                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1381                                                 fm->policy_id, NULL);
1382                         if (mtr_policy)
1383                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1384                                                 1, __ATOMIC_RELAXED);
1385                         fm->policy_id = 0;
1386                 }
1387                 fm->def_policy = 0;
1388                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1389                         return -1;
1390                 mlx5_flow_mtr_free(dev, mtr_idx);
1391         } else {
1392                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1393                                         legacy_fm->idx);
1394         }
1395         return 0;
1396 }
1397
1398 /**
1399  * Destroy meter rules.
1400  *
1401  * @param[in] dev
1402  *   Pointer to Ethernet device.
1403  * @param[in] meter_id
1404  *   Meter id.
1405  * @param[out] error
1406  *   Pointer to rte meter error structure.
1407  *
1408  * @return
1409  *   0 on success, a negative errno value otherwise and rte_errno is set.
1410  */
1411 static int
1412 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1413                         struct rte_mtr_error *error)
1414 {
1415         struct mlx5_priv *priv = dev->data->dev_private;
1416         struct mlx5_flow_meter_info *fm;
1417         uint32_t mtr_idx = 0;
1418
1419         if (!priv->mtr_en)
1420                 return -rte_mtr_error_set(error, ENOTSUP,
1421                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1422                                           NULL,
1423                                           "Meter is not supported");
1424         /* Meter object must exist. */
1425         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1426         if (fm == NULL)
1427                 return -rte_mtr_error_set(error, ENOENT,
1428                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1429                                           NULL,
1430                                           "Meter object id not valid.");
1431         /* Meter object must not have any owner. */
1432         if (fm->ref_cnt > 0)
1433                 return -rte_mtr_error_set(error, EBUSY,
1434                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1435                                           NULL,
1436                                           "Meter object is being used.");
1437         /* Destroy the meter profile. */
1438         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1439                 return -rte_mtr_error_set(error, EINVAL,
1440                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1441                                         NULL,
1442                                         "MTR object meter profile invalid.");
1443         return 0;
1444 }
1445
1446 /**
1447  * Modify meter state.
1448  *
1449  * @param[in] priv
1450  *   Pointer to mlx5 private data structure.
1451  * @param[in] fm
1452  *   Pointer to flow meter.
1453  * @param[in] new_state
1454  *   New state to update.
1455  * @param[out] error
1456  *   Pointer to rte meter error structure.
1457  *
1458  * @return
1459  *   0 on success, a negative errno value otherwise and rte_errno is set.
1460  */
1461 static int
1462 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1463                              struct mlx5_flow_meter_info *fm,
1464                              uint32_t new_state,
1465                              struct rte_mtr_error *error)
1466 {
1467         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1468                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1469                 .ebs_eir = 0,
1470         };
1471         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1472                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1473         int ret;
1474
1475         if (new_state == MLX5_FLOW_METER_DISABLE)
1476                 ret = mlx5_flow_meter_action_modify(priv, fm,
1477                                 &srtcm, modify_bits, 0, 0);
1478         else
1479                 ret = mlx5_flow_meter_action_modify(priv, fm,
1480                                                     &fm->profile->srtcm_prm,
1481                                                     modify_bits, 0, 1);
1482         if (ret)
1483                 return -rte_mtr_error_set(error, -ret,
1484                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1485                                           NULL,
1486                                           new_state ?
1487                                           "Failed to enable meter." :
1488                                           "Failed to disable meter.");
1489         return 0;
1490 }
1491
1492 /**
1493  * Callback to enable flow meter.
1494  *
1495  * @param[in] dev
1496  *   Pointer to Ethernet device.
1497  * @param[in] meter_id
1498  *   Meter id.
1499  * @param[out] error
1500  *   Pointer to rte meter error structure.
1501  *
1502  * @return
1503  *   0 on success, a negative errno value otherwise and rte_errno is set.
1504  */
1505 static int
1506 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1507                        uint32_t meter_id,
1508                        struct rte_mtr_error *error)
1509 {
1510         struct mlx5_priv *priv = dev->data->dev_private;
1511         struct mlx5_flow_meter_info *fm;
1512         int ret;
1513
1514         if (!priv->mtr_en)
1515                 return -rte_mtr_error_set(error, ENOTSUP,
1516                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1517                                           "Meter is not supported");
1518         /* Meter object must exist. */
1519         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1520         if (fm == NULL)
1521                 return -rte_mtr_error_set(error, ENOENT,
1522                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1523                                           NULL, "Meter not found.");
1524         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1525                 return 0;
1526         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1527                                            error);
1528         if (!ret)
1529                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1530         return ret;
1531 }
1532
1533 /**
1534  * Callback to disable flow meter.
1535  *
1536  * @param[in] dev
1537  *   Pointer to Ethernet device.
1538  * @param[in] meter_id
1539  *   Meter id.
1540  * @param[out] error
1541  *   Pointer to rte meter error structure.
1542  *
1543  * @return
1544  *   0 on success, a negative errno value otherwise and rte_errno is set.
1545  */
1546 static int
1547 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1548                         uint32_t meter_id,
1549                         struct rte_mtr_error *error)
1550 {
1551         struct mlx5_priv *priv = dev->data->dev_private;
1552         struct mlx5_flow_meter_info *fm;
1553         int ret;
1554
1555         if (!priv->mtr_en)
1556                 return -rte_mtr_error_set(error, ENOTSUP,
1557                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1558                                           "Meter is not supported");
1559         /* Meter object must exist. */
1560         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1561         if (fm == NULL)
1562                 return -rte_mtr_error_set(error, ENOENT,
1563                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1564                                           NULL, "Meter not found.");
1565         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1566                 return 0;
1567         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1568                                            error);
1569         if (!ret)
1570                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1571         return ret;
1572 }
1573
1574 /**
1575  * Callback to update meter profile.
1576  *
1577  * @param[in] dev
1578  *   Pointer to Ethernet device.
1579  * @param[in] meter_id
1580  *   Meter id.
1581  * @param[in] meter_profile_id
1582  *   To be updated meter profile id.
1583  * @param[out] error
1584  *   Pointer to rte meter error structure.
1585  *
1586  * @return
1587  *   0 on success, a negative errno value otherwise and rte_errno is set.
1588  */
1589 static int
1590 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1591                                uint32_t meter_id,
1592                                uint32_t meter_profile_id,
1593                                struct rte_mtr_error *error)
1594 {
1595         struct mlx5_priv *priv = dev->data->dev_private;
1596         struct mlx5_flow_meter_profile *fmp;
1597         struct mlx5_flow_meter_profile *old_fmp;
1598         struct mlx5_flow_meter_info *fm;
1599         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1600                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1601         int ret;
1602
1603         if (!priv->mtr_en)
1604                 return -rte_mtr_error_set(error, ENOTSUP,
1605                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1606                                           "Meter is not supported");
1607         /* Meter profile must exist. */
1608         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1609         if (fmp == NULL)
1610                 return -rte_mtr_error_set(error, ENOENT,
1611                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1612                                           NULL, "Meter profile not found.");
1613         /* Meter object must exist. */
1614         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1615         if (fm == NULL)
1616                 return -rte_mtr_error_set(error, ENOENT,
1617                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1618                                           NULL, "Meter not found.");
1619         /* MTR object already set to meter profile id. */
1620         old_fmp = fm->profile;
1621         if (fmp == old_fmp)
1622                 return 0;
1623         /* Update the profile. */
1624         fm->profile = fmp;
1625         /* Update meter params in HW (if not disabled). */
1626         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1627                 return 0;
1628         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1629                                               modify_bits, fm->active_state, 1);
1630         if (ret) {
1631                 fm->profile = old_fmp;
1632                 return -rte_mtr_error_set(error, -ret,
1633                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1634                                           NULL, "Failed to update meter"
1635                                           " parameters in hardware.");
1636         }
1637         old_fmp->ref_cnt--;
1638         fmp->ref_cnt++;
1639         return 0;
1640 }
1641
1642 /**
1643  * Callback to update meter stats mask.
1644  *
1645  * @param[in] dev
1646  *   Pointer to Ethernet device.
1647  * @param[in] meter_id
1648  *   Meter id.
1649  * @param[in] stats_mask
1650  *   To be updated stats_mask.
1651  * @param[out] error
1652  *   Pointer to rte meter error structure.
1653  *
1654  * @return
1655  *   0 on success, a negative errno value otherwise and rte_errno is set.
1656  */
1657 static int
1658 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1659                              uint32_t meter_id,
1660                              uint64_t stats_mask,
1661                              struct rte_mtr_error *error)
1662 {
1663         struct mlx5_priv *priv = dev->data->dev_private;
1664         struct mlx5_flow_meter_info *fm;
1665
1666         if (!priv->mtr_en)
1667                 return -rte_mtr_error_set(error, ENOTSUP,
1668                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1669                                           "Meter is not supported");
1670         /* Meter object must exist. */
1671         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1672         if (fm == NULL)
1673                 return -rte_mtr_error_set(error, ENOENT,
1674                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1675                                           NULL, "Meter object id not valid.");
1676         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1677                 return -rte_mtr_error_set(error, ENOENT,
1678                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1679                                           NULL, "Fail to allocate "
1680                                           "counter for meter.");
1681         return 0;
1682 }
1683
1684 /**
1685  * Callback to read meter statistics.
1686  *
1687  * @param[in] dev
1688  *   Pointer to Ethernet device.
1689  * @param[in] meter_id
1690  *   Meter id.
1691  * @param[out] stats
1692  *   Pointer to store the statistics.
1693  * @param[out] stats_mask
1694  *   Pointer to store the stats_mask.
1695  * @param[in] clear
1696  *   Statistic to be cleared after read or not.
1697  * @param[out] error
1698  *   Pointer to rte meter error structure.
1699  *
1700  * @return
1701  *   0 on success, a negative errno value otherwise and rte_errno is set.
1702  */
1703 static int
1704 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1705                            uint32_t meter_id,
1706                            struct rte_mtr_stats *stats,
1707                            uint64_t *stats_mask,
1708                            int clear,
1709                            struct rte_mtr_error *error)
1710 {
1711         struct mlx5_priv *priv = dev->data->dev_private;
1712         struct mlx5_flow_meter_info *fm;
1713         uint64_t pkts;
1714         uint64_t bytes;
1715         int ret = 0;
1716
1717         if (!priv->mtr_en)
1718                 return -rte_mtr_error_set(error, ENOTSUP,
1719                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1720                                           "Meter is not supported");
1721         /* Meter object must exist. */
1722         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1723         if (fm == NULL)
1724                 return -rte_mtr_error_set(error, ENOENT,
1725                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1726                                           NULL, "Meter object id not valid.");
1727         *stats_mask = 0;
1728         if (fm->bytes_dropped)
1729                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1730         if (fm->pkts_dropped)
1731                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1732         memset(stats, 0, sizeof(*stats));
1733         if (fm->drop_cnt) {
1734                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1735                                                  &bytes, NULL);
1736                 if (ret)
1737                         goto error;
1738                 /* If need to read the packets, set it. */
1739                 if (fm->pkts_dropped)
1740                         stats->n_pkts_dropped = pkts;
1741                 /* If need to read the bytes, set it. */
1742                 if (fm->bytes_dropped)
1743                         stats->n_bytes_dropped = bytes;
1744         }
1745         return 0;
1746 error:
1747         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1748                                  "Failed to read meter drop counters.");
1749 }
1750
1751 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1752         .capabilities_get = mlx5_flow_mtr_cap_get,
1753         .meter_profile_add = mlx5_flow_meter_profile_add,
1754         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1755         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1756         .meter_policy_add = mlx5_flow_meter_policy_add,
1757         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1758         .create = mlx5_flow_meter_create,
1759         .destroy = mlx5_flow_meter_destroy,
1760         .meter_enable = mlx5_flow_meter_enable,
1761         .meter_disable = mlx5_flow_meter_disable,
1762         .meter_profile_update = mlx5_flow_meter_profile_update,
1763         .meter_dscp_table_update = NULL,
1764         .stats_update = mlx5_flow_meter_stats_update,
1765         .stats_read = mlx5_flow_meter_stats_read,
1766 };
1767
1768 /**
1769  * Get meter operations.
1770  *
1771  * @param dev
1772  *   Pointer to Ethernet device structure.
1773  * @param arg
1774  *   Pointer to set the mtr operations.
1775  *
1776  * @return
1777  *   Always 0.
1778  */
1779 int
1780 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1781 {
1782         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1783         return 0;
1784 }
1785
1786 /**
1787  * Find meter by id.
1788  *
1789  * @param priv
1790  *   Pointer to mlx5_priv.
1791  * @param meter_id
1792  *   Meter id.
1793  * @param mtr_idx
1794  *   Pointer to Meter index.
1795  *
1796  * @return
1797  *   Pointer to the meter info found on success, NULL otherwise.
1798  */
1799 struct mlx5_flow_meter_info *
1800 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1801                 uint32_t *mtr_idx)
1802 {
1803         struct mlx5_legacy_flow_meter *legacy_fm;
1804         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1805         struct mlx5_aso_mtr *aso_mtr;
1806         struct mlx5_aso_mtr_pools_mng *pools_mng =
1807                                 &priv->sh->mtrmng->pools_mng;
1808         union mlx5_l3t_data data;
1809         uint16_t n_valid;
1810
1811         if (priv->sh->meter_aso_en) {
1812                 rte_rwlock_read_lock(&pools_mng->resize_mtrwl);
1813                 n_valid = pools_mng->n_valid;
1814                 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl);
1815                 if (!n_valid || !priv->mtr_idx_tbl ||
1816                     (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1817                     !data.dword))
1818                         return NULL;
1819                 if (mtr_idx)
1820                         *mtr_idx = data.dword;
1821                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1822                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1823                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1824                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1825                         return NULL;
1826                 return &aso_mtr->fm;
1827         }
1828         TAILQ_FOREACH(legacy_fm, fms, next)
1829                 if (meter_id == legacy_fm->fm.meter_id) {
1830                         if (mtr_idx)
1831                                 *mtr_idx = legacy_fm->idx;
1832                         return &legacy_fm->fm;
1833                 }
1834         return NULL;
1835 }
1836
1837 /**
1838  * Find meter by index.
1839  *
1840  * @param priv
1841  *   Pointer to mlx5_priv.
1842  * @param idx
1843  *   Meter index.
1844  *
1845  * @return
1846  *   Pointer to the meter info found on success, NULL otherwise.
1847  */
1848 struct mlx5_flow_meter_info *
1849 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1850 {
1851         struct mlx5_aso_mtr *aso_mtr;
1852
1853         if (priv->sh->meter_aso_en) {
1854                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1855                 if (!aso_mtr)
1856                         return NULL;
1857                 return &aso_mtr->fm;
1858         } else {
1859                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1860         }
1861 }
1862
1863 /**
1864  * Attach meter to flow.
1865  * Unidirectional Meter creation can only be done
1866  * when flow direction is known, i.e. when calling meter_attach.
1867  *
1868  * @param [in] priv
1869  *  Pointer to mlx5 private data.
1870  * @param[in] fm
1871  *   Pointer to flow meter.
1872  * @param [in] attr
1873  *  Pointer to flow attributes.
1874  * @param [out] error
1875  *  Pointer to error structure.
1876  *
1877  * @return
1878  *   0 on success, a negative errno value otherwise and rte_errno is set.
1879  */
1880 int
1881 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1882                        struct mlx5_flow_meter_info *fm,
1883                        const struct rte_flow_attr *attr,
1884                        struct rte_flow_error *error)
1885 {
1886         int ret = 0;
1887
1888         if (priv->sh->meter_aso_en) {
1889                 struct mlx5_aso_mtr *aso_mtr;
1890
1891                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1892                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1893                         return rte_flow_error_set(error, ENOENT,
1894                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1895                                         NULL,
1896                                         "Timeout in meter configuration");
1897                 }
1898                 rte_spinlock_lock(&fm->sl);
1899                 if (fm->shared || !fm->ref_cnt) {
1900                         fm->ref_cnt++;
1901                 } else {
1902                         rte_flow_error_set(error, EINVAL,
1903                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1904                                    "Meter cannot be shared");
1905                         ret = -1;
1906                 }
1907                 rte_spinlock_unlock(&fm->sl);
1908         } else {
1909                 rte_spinlock_lock(&fm->sl);
1910                 if (fm->meter_action) {
1911                         if (fm->shared &&
1912                             attr->transfer == fm->transfer &&
1913                             attr->ingress == fm->ingress &&
1914                             attr->egress == fm->egress) {
1915                                 fm->ref_cnt++;
1916                         } else {
1917                                 rte_flow_error_set(error, EINVAL,
1918                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1919                                         fm->shared ?
1920                                         "Meter attr not match." :
1921                                         "Meter cannot be shared.");
1922                                 ret = -1;
1923                         }
1924                 } else {
1925                         fm->ingress = attr->ingress;
1926                         fm->egress = attr->egress;
1927                         fm->transfer = attr->transfer;
1928                         fm->ref_cnt = 1;
1929                         /* This also creates the meter object. */
1930                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1931                                                                          fm);
1932                         if (!fm->meter_action) {
1933                                 fm->ref_cnt = 0;
1934                                 fm->ingress = 0;
1935                                 fm->egress = 0;
1936                                 fm->transfer = 0;
1937                                 rte_flow_error_set(error, EINVAL,
1938                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1939                                         "Meter action create failed.");
1940                                 ret = -1;
1941                         }
1942                 }
1943                 rte_spinlock_unlock(&fm->sl);
1944         }
1945         return ret ? -rte_errno : 0;
1946 }
1947
1948 /**
1949  * Detach meter from flow.
1950  *
1951  * @param [in] priv
1952  *  Pointer to mlx5 private data.
1953  * @param [in] fm
1954  *  Pointer to flow meter.
1955  */
1956 void
1957 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1958                        struct mlx5_flow_meter_info *fm)
1959 {
1960 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1961         rte_spinlock_lock(&fm->sl);
1962         MLX5_ASSERT(fm->ref_cnt);
1963         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1964                 mlx5_glue->destroy_flow_action(fm->meter_action);
1965                 fm->meter_action = NULL;
1966                 fm->ingress = 0;
1967                 fm->egress = 0;
1968                 fm->transfer = 0;
1969         }
1970         rte_spinlock_unlock(&fm->sl);
1971 #else
1972         (void)priv;
1973         (void)fm;
1974 #endif
1975 }
1976
1977 /**
1978  * Flush meter with Rx queue configuration.
1979  *
1980  * @param[in] dev
1981  *   Pointer to Ethernet device.
1982  */
1983 void
1984 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1985 {
1986         struct mlx5_priv *priv = dev->data->dev_private;
1987         struct mlx5_flow_meter_sub_policy *sub_policy;
1988         struct mlx5_flow_meter_policy *mtr_policy;
1989         void *entry;
1990         uint32_t i, policy_idx;
1991
1992         if (!priv->mtr_en)
1993                 return;
1994         if (priv->policy_idx_tbl) {
1995                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1996                         policy_idx = *(uint32_t *)entry;
1997                         sub_policy = mlx5_ipool_get
1998                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1999                                 policy_idx);
2000                         if (!sub_policy || !sub_policy->main_policy)
2001                                 continue;
2002                         mtr_policy = sub_policy->main_policy;
2003                         if (mtr_policy->is_queue || mtr_policy->is_rss)
2004                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
2005                                         mtr_policy);
2006                 }
2007         }
2008 }
2009
2010 /**
2011  * Iterate a meter hierarchy and flush all meters and policies if possible.
2012  *
2013  * @param[in] dev
2014  *   Pointer to Ethernet device.
2015  * @param[in] fm
2016  *   Pointer to flow meter.
2017  * @param[in] mtr_idx
2018  *   .Meter's index
2019  * @param[out] error
2020  *   Pointer to rte meter error structure.
2021  *
2022  * @return
2023  *   0 on success, a negative errno value otherwise and rte_errno is set.
2024  */
2025 static int
2026 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
2027                                 struct mlx5_flow_meter_info *fm,
2028                                 uint32_t mtr_idx,
2029                                 struct rte_mtr_error *error)
2030 {
2031         struct mlx5_priv *priv = dev->data->dev_private;
2032         struct mlx5_flow_meter_policy *policy;
2033         uint32_t policy_id;
2034         struct mlx5_flow_meter_info *next_fm;
2035         uint32_t next_mtr_idx;
2036         struct mlx5_flow_meter_policy *next_policy = NULL;
2037
2038         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
2039         MLX5_ASSERT(policy);
2040         while (!fm->ref_cnt && policy->is_hierarchy) {
2041                 policy_id = fm->policy_id;
2042                 next_fm = mlx5_flow_meter_find(priv,
2043                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2044                                 &next_mtr_idx);
2045                 if (next_fm) {
2046                         next_policy = mlx5_flow_meter_policy_find(dev,
2047                                                         next_fm->policy_id,
2048                                                         NULL);
2049                         MLX5_ASSERT(next_policy);
2050                 }
2051                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
2052                         return -rte_mtr_error_set(error, ENOTSUP,
2053                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
2054                                                 NULL,
2055                                                 "Failed to flush meter.");
2056                 if (policy->ref_cnt)
2057                         break;
2058                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
2059                                                 policy, error, true))
2060                         return -rte_errno;
2061                 mlx5_free(policy);
2062                 if (!next_fm || !next_policy)
2063                         break;
2064                 fm = next_fm;
2065                 mtr_idx = next_mtr_idx;
2066                 policy = next_policy;
2067         }
2068         return 0;
2069 }
2070
2071 /**
2072  * Flush all the hierarchy meters and their policies.
2073  *
2074  * @param[in] dev
2075  *   Pointer to Ethernet device.
2076  * @param[out] error
2077  *   Pointer to rte meter error structure.
2078  *
2079  * @return
2080  *   0 on success, a negative errno value otherwise and rte_errno is set.
2081  */
2082 static int
2083 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
2084                                       struct rte_mtr_error *error)
2085 {
2086         struct mlx5_priv *priv = dev->data->dev_private;
2087         struct mlx5_flow_meter_info *fm;
2088         struct mlx5_flow_meter_policy *policy;
2089         struct mlx5_flow_meter_sub_policy *sub_policy;
2090         struct mlx5_flow_meter_info *next_fm;
2091         struct mlx5_aso_mtr *aso_mtr;
2092         uint32_t mtr_idx = 0;
2093         uint32_t i, policy_idx;
2094         void *entry;
2095
2096         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2097                 return 0;
2098         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2099                 mtr_idx = *(uint32_t *)entry;
2100                 if (!mtr_idx)
2101                         continue;
2102                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2103                 fm = &aso_mtr->fm;
2104                 if (fm->ref_cnt || fm->def_policy)
2105                         continue;
2106                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2107                         return -rte_errno;
2108         }
2109         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2110                 policy_idx = *(uint32_t *)entry;
2111                 sub_policy = mlx5_ipool_get
2112                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2113                                 policy_idx);
2114                 if (!sub_policy)
2115                         return -rte_mtr_error_set(error,
2116                                         EINVAL,
2117                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2118                                         NULL, "Meter policy invalid.");
2119                 policy = sub_policy->main_policy;
2120                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2121                         continue;
2122                 next_fm = mlx5_flow_meter_find(priv,
2123                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2124                                 &mtr_idx);
2125                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2126                                                     error, true))
2127                         return -rte_mtr_error_set(error,
2128                                         EINVAL,
2129                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2130                                         NULL, "Meter policy invalid.");
2131                 mlx5_free(policy);
2132                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2133                         continue;
2134                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2135                                                     mtr_idx, error))
2136                         return -rte_errno;
2137         }
2138         return 0;
2139 }
2140 /**
2141  * Flush meter configuration.
2142  *
2143  * @param[in] dev
2144  *   Pointer to Ethernet device.
2145  * @param[out] error
2146  *   Pointer to rte meter error structure.
2147  *
2148  * @return
2149  *   0 on success, a negative errno value otherwise and rte_errno is set.
2150  */
2151 int
2152 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2153 {
2154         struct mlx5_priv *priv = dev->data->dev_private;
2155         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2156         struct mlx5_flow_meter_profile *fmp;
2157         struct mlx5_legacy_flow_meter *legacy_fm;
2158         struct mlx5_flow_meter_info *fm;
2159         struct mlx5_flow_meter_sub_policy *sub_policy;
2160         void *tmp;
2161         uint32_t i, mtr_idx, policy_idx;
2162         void *entry;
2163         struct mlx5_aso_mtr *aso_mtr;
2164
2165         if (!priv->mtr_en)
2166                 return 0;
2167         if (priv->sh->meter_aso_en) {
2168                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2169                         return -rte_errno;
2170                 if (priv->mtr_idx_tbl) {
2171                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2172                                 mtr_idx = *(uint32_t *)entry;
2173                                 if (mtr_idx) {
2174                                         aso_mtr =
2175                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2176                                         fm = &aso_mtr->fm;
2177                                         (void)mlx5_flow_meter_params_flush(dev,
2178                                                 fm, mtr_idx);
2179                                 }
2180                         }
2181                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2182                         priv->mtr_idx_tbl = NULL;
2183                 }
2184         } else {
2185                 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2186                         fm = &legacy_fm->fm;
2187                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2188                                 return -rte_mtr_error_set(error, EINVAL,
2189                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2190                                 NULL, "MTR object meter profile invalid.");
2191                 }
2192         }
2193         if (priv->policy_idx_tbl) {
2194                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2195                         policy_idx = *(uint32_t *)entry;
2196                         sub_policy = mlx5_ipool_get
2197                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2198                                 policy_idx);
2199                         if (!sub_policy)
2200                                 return -rte_mtr_error_set(error,
2201                                                 EINVAL,
2202                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2203                                                 NULL, "MTR object "
2204                                                 "meter policy invalid.");
2205                         if (__mlx5_flow_meter_policy_delete(dev, i,
2206                                                 sub_policy->main_policy,
2207                                                 error, true))
2208                                 return -rte_mtr_error_set(error,
2209                                                 EINVAL,
2210                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2211                                                 NULL, "MTR object "
2212                                                 "meter policy invalid.");
2213                         mlx5_free(sub_policy->main_policy);
2214                 }
2215                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2216                 priv->policy_idx_tbl = NULL;
2217         }
2218         if (priv->mtr_profile_tbl) {
2219                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2220                         fmp = entry;
2221                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2222                                                            error))
2223                                 return -rte_mtr_error_set(error, EINVAL,
2224                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2225                                                 NULL, "Fail to destroy "
2226                                                 "meter profile.");
2227                 }
2228                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2229                 priv->mtr_profile_tbl = NULL;
2230         }
2231         /* Delete default policy table. */
2232         mlx5_flow_destroy_def_policy(dev);
2233         if (priv->sh->refcnt == 1)
2234                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2235         return 0;
2236 }