drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133         uint32_t ls_factor;
 134         int ret;
 135         uint64_t cir, cbs;
 136         uint64_t eir, ebs;
 137         uint64_t pir, pbs;
 138
 139         /* Profile must not be NULL. */
 140         if (profile == NULL)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 143                                           NULL, "Meter profile is null.");
 144         /* Meter profile ID must be valid. */
 145         if (meter_profile_id == UINT32_MAX)
 146                 return -rte_mtr_error_set(error, EINVAL,
 147                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 148                                           NULL, "Meter profile id not valid.");
 149         /* Meter profile must not exist. */
 150         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 151         if (fmp)
 152                 return -rte_mtr_error_set(error, EEXIST,
 153                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 154                                           NULL,
 155                                           "Meter profile already exists.");
 156         if (!priv->sh->meter_aso_en) {
 157                 /* Old version is even not supported. */
 158                 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old)
 159                         return -rte_mtr_error_set(error, ENOTSUP,
 160                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 161                                 NULL, "Metering is not supported.");
 162                 /* Old FW metering only supports srTCM. */
 163                 if (profile->alg != RTE_MTR_SRTCM_RFC2697) {
 164                         return -rte_mtr_error_set(error, ENOTSUP,
 165                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 166                                 NULL, "Metering algorithm is not supported.");
 167                 } else if (profile->srtcm_rfc2697.ebs) {
 168                         /* EBS is not supported for old metering. */
 169                         return -rte_mtr_error_set(error, ENOTSUP,
 170                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 171                                 NULL, "EBS is not supported.");
 172                 }
 173                 if (profile->packet_mode)
 174                         return -rte_mtr_error_set(error, ENOTSUP,
 175                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 176                                 "Metering algorithm packet mode is not supported.");
 177         }
 178         ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0;
 179         switch (profile->alg) {
 180         case RTE_MTR_SRTCM_RFC2697:
 181                 cir = profile->srtcm_rfc2697.cir << ls_factor;
 182                 cbs = profile->srtcm_rfc2697.cbs << ls_factor;
 183                 ebs = profile->srtcm_rfc2697.ebs << ls_factor;
 184                 /* EBS could be zero for old metering. */
 185                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 186                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 187                     ebs <= MLX5_SRTCM_XBS_MAX) {
 188                         ret = 0;
 189                 } else {
 190                         ret = -rte_mtr_error_set(error, ENOTSUP,
 191                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 192                                         "Profile values out of range.");
 193                 }
 194                 break;
 195         case RTE_MTR_TRTCM_RFC2698:
 196                 cir = profile->trtcm_rfc2698.cir << ls_factor;
 197                 cbs = profile->trtcm_rfc2698.cbs << ls_factor;
 198                 pir = profile->trtcm_rfc2698.pir << ls_factor;
 199                 pbs = profile->trtcm_rfc2698.pbs << ls_factor;
 200                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 201                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 202                     pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) &&
 203                     pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) {
 204                         ret = 0;
 205                 } else {
 206                         ret = -rte_mtr_error_set(error, ENOTSUP,
 207                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 208                                         "Profile values out of range.");
 209                 }
 210                 break;
 211         case RTE_MTR_TRTCM_RFC4115:
 212                 cir = profile->trtcm_rfc4115.cir << ls_factor;
 213                 cbs = profile->trtcm_rfc4115.cbs << ls_factor;
 214                 eir = profile->trtcm_rfc4115.eir << ls_factor;
 215                 ebs = profile->trtcm_rfc4115.ebs << ls_factor;
 216                 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
 217                     cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
 218                     eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) {
 219                         ret = 0;
 220                 } else {
 221                         ret = -rte_mtr_error_set(error, ENOTSUP,
 222                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 223                                         "Profile values out of range.");
 224                 }
 225                 break;
 226         default:
 227                 ret = -rte_mtr_error_set(error, ENOTSUP,
 228                                          RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 229                                          "Unknown metering algorithm.");
 230                 break;
 231         }
 232         return ret;
 233 }
 234
 235 /*
 236  * Calculate mantissa and exponent for cir / eir.
 237  *
 238  * @param[in] xir
 239  *   Value to be calculated.
 240  * @param[out] man
 241  *   Pointer to the mantissa.
 242  * @param[out] exp
 243  *   Pointer to the exp.
 244  */
 245 static inline void
 246 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp)
 247 {
 248         int64_t _xir;
 249         int64_t delta = INT64_MAX;
 250         uint8_t _man = 0;
 251         uint8_t _exp = 0;
 252         uint64_t m, e;
 253
 254         /* Special case xir == 0 ? both exp and mantissa are 0. */
 255         if (xir == 0) {
 256                 *man = 0;
 257                 *exp = 0;
 258                 return;
 259         }
 260         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 261                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 262                         _xir = (1000000000ULL * m) >> e;
 263                         if (llabs(xir - _xir) <= delta) {
 264                                 delta = llabs(xir - _xir);
 265                                 _man = m;
 266                                 _exp = e;
 267                         }
 268                 }
 269         }
 270         *man = _man;
 271         *exp = _exp;
 272 }
 273
 274 /*
 275  * Calculate mantissa and exponent for xbs.
 276  *
 277  * @param[in] xbs
 278  *   Value to be calculated.
 279  * @param[out] man
 280  *   Pointer to the mantissa.
 281  * @param[out] exp
 282  *   Pointer to the exp.
 283  */
 284 static inline void
 285 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 286 {
 287         int _exp;
 288         double _man;
 289
 290         /* Special case xbs == 0 ? both exp and mantissa are 0. */
 291         if (xbs == 0) {
 292                 *man = 0;
 293                 *exp = 0;
 294                 return;
 295         }
 296         /* xbs = xbs_mantissa * 2^xbs_exponent */
 297         _man = frexp(xbs, &_exp);
 298         if (_exp >= MLX5_MAN_WIDTH) {
 299                 _man = _man * pow(2, MLX5_MAN_WIDTH);
 300                 _exp = _exp - MLX5_MAN_WIDTH;
 301         }
 302         *man = (uint8_t)ceil(_man);
 303         *exp = _exp;
 304 }
 305
 306 /**
 307  * Fill the prm meter parameter.
 308  *
 309  * @param[in,out] fmp
 310  *   Pointer to meter profile to be converted.
 311  * @param[out] error
 312  *   Pointer to the error structure.
 313  *
 314  * @return
 315  *   0 on success, a negative errno value otherwise and rte_errno is set.
 316  */
 317 static int
 318 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 319                            struct rte_mtr_error *error)
 320 {
 321         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 322         uint8_t man, exp;
 323         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 324         uint32_t eir_exp, eir_man, ebs_exp, ebs_man;
 325         uint64_t cir, cbs, eir, ebs;
 326
 327         switch (fmp->profile.alg) {
 328         case RTE_MTR_SRTCM_RFC2697:
 329                 cir = fmp->profile.srtcm_rfc2697.cir;
 330                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 331                 eir = 0;
 332                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 333                 break;
 334         case RTE_MTR_TRTCM_RFC2698:
 335                 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir >
 336                             fmp->profile.trtcm_rfc2698.cir &&
 337                             fmp->profile.trtcm_rfc2698.pbs >
 338                             fmp->profile.trtcm_rfc2698.cbs);
 339                 cir = fmp->profile.trtcm_rfc2698.cir;
 340                 cbs = fmp->profile.trtcm_rfc2698.cbs;
 341                 /* EIR / EBS are filled with PIR / PBS. */
 342                 eir = fmp->profile.trtcm_rfc2698.pir;
 343                 ebs = fmp->profile.trtcm_rfc2698.pbs;
 344                 break;
 345         case RTE_MTR_TRTCM_RFC4115:
 346                 cir = fmp->profile.trtcm_rfc4115.cir;
 347                 cbs = fmp->profile.trtcm_rfc4115.cbs;
 348                 eir = fmp->profile.trtcm_rfc4115.eir;
 349                 ebs = fmp->profile.trtcm_rfc4115.ebs;
 350                 break;
 351         default:
 352                 return -rte_mtr_error_set(error, EINVAL,
 353                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 354                                 "Metering algorithm mode is invalid");
 355         }
 356         /* Adjust the values for PPS mode. */
 357         if (fmp->profile.packet_mode) {
 358                 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 359                 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 360                 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 361                 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 362         }
 363         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 364         mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp);
 365         /* Check if cir mantissa is too large. */
 366         if (exp > ASO_DSEG_XIR_EXP_MASK)
 367                 return -rte_mtr_error_set(error, ENOTSUP,
 368                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 369                                           "meter profile parameter cir is not supported.");
 370         cir_man = man;
 371         cir_exp = exp;
 372          /* cbs = cbs_mantissa * 2^cbs_exponent */
 373         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 374         /* Check if cbs mantissa is too large. */
 375         if (exp > ASO_DSEG_EXP_MASK)
 376                 return -rte_mtr_error_set(error, ENOTSUP,
 377                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 378                                           "meter profile parameter cbs is not supported.");
 379         cbs_man = man;
 380         cbs_exp = exp;
 381         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 382                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 383                                           cir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 384                                           cir_man);
 385         mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp);
 386         /* Check if eir mantissa is too large. */
 387         if (exp > ASO_DSEG_XIR_EXP_MASK)
 388                 return -rte_mtr_error_set(error, ENOTSUP,
 389                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 390                                           "meter profile parameter eir is not supported.");
 391         eir_man = man;
 392         eir_exp = exp;
 393         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 394         /* Check if ebs mantissa is too large. */
 395         if (exp > ASO_DSEG_EXP_MASK)
 396                 return -rte_mtr_error_set(error, ENOTSUP,
 397                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 398                                           "meter profile parameter ebs is not supported.");
 399         ebs_man = man;
 400         ebs_exp = exp;
 401         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 402                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET |
 403                                           eir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 404                                           eir_man);
 405         if (srtcm->cbs_cir)
 406                 fmp->g_support = 1;
 407         if (srtcm->ebs_eir)
 408                 fmp->y_support = 1;
 409         return 0;
 410 }
 411
 412 /**
 413  * Callback to get MTR capabilities.
 414  *
 415  * @param[in] dev
 416  *   Pointer to Ethernet device.
 417  * @param[out] cap
 418  *   Pointer to save MTR capabilities.
 419  * @param[out] error
 420  *   Pointer to the error structure.
 421  *
 422  * @return
 423  *   0 on success, a negative errno value otherwise and rte_errno is set.
 424  */
 425 static int
 426 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 427                  struct rte_mtr_capabilities *cap,
 428                  struct rte_mtr_error *error __rte_unused)
 429 {
 430         struct mlx5_priv *priv = dev->data->dev_private;
 431         struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos;
 432
 433         if (!priv->mtr_en)
 434                 return -rte_mtr_error_set(error, ENOTSUP,
 435                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 436                                           "Meter is not supported");
 437         memset(cap, 0, sizeof(*cap));
 438         if (priv->sh->meter_aso_en) {
 439                 /* 2 meters per one ASO cache line. */
 440                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 441                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 442                 cap->trtcm_rfc2698_packet_mode_supported = 1;
 443                 cap->trtcm_rfc4115_packet_mode_supported = 1;
 444         } else {
 445                 cap->n_max = 1 << qattr->log_max_flow_meter;
 446         }
 447         cap->srtcm_rfc2697_byte_mode_supported = 1;
 448         cap->trtcm_rfc2698_byte_mode_supported = 1;
 449         cap->trtcm_rfc4115_byte_mode_supported = 1;
 450         cap->n_shared_max = cap->n_max;
 451         cap->identical = 1;
 452         cap->shared_identical = 1;
 453         cap->shared_n_flows_per_mtr_max = 4 << 20;
 454         /* 2M flows can share the same meter. */
 455         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 456         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 457         cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 458         cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 459         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 460         cap->meter_policy_n_max = cap->n_max;
 461         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 462                           RTE_MTR_STATS_N_PKTS_DROPPED;
 463         return 0;
 464 }
 465
 466 /**
 467  * Callback to add MTR profile.
 468  *
 469  * @param[in] dev
 470  *   Pointer to Ethernet device.
 471  * @param[in] meter_profile_id
 472  *   Meter profile id.
 473  * @param[in] profile
 474  *   Pointer to meter profile detail.
 475  * @param[out] error
 476  *   Pointer to the error structure.
 477  *
 478  * @return
 479  *   0 on success, a negative errno value otherwise and rte_errno is set.
 480  */
 481 static int
 482 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 483                        uint32_t meter_profile_id,
 484                        struct rte_mtr_meter_profile *profile,
 485                        struct rte_mtr_error *error)
 486 {
 487         struct mlx5_priv *priv = dev->data->dev_private;
 488         struct mlx5_flow_meter_profile *fmp;
 489         union mlx5_l3t_data data;
 490         int ret;
 491
 492         if (!priv->mtr_en)
 493                 return -rte_mtr_error_set(error, ENOTSUP,
 494                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 495                                           "Meter is not supported");
 496         /* Check input params. */
 497         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 498                                                profile, error);
 499         if (ret)
 500                 return ret;
 501         /* Meter profile memory allocation. */
 502         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 503                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 504         if (fmp == NULL)
 505                 return -rte_mtr_error_set(error, ENOMEM,
 506                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 507                                           NULL, "Meter profile memory "
 508                                           "alloc failed.");
 509         /* Fill profile info. */
 510         fmp->id = meter_profile_id;
 511         fmp->profile = *profile;
 512         /* Fill the flow meter parameters for the PRM. */
 513         ret = mlx5_flow_meter_param_fill(fmp, error);
 514         if (ret)
 515                 goto error;
 516         data.ptr = fmp;
 517         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 518                                  meter_profile_id, &data);
 519         if (ret)
 520                 return -rte_mtr_error_set(error, ENOTSUP,
 521                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 522                                           NULL, "Meter profile insert fail.");
 523         return 0;
 524 error:
 525         mlx5_free(fmp);
 526         return ret;
 527 }
 528
 529 /**
 530  * Callback to delete MTR profile.
 531  *
 532  * @param[in] dev
 533  *   Pointer to Ethernet device.
 534  * @param[in] meter_profile_id
 535  *   Meter profile id.
 536  * @param[out] error
 537  *   Pointer to the error structure.
 538  *
 539  * @return
 540  *   0 on success, a negative errno value otherwise and rte_errno is set.
 541  */
 542 static int
 543 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 544                           uint32_t meter_profile_id,
 545                           struct rte_mtr_error *error)
 546 {
 547         struct mlx5_priv *priv = dev->data->dev_private;
 548         struct mlx5_flow_meter_profile *fmp;
 549
 550         if (!priv->mtr_en)
 551                 return -rte_mtr_error_set(error, ENOTSUP,
 552                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 553                                           "Meter is not supported");
 554         /* Meter profile must exist. */
 555         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 556         if (fmp == NULL)
 557                 return -rte_mtr_error_set(error, ENOENT,
 558                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 559                                           &meter_profile_id,
 560                                           "Meter profile id is invalid.");
 561         /* Check profile is unused. */
 562         if (fmp->ref_cnt)
 563                 return -rte_mtr_error_set(error, EBUSY,
 564                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 565                                           NULL, "Meter profile is in use.");
 566         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 567                 return -rte_mtr_error_set(error, EBUSY,
 568                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 569                                           NULL, "Meter profile remove fail.");
 570         mlx5_free(fmp);
 571         return 0;
 572 }
 573
 574 /**
 575  * Find policy by id.
 576  *
 577  * @param[in] dev
 578  *   Pointer to Ethernet device.
 579  * @param policy_id
 580  *   Policy id.
 581  *
 582  * @return
 583  *   Pointer to the policy found on success, NULL otherwise.
 584  */
 585 struct mlx5_flow_meter_policy *
 586 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 587                             uint32_t policy_id,
 588                             uint32_t *policy_idx)
 589 {
 590         struct mlx5_priv *priv = dev->data->dev_private;
 591         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 592         union mlx5_l3t_data data;
 593
 594         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 595                 return NULL;
 596         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 597                                 !data.dword)
 598                 return NULL;
 599         if (policy_idx)
 600                 *policy_idx = data.dword;
 601         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 602                                         data.dword);
 603         /* Remove reference taken by the mlx5_l3t_get_entry. */
 604         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 605         if (sub_policy)
 606                 if (sub_policy->main_policy_id)
 607                         return sub_policy->main_policy;
 608         return NULL;
 609 }
 610
 611 /**
 612  * Get the last meter's policy from one meter's policy in hierarchy.
 613  *
 614  * @param[in] dev
 615  *   Pointer to Ethernet device.
 616  * @param[in] policy
 617  *   Pointer to flow meter policy.
 618  *
 619  * @return
 620  *   Pointer to the final meter's policy, or NULL when fail.
 621  */
 622 struct mlx5_flow_meter_policy *
 623 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 624                                         struct mlx5_flow_meter_policy *policy)
 625 {
 626         struct mlx5_priv *priv = dev->data->dev_private;
 627         struct mlx5_flow_meter_info *next_fm;
 628         struct mlx5_flow_meter_policy *next_policy = policy;
 629
 630         while (next_policy->is_hierarchy) {
 631                 next_fm = mlx5_flow_meter_find(priv,
 632                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 633                 if (!next_fm || next_fm->def_policy)
 634                         return NULL;
 635                 next_policy = mlx5_flow_meter_policy_find(dev,
 636                                                 next_fm->policy_id, NULL);
 637                 MLX5_ASSERT(next_policy);
 638         }
 639         return next_policy;
 640 }
 641
 642 /**
 643  * Callback to check MTR policy action validate
 644  *
 645  * @param[in] dev
 646  *   Pointer to Ethernet device.
 647  * @param[in] actions
 648  *   Pointer to meter policy action detail.
 649  * @param[out] error
 650  *   Pointer to the error structure.
 651  *
 652  * @return
 653  *   0 on success, a negative errno value otherwise and rte_errno is set.
 654  */
 655 static int
 656 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 657         struct rte_mtr_meter_policy_params *policy,
 658         struct rte_mtr_error *error)
 659 {
 660         struct mlx5_priv *priv = dev->data->dev_private;
 661         struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ?
 662                                                   1 : 0 };
 663         bool is_rss = false;
 664         uint8_t policy_mode;
 665         uint8_t domain_bitmap;
 666         int ret;
 667
 668         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 669                 return -rte_mtr_error_set(error, ENOTSUP,
 670                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 671                                 NULL, "meter policy unsupported.");
 672         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 673                         &is_rss, &domain_bitmap, &policy_mode, error);
 674         if (ret)
 675                 return ret;
 676         return 0;
 677 }
 678
 679 static int
 680 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 681                         uint32_t policy_id,
 682                         struct mlx5_flow_meter_policy *mtr_policy,
 683                         struct rte_mtr_error *error,
 684                         bool clear_l3t)
 685 {
 686         struct mlx5_priv *priv = dev->data->dev_private;
 687         struct mlx5_flow_meter_sub_policy *sub_policy;
 688         uint32_t i, j;
 689         uint16_t sub_policy_num;
 690
 691         rte_spinlock_lock(&mtr_policy->sl);
 692         if (mtr_policy->ref_cnt) {
 693                 rte_spinlock_unlock(&mtr_policy->sl);
 694                 return -rte_mtr_error_set(error, EBUSY,
 695                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 696                                  NULL,
 697                                 "Meter policy object is being used.");
 698         }
 699         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 700         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 701         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 702                 sub_policy_num = (mtr_policy->sub_policy_num >>
 703                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 704                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 705                 if (sub_policy_num) {
 706                         for (j = 0; j < sub_policy_num; j++) {
 707                                 sub_policy = mtr_policy->sub_policys[i][j];
 708                                 if (sub_policy)
 709                                         mlx5_ipool_free
 710                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 711                                         sub_policy->idx);
 712                         }
 713                 }
 714         }
 715         if (priv->policy_idx_tbl && clear_l3t) {
 716                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 717                         rte_spinlock_unlock(&mtr_policy->sl);
 718                         return -rte_mtr_error_set(error, ENOTSUP,
 719                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 720                                 "Fail to delete policy in index table.");
 721                 }
 722         }
 723         rte_spinlock_unlock(&mtr_policy->sl);
 724         return 0;
 725 }
 726
 727 /**
 728  * Callback to add MTR policy.
 729  *
 730  * @param[in] dev
 731  *   Pointer to Ethernet device.
 732  * @param[out] policy_id
 733  *   Pointer to policy id
 734  * @param[in] actions
 735  *   Pointer to meter policy action detail.
 736  * @param[out] error
 737  *   Pointer to the error structure.
 738  *
 739  * @return
 740  *   0 on success, a negative errno value otherwise and rte_errno is set.
 741  */
 742 static int
 743 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 744                         uint32_t policy_id,
 745                         struct rte_mtr_meter_policy_params *policy,
 746                         struct rte_mtr_error *error)
 747 {
 748         struct mlx5_priv *priv = dev->data->dev_private;
 749         struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ?
 750                                                   1 : 0 };
 751         uint32_t sub_policy_idx = 0;
 752         uint32_t policy_idx = 0;
 753         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 754         struct mlx5_flow_meter_sub_policy *sub_policy;
 755         bool is_rss = false;
 756         uint8_t policy_mode;
 757         uint32_t i;
 758         int ret;
 759         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 760         uint16_t sub_policy_num;
 761         uint8_t domain_bitmap = 0;
 762         union mlx5_l3t_data data;
 763         bool skip_rule = false;
 764
 765         if (!priv->mtr_en)
 766                 return -rte_mtr_error_set(error, ENOTSUP,
 767                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 768                                           NULL, "meter policy unsupported. ");
 769         if (policy_id == MLX5_INVALID_POLICY_ID)
 770                 return -rte_mtr_error_set(error, ENOTSUP,
 771                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 772                                           NULL, "policy ID is invalid. ");
 773         if (policy_id == priv->sh->mtrmng->def_policy_id)
 774                 return -rte_mtr_error_set(error, EEXIST,
 775                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 776                                           NULL, "default policy ID exists. ");
 777         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 778         if (mtr_policy)
 779                 return -rte_mtr_error_set(error, EEXIST,
 780                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 781                                           NULL, "policy ID exists. ");
 782         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 783                                           &is_rss, &domain_bitmap,
 784                                           &policy_mode, error);
 785         if (ret)
 786                 return ret;
 787         if (!domain_bitmap)
 788                 return -rte_mtr_error_set(error, ENOTSUP,
 789                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 790                                           NULL, "fail to find policy domain.");
 791         if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
 792                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 793                         return -rte_mtr_error_set(error, EEXIST,
 794                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 795                                 NULL, "a policy with similar actions "
 796                                 "is already configured");
 797                 if (mlx5_flow_create_def_policy(dev))
 798                         return -rte_mtr_error_set(error, ENOTSUP,
 799                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 800                                 NULL,
 801                                 "fail to create non-terminated policy.");
 802                 priv->sh->mtrmng->def_policy_id = policy_id;
 803                 return 0;
 804         }
 805         if (!priv->sh->meter_aso_en)
 806                 return -rte_mtr_error_set(error, ENOTSUP,
 807                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 808                         "no ASO capability to support the policy ");
 809         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 810                 if (!(domain_bitmap & (1 << i)))
 811                         continue;
 812                 /*
 813                  * If RSS is found, it means that only the ingress domain can
 814                  * be supported. It is invalid to support RSS for one color
 815                  * and egress / transfer domain actions for another. Drop and
 816                  * jump action should have no impact.
 817                  */
 818                 if (is_rss) {
 819                         policy_size +=
 820                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 821                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 822                         break;
 823                 }
 824                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 825         }
 826         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 827                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 828         if (!mtr_policy)
 829                 return -rte_mtr_error_set(error, ENOMEM,
 830                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 831                                 "Memory alloc failed for meter policy.");
 832         if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
 833                 mtr_policy->skip_y = 1;
 834         else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
 835                 mtr_policy->skip_g = 1;
 836         policy_size = sizeof(struct mlx5_flow_meter_policy);
 837         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 838                 if (!(domain_bitmap & (1 << i)))
 839                         continue;
 840                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 841                         mtr_policy->ingress = 1;
 842                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 843                         mtr_policy->egress = 1;
 844                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 845                         mtr_policy->transfer = 1;
 846                 sub_policy = mlx5_ipool_zmalloc
 847                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 848                                  &sub_policy_idx);
 849                 if (!sub_policy ||
 850                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 851                         goto policy_add_err;
 852                 sub_policy->idx = sub_policy_idx;
 853                 sub_policy->main_policy = mtr_policy;
 854                 if (!policy_idx) {
 855                         policy_idx = sub_policy_idx;
 856                         sub_policy->main_policy_id = 1;
 857                 }
 858                 mtr_policy->sub_policys[i] =
 859                         (struct mlx5_flow_meter_sub_policy **)
 860                         ((uint8_t *)mtr_policy + policy_size);
 861                 mtr_policy->sub_policys[i][0] = sub_policy;
 862                 sub_policy_num = (mtr_policy->sub_policy_num >>
 863                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 864                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 865                 sub_policy_num++;
 866                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 867                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 868                 mtr_policy->sub_policy_num |=
 869                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 870                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 871                 /*
 872                  * If RSS is found, it means that only the ingress domain can
 873                  * be supported. It is invalid to support RSS for one color
 874                  * and egress / transfer domain actions for another. Drop and
 875                  * jump action should have no impact.
 876                  */
 877                 if (is_rss) {
 878                         mtr_policy->is_rss = 1;
 879                         break;
 880                 }
 881                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 882         }
 883         rte_spinlock_init(&mtr_policy->sl);
 884         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 885                                         policy->actions, error);
 886         if (ret)
 887                 goto policy_add_err;
 888         if (mtr_policy->is_hierarchy) {
 889                 struct mlx5_flow_meter_policy *final_policy;
 890
 891                 final_policy =
 892                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 893                 if (!final_policy)
 894                         goto policy_add_err;
 895                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 896         }
 897         /*
 898          * If either Green or Yellow has queue / RSS action, all the policy
 899          * rules will be created later in the flow splitting stage.
 900          */
 901         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 902                 /* Create policy rules in HW. */
 903                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 904                 if (ret)
 905                         goto policy_add_err;
 906         }
 907         data.dword = policy_idx;
 908         if (!priv->policy_idx_tbl) {
 909                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 910                 if (!priv->policy_idx_tbl)
 911                         goto policy_add_err;
 912         }
 913         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 914                 goto policy_add_err;
 915         return 0;
 916 policy_add_err:
 917         if (mtr_policy) {
 918                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 919                         mtr_policy, error, false);
 920                 mlx5_free(mtr_policy);
 921                 if (ret)
 922                         return ret;
 923         }
 924         return -rte_mtr_error_set(error, ENOTSUP,
 925                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 926                                   NULL, "Failed to create devx policy.");
 927 }
 928
 929 /**
 930  * Callback to delete MTR policy.
 931  *
 932  * @param[in] dev
 933  *   Pointer to Ethernet device.
 934  * @param[in] policy_id
 935  *   Meter policy id.
 936  * @param[out] error
 937  *   Pointer to the error structure.
 938  *
 939  * @return
 940  *   0 on success, a negative errno value otherwise and rte_errno is set.
 941  */
 942 static int
 943 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 944                           uint32_t policy_id,
 945                           struct rte_mtr_error *error)
 946 {
 947         struct mlx5_priv *priv = dev->data->dev_private;
 948         struct mlx5_flow_meter_policy *mtr_policy;
 949         uint32_t policy_idx;
 950         int ret;
 951
 952         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 953                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 954                         return -rte_mtr_error_set(error, ENOTSUP,
 955                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 956                                 "Meter policy object is being used.");
 957                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 958                 return 0;
 959         }
 960         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 961         if (!mtr_policy)
 962                 return -rte_mtr_error_set(error, ENOTSUP,
 963                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 964                         "Meter policy id is invalid. ");
 965         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 966                                                 error, true);
 967         if (ret)
 968                 return ret;
 969         mlx5_free(mtr_policy);
 970         return 0;
 971 }
 972
 973 /**
 974  * Check meter validation.
 975  *
 976  * @param[in] priv
 977  *   Pointer to mlx5 private data structure.
 978  * @param[in] meter_id
 979  *   Meter id.
 980  * @param[in] params
 981  *   Pointer to rte meter parameters.
 982  * @param[out] error
 983  *   Pointer to rte meter error structure.
 984  *
 985  * @return
 986  *   0 on success, a negative errno value otherwise and rte_errno is set.
 987  */
 988 static int
 989 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 990                          struct rte_mtr_params *params,
 991                          struct rte_mtr_error *error)
 992 {
 993         /* Meter must use global drop action. */
 994         if (!priv->sh->dr_drop_action)
 995                 return -rte_mtr_error_set(error, ENOTSUP,
 996                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 997                                           NULL,
 998                                           "No drop action ready for meter.");
 999         /* Meter params must not be NULL. */
1000         if (params == NULL)
1001                 return -rte_mtr_error_set(error, EINVAL,
1002                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1003                                           NULL, "Meter object params null.");
1004         /* Previous meter color is not supported. */
1005         if (params->use_prev_mtr_color)
1006                 return -rte_mtr_error_set(error, ENOTSUP,
1007                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1008                                           NULL,
1009                                           "Previous meter color "
1010                                           "not supported.");
1011         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
1012                 return -rte_mtr_error_set(error, ENOENT,
1013                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1014                                 NULL, "Meter policy id not valid.");
1015         /* Validate meter id. */
1016         if (mlx5_flow_meter_find(priv, meter_id, NULL))
1017                 return -rte_mtr_error_set(error, EEXIST,
1018                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
1019                         "Meter object already exists.");
1020         return 0;
1021 }
1022
1023 /**
1024  * Modify the flow meter action.
1025  *
1026  * @param[in] priv
1027  *   Pointer to mlx5 private data structure.
1028  * @param[in] fm
1029  *   Pointer to flow meter to be modified.
1030  * @param[in] srtcm
1031  *   Pointer to meter srtcm description parameter.
1032  * @param[in] modify_bits
1033  *   The bit in srtcm to be updated.
1034  * @param[in] active_state
1035  *   The state to be updated.
1036  * @return
1037  *   0 on success, o negative value otherwise.
1038  */
1039 static int
1040 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
1041                 struct mlx5_flow_meter_info *fm,
1042                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
1043                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
1044 {
1045 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1046         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
1047         uint32_t *attr;
1048         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
1049         int ret;
1050         struct mlx5_aso_mtr *aso_mtr = NULL;
1051         uint32_t cbs_cir, ebs_eir, val;
1052
1053         if (priv->sh->meter_aso_en) {
1054                 fm->is_enable = !!is_enable;
1055                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1056                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1057                 if (ret)
1058                         return ret;
1059                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
1060                 if (ret)
1061                         return ret;
1062         } else {
1063                 /* Fill command parameters. */
1064                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
1065                 mod_attr.flow_meter_parameter = in;
1066                 mod_attr.flow_meter_parameter_sz =
1067                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
1068                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1069                         mod_attr.active = !!active_state;
1070                 else
1071                         mod_attr.active = 0;
1072                 attr = in;
1073                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
1074                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1075                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1076                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1077                                 ASO_DSEG_EXP_MASK;
1078                         MLX5_SET(flow_meter_parameters, attr,
1079                                 cbs_exponent, val);
1080                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1081                                 ASO_DSEG_MAN_MASK;
1082                         MLX5_SET(flow_meter_parameters, attr,
1083                                 cbs_mantissa, val);
1084                 }
1085                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1086                         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) &
1087                                 ASO_DSEG_EXP_MASK;
1088                         MLX5_SET(flow_meter_parameters, attr,
1089                                 cir_exponent, val);
1090                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1091                         MLX5_SET(flow_meter_parameters, attr,
1092                                 cir_mantissa, val);
1093                 }
1094                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1095                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1096                                 ASO_DSEG_EXP_MASK;
1097                         MLX5_SET(flow_meter_parameters, attr,
1098                                 ebs_exponent, val);
1099                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1100                                 ASO_DSEG_MAN_MASK;
1101                         MLX5_SET(flow_meter_parameters, attr,
1102                                 ebs_mantissa, val);
1103                 }
1104                 /* Apply modifications to meter only if it was created. */
1105                 if (fm->meter_action) {
1106                         ret = mlx5_glue->dv_modify_flow_action_meter
1107                                         (fm->meter_action, &mod_attr,
1108                                         rte_cpu_to_be_64(modify_bits));
1109                         if (ret)
1110                                 return ret;
1111                 }
1112                 /* Update succeeded modify meter parameters. */
1113                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1114                         fm->active_state = !!active_state;
1115         }
1116         return 0;
1117 #else
1118         (void)priv;
1119         (void)fm;
1120         (void)srtcm;
1121         (void)modify_bits;
1122         (void)active_state;
1123         (void)is_enable;
1124         return -ENOTSUP;
1125 #endif
1126 }
1127
1128 static int
1129 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1130                                 struct mlx5_flow_meter_info *fm,
1131                                 uint64_t stats_mask)
1132 {
1133         fm->bytes_dropped =
1134                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1135         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1136         if (fm->bytes_dropped || fm->pkts_dropped) {
1137                 if (!fm->drop_cnt) {
1138                         /* Alloc policer counters. */
1139                         fm->drop_cnt = mlx5_counter_alloc(dev);
1140                         if (!fm->drop_cnt)
1141                                 return -1;
1142                 }
1143         } else {
1144                 if (fm->drop_cnt) {
1145                         mlx5_counter_free(dev, fm->drop_cnt);
1146                         fm->drop_cnt = 0;
1147                 }
1148         }
1149         return 0;
1150 }
1151
1152 /**
1153  * Create meter rules.
1154  *
1155  * @param[in] dev
1156  *   Pointer to Ethernet device.
1157  * @param[in] meter_id
1158  *   Meter id.
1159  * @param[in] params
1160  *   Pointer to rte meter parameters.
1161  * @param[in] shared
1162  *   Meter shared with other flow or not.
1163  * @param[out] error
1164  *   Pointer to rte meter error structure.
1165  *
1166  * @return
1167  *   0 on success, a negative errno value otherwise and rte_errno is set.
1168  */
1169 static int
1170 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1171                        struct rte_mtr_params *params, int shared,
1172                        struct rte_mtr_error *error)
1173 {
1174         struct mlx5_priv *priv = dev->data->dev_private;
1175         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1176         struct mlx5_flow_meter_profile *fmp;
1177         struct mlx5_flow_meter_info *fm;
1178         /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */
1179         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1180         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1181         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1182                 .size = 0,
1183                 .trunk_size = 64,
1184                 .need_lock = 1,
1185                 .type = "mlx5_flow_mtr_flow_id_pool",
1186         };
1187         struct mlx5_aso_mtr *aso_mtr;
1188         uint32_t mtr_idx, policy_idx;
1189         union mlx5_l3t_data data;
1190         int ret;
1191         uint8_t domain_bitmap;
1192         uint8_t mtr_id_bits;
1193         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1194                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1195
1196         if (!priv->mtr_en)
1197                 return -rte_mtr_error_set(error, ENOTSUP,
1198                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1199                                         "Meter is not supported");
1200         /* Validate the parameters. */
1201         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1202         if (ret)
1203                 return ret;
1204         /* Meter profile must exist. */
1205         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1206         if (fmp == NULL)
1207                 return -rte_mtr_error_set(error, ENOENT,
1208                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1209                         NULL, "Meter profile id not valid.");
1210         /* Meter policy must exist. */
1211         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1212                 __atomic_add_fetch
1213                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1214                         1, __ATOMIC_RELAXED);
1215                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1216                 if (!priv->sh->config.dv_esw_en)
1217                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1218         } else {
1219                 if (!priv->sh->meter_aso_en)
1220                         return -rte_mtr_error_set(error, ENOTSUP,
1221                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1222                                 "Part of the policies cannot be "
1223                                 "supported without ASO ");
1224                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1225                                 params->meter_policy_id, &policy_idx);
1226                 if (!mtr_policy)
1227                         return -rte_mtr_error_set(error, ENOENT,
1228                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1229                                 NULL, "Meter policy id not valid.");
1230                 domain_bitmap = (mtr_policy->ingress ?
1231                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1232                                 (mtr_policy->egress ?
1233                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1234                                 (mtr_policy->transfer ?
1235                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1236                 if (fmp->g_support && mtr_policy->skip_g)
1237                         return -rte_mtr_error_set(error, ENOTSUP,
1238                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1239                                         NULL, "Meter green policy is empty.");
1240                 if (fmp->y_support && mtr_policy->skip_y)
1241                         return -rte_mtr_error_set(error, ENOTSUP,
1242                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1243                                         NULL, "Meter yellow policy is empty.");
1244         }
1245         /* Allocate the flow meter memory. */
1246         if (priv->sh->meter_aso_en) {
1247                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1248                 if (!mtr_idx)
1249                         return -rte_mtr_error_set(error, ENOMEM,
1250                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1251                                 "Memory alloc failed for meter.");
1252                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1253                 fm = &aso_mtr->fm;
1254         } else {
1255                 if (fmp->y_support)
1256                         return -rte_mtr_error_set(error, ENOMEM,
1257                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1258                                 "Unsupported profile with yellow.");
1259                 legacy_fm = mlx5_ipool_zmalloc
1260                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1261                 if (legacy_fm == NULL)
1262                         return -rte_mtr_error_set(error, ENOMEM,
1263                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1264                                 "Memory alloc failed for meter.");
1265                 legacy_fm->idx = mtr_idx;
1266                 fm = &legacy_fm->fm;
1267         }
1268         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1269         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1270             mtr_reg_bits) {
1271                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1272                 goto error;
1273         }
1274         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1275                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1276         /* Fill the flow meter parameters. */
1277         fm->meter_id = meter_id;
1278         fm->policy_id = params->meter_policy_id;
1279         fm->profile = fmp;
1280         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1281                 goto error;
1282         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1283                 goto error;
1284         /* Add to the flow meter list. */
1285         if (!priv->sh->meter_aso_en) {
1286                 MLX5_ASSERT(legacy_fm != NULL);
1287                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1288         }
1289         /* Add to the flow meter list. */
1290         fm->active_state = 1; /* Config meter starts as active. */
1291         fm->is_enable = 1;
1292         fm->shared = !!shared;
1293         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1294         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1295                 fm->def_policy = 1;
1296                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1297                 if (!fm->flow_ipool)
1298                         goto error;
1299         }
1300         rte_spinlock_init(&fm->sl);
1301         /* If ASO meter supported, update ASO flow meter by wqe. */
1302         if (priv->sh->meter_aso_en) {
1303                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1304                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1305                 if (ret)
1306                         goto error;
1307                 if (!priv->mtr_idx_tbl) {
1308                         priv->mtr_idx_tbl =
1309                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1310                         if (!priv->mtr_idx_tbl)
1311                                 goto error;
1312                 }
1313                 data.dword = mtr_idx;
1314                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1315                         goto error;
1316         }
1317         if (mtr_policy)
1318                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1319         return 0;
1320 error:
1321         mlx5_flow_destroy_mtr_tbls(dev, fm);
1322         /* Free policer counters. */
1323         if (fm->drop_cnt)
1324                 mlx5_counter_free(dev, fm->drop_cnt);
1325         if (priv->sh->meter_aso_en)
1326                 mlx5_flow_mtr_free(dev, mtr_idx);
1327         else
1328                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1329         return -rte_mtr_error_set(error, ENOTSUP,
1330                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1331                 NULL, "Failed to create devx meter.");
1332 }
1333
1334 static int
1335 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1336                         struct mlx5_flow_meter_info *fm,
1337                         uint32_t mtr_idx)
1338 {
1339         struct mlx5_priv *priv = dev->data->dev_private;
1340         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1341         struct mlx5_flow_meter_profile *fmp;
1342         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1343         struct mlx5_flow_meter_policy *mtr_policy;
1344
1345         /* Meter object must not have any owner. */
1346         MLX5_ASSERT(!fm->ref_cnt);
1347         /* Get meter profile. */
1348         fmp = fm->profile;
1349         if (fmp == NULL)
1350                 return -1;
1351         /* Update dependencies. */
1352         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1353         fm->profile = NULL;
1354         /* Remove from list. */
1355         if (!priv->sh->meter_aso_en) {
1356                 legacy_fm = container_of(fm,
1357                         struct mlx5_legacy_flow_meter, fm);
1358                 TAILQ_REMOVE(fms, legacy_fm, next);
1359         }
1360         /* Free drop counters. */
1361         if (fm->drop_cnt)
1362                 mlx5_counter_free(dev, fm->drop_cnt);
1363         /* Free meter flow table. */
1364         if (fm->flow_ipool) {
1365                 mlx5_ipool_destroy(fm->flow_ipool);
1366                 fm->flow_ipool = 0;
1367         }
1368         mlx5_flow_destroy_mtr_tbls(dev, fm);
1369         if (fm->def_policy)
1370                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1371                                 1, __ATOMIC_RELAXED);
1372         if (priv->sh->meter_aso_en) {
1373                 if (!fm->def_policy) {
1374                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1375                                                 fm->policy_id, NULL);
1376                         if (mtr_policy)
1377                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1378                                                 1, __ATOMIC_RELAXED);
1379                         fm->policy_id = 0;
1380                 }
1381                 fm->def_policy = 0;
1382                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1383                         return -1;
1384                 mlx5_flow_mtr_free(dev, mtr_idx);
1385         } else {
1386                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1387                                         legacy_fm->idx);
1388         }
1389         return 0;
1390 }
1391
1392 /**
1393  * Destroy meter rules.
1394  *
1395  * @param[in] dev
1396  *   Pointer to Ethernet device.
1397  * @param[in] meter_id
1398  *   Meter id.
1399  * @param[out] error
1400  *   Pointer to rte meter error structure.
1401  *
1402  * @return
1403  *   0 on success, a negative errno value otherwise and rte_errno is set.
1404  */
1405 static int
1406 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1407                         struct rte_mtr_error *error)
1408 {
1409         struct mlx5_priv *priv = dev->data->dev_private;
1410         struct mlx5_flow_meter_info *fm;
1411         uint32_t mtr_idx = 0;
1412
1413         if (!priv->mtr_en)
1414                 return -rte_mtr_error_set(error, ENOTSUP,
1415                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1416                                           NULL,
1417                                           "Meter is not supported");
1418         /* Meter object must exist. */
1419         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1420         if (fm == NULL)
1421                 return -rte_mtr_error_set(error, ENOENT,
1422                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1423                                           NULL,
1424                                           "Meter object id not valid.");
1425         /* Meter object must not have any owner. */
1426         if (fm->ref_cnt > 0)
1427                 return -rte_mtr_error_set(error, EBUSY,
1428                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1429                                           NULL,
1430                                           "Meter object is being used.");
1431         /* Destroy the meter profile. */
1432         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1433                 return -rte_mtr_error_set(error, EINVAL,
1434                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1435                                         NULL,
1436                                         "MTR object meter profile invalid.");
1437         return 0;
1438 }
1439
1440 /**
1441  * Modify meter state.
1442  *
1443  * @param[in] priv
1444  *   Pointer to mlx5 private data structure.
1445  * @param[in] fm
1446  *   Pointer to flow meter.
1447  * @param[in] new_state
1448  *   New state to update.
1449  * @param[out] error
1450  *   Pointer to rte meter error structure.
1451  *
1452  * @return
1453  *   0 on success, a negative errno value otherwise and rte_errno is set.
1454  */
1455 static int
1456 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1457                              struct mlx5_flow_meter_info *fm,
1458                              uint32_t new_state,
1459                              struct rte_mtr_error *error)
1460 {
1461         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1462                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1463                 .ebs_eir = 0,
1464         };
1465         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1466                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1467         int ret;
1468
1469         if (new_state == MLX5_FLOW_METER_DISABLE)
1470                 ret = mlx5_flow_meter_action_modify(priv, fm,
1471                                 &srtcm, modify_bits, 0, 0);
1472         else
1473                 ret = mlx5_flow_meter_action_modify(priv, fm,
1474                                                     &fm->profile->srtcm_prm,
1475                                                     modify_bits, 0, 1);
1476         if (ret)
1477                 return -rte_mtr_error_set(error, -ret,
1478                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1479                                           NULL,
1480                                           new_state ?
1481                                           "Failed to enable meter." :
1482                                           "Failed to disable meter.");
1483         return 0;
1484 }
1485
1486 /**
1487  * Callback to enable flow meter.
1488  *
1489  * @param[in] dev
1490  *   Pointer to Ethernet device.
1491  * @param[in] meter_id
1492  *   Meter id.
1493  * @param[out] error
1494  *   Pointer to rte meter error structure.
1495  *
1496  * @return
1497  *   0 on success, a negative errno value otherwise and rte_errno is set.
1498  */
1499 static int
1500 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1501                        uint32_t meter_id,
1502                        struct rte_mtr_error *error)
1503 {
1504         struct mlx5_priv *priv = dev->data->dev_private;
1505         struct mlx5_flow_meter_info *fm;
1506         int ret;
1507
1508         if (!priv->mtr_en)
1509                 return -rte_mtr_error_set(error, ENOTSUP,
1510                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1511                                           "Meter is not supported");
1512         /* Meter object must exist. */
1513         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1514         if (fm == NULL)
1515                 return -rte_mtr_error_set(error, ENOENT,
1516                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1517                                           NULL, "Meter not found.");
1518         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1519                 return 0;
1520         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1521                                            error);
1522         if (!ret)
1523                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1524         return ret;
1525 }
1526
1527 /**
1528  * Callback to disable flow meter.
1529  *
1530  * @param[in] dev
1531  *   Pointer to Ethernet device.
1532  * @param[in] meter_id
1533  *   Meter id.
1534  * @param[out] error
1535  *   Pointer to rte meter error structure.
1536  *
1537  * @return
1538  *   0 on success, a negative errno value otherwise and rte_errno is set.
1539  */
1540 static int
1541 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1542                         uint32_t meter_id,
1543                         struct rte_mtr_error *error)
1544 {
1545         struct mlx5_priv *priv = dev->data->dev_private;
1546         struct mlx5_flow_meter_info *fm;
1547         int ret;
1548
1549         if (!priv->mtr_en)
1550                 return -rte_mtr_error_set(error, ENOTSUP,
1551                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1552                                           "Meter is not supported");
1553         /* Meter object must exist. */
1554         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1555         if (fm == NULL)
1556                 return -rte_mtr_error_set(error, ENOENT,
1557                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1558                                           NULL, "Meter not found.");
1559         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1560                 return 0;
1561         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1562                                            error);
1563         if (!ret)
1564                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1565         return ret;
1566 }
1567
1568 /**
1569  * Callback to update meter profile.
1570  *
1571  * @param[in] dev
1572  *   Pointer to Ethernet device.
1573  * @param[in] meter_id
1574  *   Meter id.
1575  * @param[in] meter_profile_id
1576  *   To be updated meter profile id.
1577  * @param[out] error
1578  *   Pointer to rte meter error structure.
1579  *
1580  * @return
1581  *   0 on success, a negative errno value otherwise and rte_errno is set.
1582  */
1583 static int
1584 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1585                                uint32_t meter_id,
1586                                uint32_t meter_profile_id,
1587                                struct rte_mtr_error *error)
1588 {
1589         struct mlx5_priv *priv = dev->data->dev_private;
1590         struct mlx5_flow_meter_profile *fmp;
1591         struct mlx5_flow_meter_profile *old_fmp;
1592         struct mlx5_flow_meter_info *fm;
1593         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1594                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1595         int ret;
1596
1597         if (!priv->mtr_en)
1598                 return -rte_mtr_error_set(error, ENOTSUP,
1599                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1600                                           "Meter is not supported");
1601         /* Meter profile must exist. */
1602         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1603         if (fmp == NULL)
1604                 return -rte_mtr_error_set(error, ENOENT,
1605                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1606                                           NULL, "Meter profile not found.");
1607         /* Meter object must exist. */
1608         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1609         if (fm == NULL)
1610                 return -rte_mtr_error_set(error, ENOENT,
1611                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1612                                           NULL, "Meter not found.");
1613         /* MTR object already set to meter profile id. */
1614         old_fmp = fm->profile;
1615         if (fmp == old_fmp)
1616                 return 0;
1617         /* Update the profile. */
1618         fm->profile = fmp;
1619         /* Update meter params in HW (if not disabled). */
1620         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1621                 return 0;
1622         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1623                                               modify_bits, fm->active_state, 1);
1624         if (ret) {
1625                 fm->profile = old_fmp;
1626                 return -rte_mtr_error_set(error, -ret,
1627                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1628                                           NULL, "Failed to update meter"
1629                                           " parameters in hardware.");
1630         }
1631         old_fmp->ref_cnt--;
1632         fmp->ref_cnt++;
1633         return 0;
1634 }
1635
1636 /**
1637  * Callback to update meter stats mask.
1638  *
1639  * @param[in] dev
1640  *   Pointer to Ethernet device.
1641  * @param[in] meter_id
1642  *   Meter id.
1643  * @param[in] stats_mask
1644  *   To be updated stats_mask.
1645  * @param[out] error
1646  *   Pointer to rte meter error structure.
1647  *
1648  * @return
1649  *   0 on success, a negative errno value otherwise and rte_errno is set.
1650  */
1651 static int
1652 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1653                              uint32_t meter_id,
1654                              uint64_t stats_mask,
1655                              struct rte_mtr_error *error)
1656 {
1657         struct mlx5_priv *priv = dev->data->dev_private;
1658         struct mlx5_flow_meter_info *fm;
1659
1660         if (!priv->mtr_en)
1661                 return -rte_mtr_error_set(error, ENOTSUP,
1662                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1663                                           "Meter is not supported");
1664         /* Meter object must exist. */
1665         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1666         if (fm == NULL)
1667                 return -rte_mtr_error_set(error, ENOENT,
1668                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1669                                           NULL, "Meter object id not valid.");
1670         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1671                 return -rte_mtr_error_set(error, ENOENT,
1672                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1673                                           NULL, "Fail to allocate "
1674                                           "counter for meter.");
1675         return 0;
1676 }
1677
1678 /**
1679  * Callback to read meter statistics.
1680  *
1681  * @param[in] dev
1682  *   Pointer to Ethernet device.
1683  * @param[in] meter_id
1684  *   Meter id.
1685  * @param[out] stats
1686  *   Pointer to store the statistics.
1687  * @param[out] stats_mask
1688  *   Pointer to store the stats_mask.
1689  * @param[in] clear
1690  *   Statistic to be cleared after read or not.
1691  * @param[out] error
1692  *   Pointer to rte meter error structure.
1693  *
1694  * @return
1695  *   0 on success, a negative errno value otherwise and rte_errno is set.
1696  */
1697 static int
1698 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1699                            uint32_t meter_id,
1700                            struct rte_mtr_stats *stats,
1701                            uint64_t *stats_mask,
1702                            int clear,
1703                            struct rte_mtr_error *error)
1704 {
1705         struct mlx5_priv *priv = dev->data->dev_private;
1706         struct mlx5_flow_meter_info *fm;
1707         uint64_t pkts;
1708         uint64_t bytes;
1709         int ret = 0;
1710
1711         if (!priv->mtr_en)
1712                 return -rte_mtr_error_set(error, ENOTSUP,
1713                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1714                                           "Meter is not supported");
1715         /* Meter object must exist. */
1716         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1717         if (fm == NULL)
1718                 return -rte_mtr_error_set(error, ENOENT,
1719                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1720                                           NULL, "Meter object id not valid.");
1721         *stats_mask = 0;
1722         if (fm->bytes_dropped)
1723                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1724         if (fm->pkts_dropped)
1725                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1726         memset(stats, 0, sizeof(*stats));
1727         if (fm->drop_cnt) {
1728                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1729                                                  &bytes, NULL);
1730                 if (ret)
1731                         goto error;
1732                 /* If need to read the packets, set it. */
1733                 if (fm->pkts_dropped)
1734                         stats->n_pkts_dropped = pkts;
1735                 /* If need to read the bytes, set it. */
1736                 if (fm->bytes_dropped)
1737                         stats->n_bytes_dropped = bytes;
1738         }
1739         return 0;
1740 error:
1741         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1742                                  "Failed to read meter drop counters.");
1743 }
1744
1745 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1746         .capabilities_get = mlx5_flow_mtr_cap_get,
1747         .meter_profile_add = mlx5_flow_meter_profile_add,
1748         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1749         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1750         .meter_policy_add = mlx5_flow_meter_policy_add,
1751         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1752         .create = mlx5_flow_meter_create,
1753         .destroy = mlx5_flow_meter_destroy,
1754         .meter_enable = mlx5_flow_meter_enable,
1755         .meter_disable = mlx5_flow_meter_disable,
1756         .meter_profile_update = mlx5_flow_meter_profile_update,
1757         .meter_dscp_table_update = NULL,
1758         .stats_update = mlx5_flow_meter_stats_update,
1759         .stats_read = mlx5_flow_meter_stats_read,
1760 };
1761
1762 /**
1763  * Get meter operations.
1764  *
1765  * @param dev
1766  *   Pointer to Ethernet device structure.
1767  * @param arg
1768  *   Pointer to set the mtr operations.
1769  *
1770  * @return
1771  *   Always 0.
1772  */
1773 int
1774 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1775 {
1776         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1777         return 0;
1778 }
1779
1780 /**
1781  * Find meter by id.
1782  *
1783  * @param priv
1784  *   Pointer to mlx5_priv.
1785  * @param meter_id
1786  *   Meter id.
1787  * @param mtr_idx
1788  *   Pointer to Meter index.
1789  *
1790  * @return
1791  *   Pointer to the meter info found on success, NULL otherwise.
1792  */
1793 struct mlx5_flow_meter_info *
1794 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1795                 uint32_t *mtr_idx)
1796 {
1797         struct mlx5_legacy_flow_meter *legacy_fm;
1798         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1799         struct mlx5_aso_mtr *aso_mtr;
1800         struct mlx5_aso_mtr_pools_mng *pools_mng =
1801                                 &priv->sh->mtrmng->pools_mng;
1802         union mlx5_l3t_data data;
1803         uint16_t n_valid;
1804
1805         if (priv->sh->meter_aso_en) {
1806                 rte_rwlock_read_lock(&pools_mng->resize_mtrwl);
1807                 n_valid = pools_mng->n_valid;
1808                 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl);
1809                 if (!n_valid || !priv->mtr_idx_tbl ||
1810                     (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1811                     !data.dword))
1812                         return NULL;
1813                 if (mtr_idx)
1814                         *mtr_idx = data.dword;
1815                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1816                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1817                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1818                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1819                         return NULL;
1820                 return &aso_mtr->fm;
1821         }
1822         TAILQ_FOREACH(legacy_fm, fms, next)
1823                 if (meter_id == legacy_fm->fm.meter_id) {
1824                         if (mtr_idx)
1825                                 *mtr_idx = legacy_fm->idx;
1826                         return &legacy_fm->fm;
1827                 }
1828         return NULL;
1829 }
1830
1831 /**
1832  * Find meter by index.
1833  *
1834  * @param priv
1835  *   Pointer to mlx5_priv.
1836  * @param idx
1837  *   Meter index.
1838  *
1839  * @return
1840  *   Pointer to the meter info found on success, NULL otherwise.
1841  */
1842 struct mlx5_flow_meter_info *
1843 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1844 {
1845         struct mlx5_aso_mtr *aso_mtr;
1846
1847         if (priv->sh->meter_aso_en) {
1848                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1849                 if (!aso_mtr)
1850                         return NULL;
1851                 return &aso_mtr->fm;
1852         } else {
1853                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1854         }
1855 }
1856
1857 /**
1858  * Attach meter to flow.
1859  * Unidirectional Meter creation can only be done
1860  * when flow direction is known, i.e. when calling meter_attach.
1861  *
1862  * @param [in] priv
1863  *  Pointer to mlx5 private data.
1864  * @param[in] fm
1865  *   Pointer to flow meter.
1866  * @param [in] attr
1867  *  Pointer to flow attributes.
1868  * @param [out] error
1869  *  Pointer to error structure.
1870  *
1871  * @return
1872  *   0 on success, a negative errno value otherwise and rte_errno is set.
1873  */
1874 int
1875 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1876                        struct mlx5_flow_meter_info *fm,
1877                        const struct rte_flow_attr *attr,
1878                        struct rte_flow_error *error)
1879 {
1880         int ret = 0;
1881
1882         if (priv->sh->meter_aso_en) {
1883                 struct mlx5_aso_mtr *aso_mtr;
1884
1885                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1886                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1887                         return rte_flow_error_set(error, ENOENT,
1888                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1889                                         NULL,
1890                                         "Timeout in meter configuration");
1891                 }
1892                 rte_spinlock_lock(&fm->sl);
1893                 if (fm->shared || !fm->ref_cnt) {
1894                         fm->ref_cnt++;
1895                 } else {
1896                         rte_flow_error_set(error, EINVAL,
1897                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1898                                    "Meter cannot be shared");
1899                         ret = -1;
1900                 }
1901                 rte_spinlock_unlock(&fm->sl);
1902         } else {
1903                 rte_spinlock_lock(&fm->sl);
1904                 if (fm->meter_action) {
1905                         if (fm->shared &&
1906                             attr->transfer == fm->transfer &&
1907                             attr->ingress == fm->ingress &&
1908                             attr->egress == fm->egress) {
1909                                 fm->ref_cnt++;
1910                         } else {
1911                                 rte_flow_error_set(error, EINVAL,
1912                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1913                                         fm->shared ?
1914                                         "Meter attr not match." :
1915                                         "Meter cannot be shared.");
1916                                 ret = -1;
1917                         }
1918                 } else {
1919                         fm->ingress = attr->ingress;
1920                         fm->egress = attr->egress;
1921                         fm->transfer = attr->transfer;
1922                         fm->ref_cnt = 1;
1923                         /* This also creates the meter object. */
1924                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1925                                                                          fm);
1926                         if (!fm->meter_action) {
1927                                 fm->ref_cnt = 0;
1928                                 fm->ingress = 0;
1929                                 fm->egress = 0;
1930                                 fm->transfer = 0;
1931                                 rte_flow_error_set(error, EINVAL,
1932                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1933                                         "Meter action create failed.");
1934                                 ret = -1;
1935                         }
1936                 }
1937                 rte_spinlock_unlock(&fm->sl);
1938         }
1939         return ret ? -rte_errno : 0;
1940 }
1941
1942 /**
1943  * Detach meter from flow.
1944  *
1945  * @param [in] priv
1946  *  Pointer to mlx5 private data.
1947  * @param [in] fm
1948  *  Pointer to flow meter.
1949  */
1950 void
1951 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1952                        struct mlx5_flow_meter_info *fm)
1953 {
1954 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1955         rte_spinlock_lock(&fm->sl);
1956         MLX5_ASSERT(fm->ref_cnt);
1957         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1958                 mlx5_glue->destroy_flow_action(fm->meter_action);
1959                 fm->meter_action = NULL;
1960                 fm->ingress = 0;
1961                 fm->egress = 0;
1962                 fm->transfer = 0;
1963         }
1964         rte_spinlock_unlock(&fm->sl);
1965 #else
1966         (void)priv;
1967         (void)fm;
1968 #endif
1969 }
1970
1971 /**
1972  * Flush meter with Rx queue configuration.
1973  *
1974  * @param[in] dev
1975  *   Pointer to Ethernet device.
1976  */
1977 void
1978 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1979 {
1980         struct mlx5_priv *priv = dev->data->dev_private;
1981         struct mlx5_flow_meter_sub_policy *sub_policy;
1982         struct mlx5_flow_meter_policy *mtr_policy;
1983         void *entry;
1984         uint32_t i, policy_idx;
1985
1986         if (!priv->mtr_en)
1987                 return;
1988         if (priv->policy_idx_tbl) {
1989                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1990                         policy_idx = *(uint32_t *)entry;
1991                         sub_policy = mlx5_ipool_get
1992                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1993                                 policy_idx);
1994                         if (!sub_policy || !sub_policy->main_policy)
1995                                 continue;
1996                         mtr_policy = sub_policy->main_policy;
1997                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1998                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1999                                         mtr_policy);
2000                 }
2001         }
2002 }
2003
2004 /**
2005  * Iterate a meter hierarchy and flush all meters and policies if possible.
2006  *
2007  * @param[in] dev
2008  *   Pointer to Ethernet device.
2009  * @param[in] fm
2010  *   Pointer to flow meter.
2011  * @param[in] mtr_idx
2012  *   .Meter's index
2013  * @param[out] error
2014  *   Pointer to rte meter error structure.
2015  *
2016  * @return
2017  *   0 on success, a negative errno value otherwise and rte_errno is set.
2018  */
2019 static int
2020 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
2021                                 struct mlx5_flow_meter_info *fm,
2022                                 uint32_t mtr_idx,
2023                                 struct rte_mtr_error *error)
2024 {
2025         struct mlx5_priv *priv = dev->data->dev_private;
2026         struct mlx5_flow_meter_policy *policy;
2027         uint32_t policy_id;
2028         struct mlx5_flow_meter_info *next_fm;
2029         uint32_t next_mtr_idx;
2030         struct mlx5_flow_meter_policy *next_policy = NULL;
2031
2032         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
2033         MLX5_ASSERT(policy);
2034         while (!fm->ref_cnt && policy->is_hierarchy) {
2035                 policy_id = fm->policy_id;
2036                 next_fm = mlx5_flow_meter_find(priv,
2037                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2038                                 &next_mtr_idx);
2039                 if (next_fm) {
2040                         next_policy = mlx5_flow_meter_policy_find(dev,
2041                                                         next_fm->policy_id,
2042                                                         NULL);
2043                         MLX5_ASSERT(next_policy);
2044                 }
2045                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
2046                         return -rte_mtr_error_set(error, ENOTSUP,
2047                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
2048                                                 NULL,
2049                                                 "Failed to flush meter.");
2050                 if (policy->ref_cnt)
2051                         break;
2052                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
2053                                                 policy, error, true))
2054                         return -rte_errno;
2055                 mlx5_free(policy);
2056                 if (!next_fm || !next_policy)
2057                         break;
2058                 fm = next_fm;
2059                 mtr_idx = next_mtr_idx;
2060                 policy = next_policy;
2061         }
2062         return 0;
2063 }
2064
2065 /**
2066  * Flush all the hierarchy meters and their policies.
2067  *
2068  * @param[in] dev
2069  *   Pointer to Ethernet device.
2070  * @param[out] error
2071  *   Pointer to rte meter error structure.
2072  *
2073  * @return
2074  *   0 on success, a negative errno value otherwise and rte_errno is set.
2075  */
2076 static int
2077 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
2078                                       struct rte_mtr_error *error)
2079 {
2080         struct mlx5_priv *priv = dev->data->dev_private;
2081         struct mlx5_flow_meter_info *fm;
2082         struct mlx5_flow_meter_policy *policy;
2083         struct mlx5_flow_meter_sub_policy *sub_policy;
2084         struct mlx5_flow_meter_info *next_fm;
2085         struct mlx5_aso_mtr *aso_mtr;
2086         uint32_t mtr_idx = 0;
2087         uint32_t i, policy_idx;
2088         void *entry;
2089
2090         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2091                 return 0;
2092         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2093                 mtr_idx = *(uint32_t *)entry;
2094                 if (!mtr_idx)
2095                         continue;
2096                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2097                 fm = &aso_mtr->fm;
2098                 if (fm->ref_cnt || fm->def_policy)
2099                         continue;
2100                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2101                         return -rte_errno;
2102         }
2103         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2104                 policy_idx = *(uint32_t *)entry;
2105                 sub_policy = mlx5_ipool_get
2106                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2107                                 policy_idx);
2108                 if (!sub_policy)
2109                         return -rte_mtr_error_set(error,
2110                                         EINVAL,
2111                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2112                                         NULL, "Meter policy invalid.");
2113                 policy = sub_policy->main_policy;
2114                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2115                         continue;
2116                 next_fm = mlx5_flow_meter_find(priv,
2117                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2118                                 &mtr_idx);
2119                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2120                                                     error, true))
2121                         return -rte_mtr_error_set(error,
2122                                         EINVAL,
2123                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2124                                         NULL, "Meter policy invalid.");
2125                 mlx5_free(policy);
2126                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2127                         continue;
2128                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2129                                                     mtr_idx, error))
2130                         return -rte_errno;
2131         }
2132         return 0;
2133 }
2134 /**
2135  * Flush meter configuration.
2136  *
2137  * @param[in] dev
2138  *   Pointer to Ethernet device.
2139  * @param[out] error
2140  *   Pointer to rte meter error structure.
2141  *
2142  * @return
2143  *   0 on success, a negative errno value otherwise and rte_errno is set.
2144  */
2145 int
2146 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2147 {
2148         struct mlx5_priv *priv = dev->data->dev_private;
2149         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2150         struct mlx5_flow_meter_profile *fmp;
2151         struct mlx5_legacy_flow_meter *legacy_fm;
2152         struct mlx5_flow_meter_info *fm;
2153         struct mlx5_flow_meter_sub_policy *sub_policy;
2154         void *tmp;
2155         uint32_t i, mtr_idx, policy_idx;
2156         void *entry;
2157         struct mlx5_aso_mtr *aso_mtr;
2158
2159         if (!priv->mtr_en)
2160                 return 0;
2161         if (priv->sh->meter_aso_en) {
2162                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2163                         return -rte_errno;
2164                 if (priv->mtr_idx_tbl) {
2165                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2166                                 mtr_idx = *(uint32_t *)entry;
2167                                 if (mtr_idx) {
2168                                         aso_mtr =
2169                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2170                                         fm = &aso_mtr->fm;
2171                                         (void)mlx5_flow_meter_params_flush(dev,
2172                                                 fm, mtr_idx);
2173                                 }
2174                         }
2175                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2176                         priv->mtr_idx_tbl = NULL;
2177                 }
2178         } else {
2179                 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2180                         fm = &legacy_fm->fm;
2181                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2182                                 return -rte_mtr_error_set(error, EINVAL,
2183                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2184                                 NULL, "MTR object meter profile invalid.");
2185                 }
2186         }
2187         if (priv->policy_idx_tbl) {
2188                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2189                         policy_idx = *(uint32_t *)entry;
2190                         sub_policy = mlx5_ipool_get
2191                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2192                                 policy_idx);
2193                         if (!sub_policy)
2194                                 return -rte_mtr_error_set(error,
2195                                                 EINVAL,
2196                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2197                                                 NULL, "MTR object "
2198                                                 "meter policy invalid.");
2199                         if (__mlx5_flow_meter_policy_delete(dev, i,
2200                                                 sub_policy->main_policy,
2201                                                 error, true))
2202                                 return -rte_mtr_error_set(error,
2203                                                 EINVAL,
2204                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2205                                                 NULL, "MTR object "
2206                                                 "meter policy invalid.");
2207                         mlx5_free(sub_policy->main_policy);
2208                 }
2209                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2210                 priv->policy_idx_tbl = NULL;
2211         }
2212         if (priv->mtr_profile_tbl) {
2213                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2214                         fmp = entry;
2215                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2216                                                            error))
2217                                 return -rte_mtr_error_set(error, EINVAL,
2218                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2219                                                 NULL, "Fail to destroy "
2220                                                 "meter profile.");
2221                 }
2222                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2223                 priv->mtr_profile_tbl = NULL;
2224         }
2225         /* Delete default policy table. */
2226         mlx5_flow_destroy_def_policy(dev);
2227         if (priv->sh->refcnt == 1)
2228                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2229         return 0;
2230 }