drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133
 134         /* Profile must not be NULL. */
 135         if (profile == NULL)
 136                 return -rte_mtr_error_set(error, EINVAL,
 137                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 138                                           NULL, "Meter profile is null.");
 139         /* Meter profile ID must be valid. */
 140         if (meter_profile_id == UINT32_MAX)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 143                                           NULL, "Meter profile id not valid.");
 144         /* Meter profile must not exist. */
 145         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 146         if (fmp)
 147                 return -rte_mtr_error_set(error, EEXIST,
 148                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 149                                           NULL,
 150                                           "Meter profile already exists.");
 151         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 152                 if (priv->config.hca_attr.qos.flow_meter_old) {
 153                         /* Verify support for flow meter parameters. */
 154                         if (priv->sh->meter_aso_en && profile->packet_mode) {
 155                                 if (profile->srtcm_rfc2697.cir > 0 &&
 156                                         (profile->srtcm_rfc2697.cir <<
 157                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 158                                         <= MLX5_SRTCM_CIR_MAX &&
 159                                         profile->srtcm_rfc2697.cbs > 0 &&
 160                                         (profile->srtcm_rfc2697.cbs <<
 161                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 162                                         <= MLX5_SRTCM_CBS_MAX &&
 163                                         (profile->srtcm_rfc2697.ebs <<
 164                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 165                                         <= MLX5_SRTCM_EBS_MAX)
 166                                         return 0;
 167                                 return -rte_mtr_error_set
 168                                              (error, ENOTSUP,
 169                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 170                                               NULL,
 171                                               profile->srtcm_rfc2697.ebs ?
 172                                               "Metering value ebs must be 0." :
 173                                               "Invalid metering parameters.");
 174                         }
 175                         if (profile->srtcm_rfc2697.cir > 0 &&
 176                                 profile->srtcm_rfc2697.cir <=
 177                                                 MLX5_SRTCM_CIR_MAX &&
 178                                 profile->srtcm_rfc2697.cbs > 0 &&
 179                                 profile->srtcm_rfc2697.cbs <=
 180                                                 MLX5_SRTCM_CBS_MAX &&
 181                                 profile->srtcm_rfc2697.ebs <=
 182                                                 MLX5_SRTCM_EBS_MAX)
 183                                 return 0;
 184                         return -rte_mtr_error_set(error, ENOTSUP,
 185                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 186                                         NULL,
 187                                         profile->srtcm_rfc2697.ebs ?
 188                                         "Metering value ebs must be 0." :
 189                                         "Invalid metering parameters.");
 190                 }
 191         }
 192         return -rte_mtr_error_set(error, ENOTSUP,
 193                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 194                                   NULL, "Metering algorithm not supported.");
 195 }
 196
 197 /*
 198  * Calculate mantissa and exponent for cir / eir.
 199  *
 200  * @param[in] xir
 201  *   Value to be calculated.
 202  * @param[out] man
 203  *   Pointer to the mantissa.
 204  * @param[out] exp
 205  *   Pointer to the exp.
 206  */
 207 static inline void
 208 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp)
 209 {
 210         int64_t _cir;
 211         int64_t delta = INT64_MAX;
 212         uint8_t _man = 0;
 213         uint8_t _exp = 0;
 214         uint64_t m, e;
 215
 216         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 217                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 218                         _cir = (1000000000ULL * m) >> e;
 219                         if (llabs(xir - _cir) <= delta) {
 220                                 delta = llabs(xir - _cir);
 221                                 _man = m;
 222                                 _exp = e;
 223                         }
 224                 }
 225         }
 226         *man = _man;
 227         *exp = _exp;
 228 }
 229
 230 /*
 231  * Calculate mantissa and exponent for xbs.
 232  *
 233  * @param[in] xbs
 234  *   Value to be calculated.
 235  * @param[out] man
 236  *   Pointer to the mantissa.
 237  * @param[out] exp
 238  *   Pointer to the exp.
 239  */
 240 static inline void
 241 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 242 {
 243         int _exp;
 244         double _man;
 245
 246         /* Special case xbs == 0 ? both exp and matissa are 0. */
 247         if (xbs == 0) {
 248                 *man = 0;
 249                 *exp = 0;
 250                 return;
 251         }
 252         /* xbs = xbs_mantissa * 2^xbs_exponent */
 253         _man = frexp(xbs, &_exp);
 254         _man = _man * pow(2, MLX5_MAN_WIDTH);
 255         _exp = _exp - MLX5_MAN_WIDTH;
 256         *man = (uint8_t)ceil(_man);
 257         *exp = _exp;
 258 }
 259
 260 /**
 261  * Fill the prm meter parameter.
 262  *
 263  * @param[in,out] fmp
 264  *   Pointer to meter profie to be converted.
 265  * @param[out] error
 266  *   Pointer to the error structure.
 267  *
 268  * @return
 269  *   0 on success, a negative errno value otherwise and rte_errno is set.
 270  */
 271 static int
 272 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 273                         struct mlx5_priv *priv, struct rte_mtr_error *error)
 274 {
 275         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 276         uint8_t man, exp;
 277         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 278         uint32_t eir_exp, eir_man, ebs_exp, ebs_man;
 279         uint64_t cir, cbs, eir, ebs;
 280
 281         if (!priv->sh->meter_aso_en) {
 282                 /* Legacy FW metering will only support srTCM. */
 283                 if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 284                         return -rte_mtr_error_set(error, ENOTSUP,
 285                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 286                                 NULL, "Metering algorithm is not supported.");
 287                 if (fmp->profile.packet_mode)
 288                         return -rte_mtr_error_set(error, ENOTSUP,
 289                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 290                                 "Metering algorithm packet mode is not supported.");
 291         }
 292         switch (fmp->profile.alg) {
 293         case RTE_MTR_SRTCM_RFC2697:
 294                 cir = fmp->profile.srtcm_rfc2697.cir;
 295                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 296                 eir = 0;
 297                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 298                 break;
 299         case RTE_MTR_TRTCM_RFC2698:
 300                 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir >
 301                             fmp->profile.trtcm_rfc2698.cir &&
 302                             fmp->profile.trtcm_rfc2698.pbs >
 303                             fmp->profile.trtcm_rfc2698.cbs);
 304                 cir = fmp->profile.trtcm_rfc2698.cir;
 305                 cbs = fmp->profile.trtcm_rfc2698.cbs;
 306                 /* EIR / EBS are filled with PIR / PBS. */
 307                 eir = fmp->profile.trtcm_rfc2698.pir;
 308                 ebs = fmp->profile.trtcm_rfc2698.pbs;
 309                 break;
 310         case RTE_MTR_TRTCM_RFC4115:
 311                 cir = fmp->profile.trtcm_rfc4115.cir;
 312                 cbs = fmp->profile.trtcm_rfc4115.cbs;
 313                 eir = fmp->profile.trtcm_rfc4115.eir;
 314                 ebs = fmp->profile.trtcm_rfc4115.ebs;
 315                 break;
 316         default:
 317                 return -rte_mtr_error_set(error, EINVAL,
 318                                 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
 319                                 "Metering algorithm mode is invalid");
 320         }
 321         /* Adjust the values for PPS mode. */
 322         if (fmp->profile.packet_mode) {
 323                 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 324                 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 325                 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 326                 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 327         }
 328         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 329         mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp);
 330         /* Check if cir mantissa is too large. */
 331         if (exp > ASO_DSEG_XIR_EXP_MASK)
 332                 return -rte_mtr_error_set(error, ENOTSUP,
 333                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 334                                           "meter profile parameter cir is not supported.");
 335         cir_man = man;
 336         cir_exp = exp;
 337          /* cbs = cbs_mantissa * 2^cbs_exponent */
 338         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 339         /* Check if cbs mantissa is too large. */
 340         if (exp > ASO_DSEG_EXP_MASK)
 341                 return -rte_mtr_error_set(error, ENOTSUP,
 342                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 343                                           "meter profile parameter cbs is not supported.");
 344         cbs_man = man;
 345         cbs_exp = exp;
 346         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 347                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 348                                           cir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 349                                           cir_man);
 350         mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp);
 351         /* Check if eir mantissa is too large. */
 352         if (exp > ASO_DSEG_XIR_EXP_MASK)
 353                 return -rte_mtr_error_set(error, ENOTSUP,
 354                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 355                                           "meter profile parameter eir is not supported.");
 356         eir_man = man;
 357         eir_exp = exp;
 358         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 359         /* Check if ebs mantissa is too large. */
 360         if (exp > ASO_DSEG_EXP_MASK)
 361                 return -rte_mtr_error_set(error, ENOTSUP,
 362                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 363                                           "meter profile parameter ebs is not supported.");
 364         ebs_man = man;
 365         ebs_exp = exp;
 366         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 367                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET |
 368                                           eir_exp << ASO_DSEG_XIR_EXP_OFFSET |
 369                                           eir_man);
 370         if (srtcm->cbs_cir)
 371                 fmp->g_support = 1;
 372         if (srtcm->ebs_eir)
 373                 fmp->y_support = 1;
 374         return 0;
 375 }
 376
 377 /**
 378  * Callback to get MTR capabilities.
 379  *
 380  * @param[in] dev
 381  *   Pointer to Ethernet device.
 382  * @param[out] cap
 383  *   Pointer to save MTR capabilities.
 384  * @param[out] error
 385  *   Pointer to the error structure.
 386  *
 387  * @return
 388  *   0 on success, a negative errno value otherwise and rte_errno is set.
 389  */
 390 static int
 391 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 392                  struct rte_mtr_capabilities *cap,
 393                  struct rte_mtr_error *error __rte_unused)
 394 {
 395         struct mlx5_priv *priv = dev->data->dev_private;
 396         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 397
 398         if (!priv->mtr_en)
 399                 return -rte_mtr_error_set(error, ENOTSUP,
 400                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 401                                           "Meter is not supported");
 402         memset(cap, 0, sizeof(*cap));
 403         if (priv->sh->meter_aso_en) {
 404                 /* 2 meters per one ASO cache line. */
 405                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 406                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 407         } else {
 408                 cap->n_max = 1 << qattr->log_max_flow_meter;
 409                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 410         }
 411         cap->srtcm_rfc2697_byte_mode_supported = 1;
 412         cap->n_shared_max = cap->n_max;
 413         cap->identical = 1;
 414         cap->shared_identical = 1;
 415         cap->shared_n_flows_per_mtr_max = 4 << 20;
 416         /* 2M flows can share the same meter. */
 417         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 418         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 419         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 420         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 421                           RTE_MTR_STATS_N_PKTS_DROPPED;
 422         return 0;
 423 }
 424
 425 /**
 426  * Callback to add MTR profile.
 427  *
 428  * @param[in] dev
 429  *   Pointer to Ethernet device.
 430  * @param[in] meter_profile_id
 431  *   Meter profile id.
 432  * @param[in] profile
 433  *   Pointer to meter profile detail.
 434  * @param[out] error
 435  *   Pointer to the error structure.
 436  *
 437  * @return
 438  *   0 on success, a negative errno value otherwise and rte_errno is set.
 439  */
 440 static int
 441 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 442                        uint32_t meter_profile_id,
 443                        struct rte_mtr_meter_profile *profile,
 444                        struct rte_mtr_error *error)
 445 {
 446         struct mlx5_priv *priv = dev->data->dev_private;
 447         struct mlx5_flow_meter_profile *fmp;
 448         union mlx5_l3t_data data;
 449         int ret;
 450
 451         if (!priv->mtr_en)
 452                 return -rte_mtr_error_set(error, ENOTSUP,
 453                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 454                                           "Meter is not supported");
 455         /* Check input params. */
 456         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 457                                                profile, error);
 458         if (ret)
 459                 return ret;
 460         /* Meter profile memory allocation. */
 461         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 462                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 463         if (fmp == NULL)
 464                 return -rte_mtr_error_set(error, ENOMEM,
 465                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 466                                           NULL, "Meter profile memory "
 467                                           "alloc failed.");
 468         /* Fill profile info. */
 469         fmp->id = meter_profile_id;
 470         fmp->profile = *profile;
 471         /* Fill the flow meter parameters for the PRM. */
 472         ret = mlx5_flow_meter_param_fill(fmp, priv, error);
 473         if (ret)
 474                 goto error;
 475         data.ptr = fmp;
 476         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 477                                  meter_profile_id, &data);
 478         if (ret)
 479                 return -rte_mtr_error_set(error, ENOTSUP,
 480                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 481                                           NULL, "Meter profile insert fail.");
 482         return 0;
 483 error:
 484         mlx5_free(fmp);
 485         return ret;
 486 }
 487
 488 /**
 489  * Callback to delete MTR profile.
 490  *
 491  * @param[in] dev
 492  *   Pointer to Ethernet device.
 493  * @param[in] meter_profile_id
 494  *   Meter profile id.
 495  * @param[out] error
 496  *   Pointer to the error structure.
 497  *
 498  * @return
 499  *   0 on success, a negative errno value otherwise and rte_errno is set.
 500  */
 501 static int
 502 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 503                           uint32_t meter_profile_id,
 504                           struct rte_mtr_error *error)
 505 {
 506         struct mlx5_priv *priv = dev->data->dev_private;
 507         struct mlx5_flow_meter_profile *fmp;
 508
 509         if (!priv->mtr_en)
 510                 return -rte_mtr_error_set(error, ENOTSUP,
 511                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 512                                           "Meter is not supported");
 513         /* Meter profile must exist. */
 514         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 515         if (fmp == NULL)
 516                 return -rte_mtr_error_set(error, ENOENT,
 517                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 518                                           &meter_profile_id,
 519                                           "Meter profile id is invalid.");
 520         /* Check profile is unused. */
 521         if (fmp->ref_cnt)
 522                 return -rte_mtr_error_set(error, EBUSY,
 523                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 524                                           NULL, "Meter profile is in use.");
 525         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 526                 return -rte_mtr_error_set(error, EBUSY,
 527                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 528                                           NULL, "Meter profile remove fail.");
 529         mlx5_free(fmp);
 530         return 0;
 531 }
 532
 533 /**
 534  * Find policy by id.
 535  *
 536  * @param[in] dev
 537  *   Pointer to Ethernet device.
 538  * @param policy_id
 539  *   Policy id.
 540  *
 541  * @return
 542  *   Pointer to the policy found on success, NULL otherwise.
 543  */
 544 struct mlx5_flow_meter_policy *
 545 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 546                             uint32_t policy_id,
 547                             uint32_t *policy_idx)
 548 {
 549         struct mlx5_priv *priv = dev->data->dev_private;
 550         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 551         union mlx5_l3t_data data;
 552
 553         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 554                 return NULL;
 555         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 556                                 !data.dword)
 557                 return NULL;
 558         if (policy_idx)
 559                 *policy_idx = data.dword;
 560         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 561                                         data.dword);
 562         /* Remove reference taken by the mlx5_l3t_get_entry. */
 563         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 564         if (sub_policy)
 565                 if (sub_policy->main_policy_id)
 566                         return sub_policy->main_policy;
 567         return NULL;
 568 }
 569
 570 /**
 571  * Get the last meter's policy from one meter's policy in hierarchy.
 572  *
 573  * @param[in] dev
 574  *   Pointer to Ethernet device.
 575  * @param[in] policy
 576  *   Pointer to flow meter policy.
 577  *
 578  * @return
 579  *   Pointer to the final meter's policy, or NULL when fail.
 580  */
 581 struct mlx5_flow_meter_policy *
 582 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 583                                         struct mlx5_flow_meter_policy *policy)
 584 {
 585         struct mlx5_priv *priv = dev->data->dev_private;
 586         struct mlx5_flow_meter_info *next_fm;
 587         struct mlx5_flow_meter_policy *next_policy = policy;
 588
 589         while (next_policy->is_hierarchy) {
 590                 next_fm = mlx5_flow_meter_find(priv,
 591                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 592                 if (!next_fm || next_fm->def_policy)
 593                         return NULL;
 594                 next_policy = mlx5_flow_meter_policy_find(dev,
 595                                                 next_fm->policy_id, NULL);
 596                 MLX5_ASSERT(next_policy);
 597         }
 598         return next_policy;
 599 }
 600
 601 /**
 602  * Callback to check MTR policy action validate
 603  *
 604  * @param[in] dev
 605  *   Pointer to Ethernet device.
 606  * @param[in] actions
 607  *   Pointer to meter policy action detail.
 608  * @param[out] error
 609  *   Pointer to the error structure.
 610  *
 611  * @return
 612  *   0 on success, a negative errno value otherwise and rte_errno is set.
 613  */
 614 static int
 615 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 616         struct rte_mtr_meter_policy_params *policy,
 617         struct rte_mtr_error *error)
 618 {
 619         struct mlx5_priv *priv = dev->data->dev_private;
 620         struct rte_flow_attr attr = { .transfer =
 621                         priv->config.dv_esw_en ? 1 : 0};
 622         bool is_rss = false;
 623         uint8_t policy_mode;
 624         uint8_t domain_bitmap;
 625         int ret;
 626
 627         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 628                 return -rte_mtr_error_set(error, ENOTSUP,
 629                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 630                                 NULL, "meter policy unsupported.");
 631         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 632                         &is_rss, &domain_bitmap, &policy_mode, error);
 633         if (ret)
 634                 return ret;
 635         return 0;
 636 }
 637
 638 static int
 639 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 640                         uint32_t policy_id,
 641                         struct mlx5_flow_meter_policy *mtr_policy,
 642                         struct rte_mtr_error *error,
 643                         bool clear_l3t)
 644 {
 645         struct mlx5_priv *priv = dev->data->dev_private;
 646         struct mlx5_flow_meter_sub_policy *sub_policy;
 647         uint32_t i, j;
 648         uint16_t sub_policy_num;
 649
 650         rte_spinlock_lock(&mtr_policy->sl);
 651         if (mtr_policy->ref_cnt) {
 652                 rte_spinlock_unlock(&mtr_policy->sl);
 653                 return -rte_mtr_error_set(error, EBUSY,
 654                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 655                                  NULL,
 656                                 "Meter policy object is being used.");
 657         }
 658         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 659         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 660         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 661                 sub_policy_num = (mtr_policy->sub_policy_num >>
 662                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 663                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 664                 if (sub_policy_num) {
 665                         for (j = 0; j < sub_policy_num; j++) {
 666                                 sub_policy = mtr_policy->sub_policys[i][j];
 667                                 if (sub_policy)
 668                                         mlx5_ipool_free
 669                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 670                                         sub_policy->idx);
 671                         }
 672                 }
 673         }
 674         if (priv->policy_idx_tbl && clear_l3t) {
 675                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 676                         rte_spinlock_unlock(&mtr_policy->sl);
 677                         return -rte_mtr_error_set(error, ENOTSUP,
 678                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 679                                 "Fail to delete policy in index table.");
 680                 }
 681         }
 682         rte_spinlock_unlock(&mtr_policy->sl);
 683         return 0;
 684 }
 685
 686 /**
 687  * Callback to add MTR policy.
 688  *
 689  * @param[in] dev
 690  *   Pointer to Ethernet device.
 691  * @param[out] policy_id
 692  *   Pointer to policy id
 693  * @param[in] actions
 694  *   Pointer to meter policy action detail.
 695  * @param[out] error
 696  *   Pointer to the error structure.
 697  *
 698  * @return
 699  *   0 on success, a negative errno value otherwise and rte_errno is set.
 700  */
 701 static int
 702 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 703                         uint32_t policy_id,
 704                         struct rte_mtr_meter_policy_params *policy,
 705                         struct rte_mtr_error *error)
 706 {
 707         struct mlx5_priv *priv = dev->data->dev_private;
 708         struct rte_flow_attr attr = { .transfer =
 709                         priv->config.dv_esw_en ? 1 : 0};
 710         uint32_t sub_policy_idx = 0;
 711         uint32_t policy_idx = 0;
 712         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 713         struct mlx5_flow_meter_sub_policy *sub_policy;
 714         bool is_rss = false;
 715         uint8_t policy_mode;
 716         uint32_t i;
 717         int ret;
 718         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 719         uint16_t sub_policy_num;
 720         uint8_t domain_bitmap = 0;
 721         union mlx5_l3t_data data;
 722         bool skip_rule = false;
 723
 724         if (!priv->mtr_en)
 725                 return -rte_mtr_error_set(error, ENOTSUP,
 726                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 727                                           NULL, "meter policy unsupported. ");
 728         if (policy_id == MLX5_INVALID_POLICY_ID)
 729                 return -rte_mtr_error_set(error, ENOTSUP,
 730                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 731                                           NULL, "policy ID is invalid. ");
 732         if (policy_id == priv->sh->mtrmng->def_policy_id)
 733                 return -rte_mtr_error_set(error, EEXIST,
 734                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 735                                           NULL, "default policy ID exists. ");
 736         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 737         if (mtr_policy)
 738                 return -rte_mtr_error_set(error, EEXIST,
 739                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 740                                           NULL, "policy ID exists. ");
 741         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 742                                           &is_rss, &domain_bitmap,
 743                                           &policy_mode, error);
 744         if (ret)
 745                 return ret;
 746         if (!domain_bitmap)
 747                 return -rte_mtr_error_set(error, ENOTSUP,
 748                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 749                                           NULL, "fail to find policy domain.");
 750         if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
 751                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 752                         return -rte_mtr_error_set(error, EEXIST,
 753                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 754                                 NULL, "a policy with similar actions "
 755                                 "is already configured");
 756                 if (mlx5_flow_create_def_policy(dev))
 757                         return -rte_mtr_error_set(error, ENOTSUP,
 758                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 759                                 NULL,
 760                                 "fail to create non-terminated policy.");
 761                 priv->sh->mtrmng->def_policy_id = policy_id;
 762                 return 0;
 763         }
 764         if (!priv->sh->meter_aso_en)
 765                 return -rte_mtr_error_set(error, ENOTSUP,
 766                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 767                         "no ASO capability to support the policy ");
 768         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 769                 if (!(domain_bitmap & (1 << i)))
 770                         continue;
 771                 /*
 772                  * If RSS is found, it means that only the ingress domain can
 773                  * be supported. It is invalid to support RSS for one color
 774                  * and egress / transfer domain actions for another. Drop and
 775                  * jump action should have no impact.
 776                  */
 777                 if (is_rss) {
 778                         policy_size +=
 779                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 780                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 781                         break;
 782                 }
 783                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 784         }
 785         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 786                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 787         if (!mtr_policy)
 788                 return -rte_mtr_error_set(error, ENOMEM,
 789                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 790                                 "Memory alloc failed for meter policy.");
 791         if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
 792                 mtr_policy->skip_y = 1;
 793         else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
 794                 mtr_policy->skip_g = 1;
 795         policy_size = sizeof(struct mlx5_flow_meter_policy);
 796         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 797                 if (!(domain_bitmap & (1 << i)))
 798                         continue;
 799                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 800                         mtr_policy->ingress = 1;
 801                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 802                         mtr_policy->egress = 1;
 803                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 804                         mtr_policy->transfer = 1;
 805                 sub_policy = mlx5_ipool_zmalloc
 806                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 807                                  &sub_policy_idx);
 808                 if (!sub_policy ||
 809                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 810                         goto policy_add_err;
 811                 sub_policy->idx = sub_policy_idx;
 812                 sub_policy->main_policy = mtr_policy;
 813                 if (!policy_idx) {
 814                         policy_idx = sub_policy_idx;
 815                         sub_policy->main_policy_id = 1;
 816                 }
 817                 mtr_policy->sub_policys[i] =
 818                         (struct mlx5_flow_meter_sub_policy **)
 819                         ((uint8_t *)mtr_policy + policy_size);
 820                 mtr_policy->sub_policys[i][0] = sub_policy;
 821                 sub_policy_num = (mtr_policy->sub_policy_num >>
 822                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 823                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 824                 sub_policy_num++;
 825                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 826                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 827                 mtr_policy->sub_policy_num |=
 828                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 829                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 830                 /*
 831                  * If RSS is found, it means that only the ingress domain can
 832                  * be supported. It is invalid to support RSS for one color
 833                  * and egress / transfer domain actions for another. Drop and
 834                  * jump action should have no impact.
 835                  */
 836                 if (is_rss) {
 837                         mtr_policy->is_rss = 1;
 838                         break;
 839                 }
 840                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 841         }
 842         rte_spinlock_init(&mtr_policy->sl);
 843         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 844                                         policy->actions, error);
 845         if (ret)
 846                 goto policy_add_err;
 847         if (mtr_policy->is_hierarchy) {
 848                 struct mlx5_flow_meter_policy *final_policy;
 849
 850                 final_policy =
 851                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 852                 if (!final_policy)
 853                         goto policy_add_err;
 854                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 855         }
 856         /*
 857          * If either Green or Yellow has queue / RSS action, all the policy
 858          * rules will be created later in the flow splitting stage.
 859          */
 860         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 861                 /* Create policy rules in HW. */
 862                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 863                 if (ret)
 864                         goto policy_add_err;
 865         }
 866         data.dword = policy_idx;
 867         if (!priv->policy_idx_tbl) {
 868                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 869                 if (!priv->policy_idx_tbl)
 870                         goto policy_add_err;
 871         }
 872         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 873                 goto policy_add_err;
 874         return 0;
 875 policy_add_err:
 876         if (mtr_policy) {
 877                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 878                         mtr_policy, error, false);
 879                 mlx5_free(mtr_policy);
 880                 if (ret)
 881                         return ret;
 882         }
 883         return -rte_mtr_error_set(error, ENOTSUP,
 884                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 885                                   NULL, "Failed to create devx policy.");
 886 }
 887
 888 /**
 889  * Callback to delete MTR policy.
 890  *
 891  * @param[in] dev
 892  *   Pointer to Ethernet device.
 893  * @param[in] policy_id
 894  *   Meter policy id.
 895  * @param[out] error
 896  *   Pointer to the error structure.
 897  *
 898  * @return
 899  *   0 on success, a negative errno value otherwise and rte_errno is set.
 900  */
 901 static int
 902 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 903                           uint32_t policy_id,
 904                           struct rte_mtr_error *error)
 905 {
 906         struct mlx5_priv *priv = dev->data->dev_private;
 907         struct mlx5_flow_meter_policy *mtr_policy;
 908         uint32_t policy_idx;
 909         int ret;
 910
 911         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 912                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 913                         return -rte_mtr_error_set(error, ENOTSUP,
 914                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 915                                 "Meter policy object is being used.");
 916                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 917                 return 0;
 918         }
 919         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 920         if (!mtr_policy)
 921                 return -rte_mtr_error_set(error, ENOTSUP,
 922                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 923                         "Meter policy id is invalid. ");
 924         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 925                                                 error, true);
 926         if (ret)
 927                 return ret;
 928         mlx5_free(mtr_policy);
 929         return 0;
 930 }
 931
 932 /**
 933  * Check meter validation.
 934  *
 935  * @param[in] priv
 936  *   Pointer to mlx5 private data structure.
 937  * @param[in] meter_id
 938  *   Meter id.
 939  * @param[in] params
 940  *   Pointer to rte meter parameters.
 941  * @param[out] error
 942  *   Pointer to rte meter error structure.
 943  *
 944  * @return
 945  *   0 on success, a negative errno value otherwise and rte_errno is set.
 946  */
 947 static int
 948 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 949                          struct rte_mtr_params *params,
 950                          struct rte_mtr_error *error)
 951 {
 952         /* Meter must use global drop action. */
 953         if (!priv->sh->dr_drop_action)
 954                 return -rte_mtr_error_set(error, ENOTSUP,
 955                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 956                                           NULL,
 957                                           "No drop action ready for meter.");
 958         /* Meter params must not be NULL. */
 959         if (params == NULL)
 960                 return -rte_mtr_error_set(error, EINVAL,
 961                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 962                                           NULL, "Meter object params null.");
 963         /* Previous meter color is not supported. */
 964         if (params->use_prev_mtr_color)
 965                 return -rte_mtr_error_set(error, ENOTSUP,
 966                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 967                                           NULL,
 968                                           "Previous meter color "
 969                                           "not supported.");
 970         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 971                 return -rte_mtr_error_set(error, ENOENT,
 972                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 973                                 NULL, "Meter policy id not valid.");
 974         /* Validate meter id. */
 975         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 976                 return -rte_mtr_error_set(error, EEXIST,
 977                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 978                         "Meter object already exists.");
 979         return 0;
 980 }
 981
 982 /**
 983  * Modify the flow meter action.
 984  *
 985  * @param[in] priv
 986  *   Pointer to mlx5 private data structure.
 987  * @param[in] fm
 988  *   Pointer to flow meter to be modified.
 989  * @param[in] srtcm
 990  *   Pointer to meter srtcm description parameter.
 991  * @param[in] modify_bits
 992  *   The bit in srtcm to be updated.
 993  * @param[in] active_state
 994  *   The state to be updated.
 995  * @return
 996  *   0 on success, o negative value otherwise.
 997  */
 998 static int
 999 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
1000                 struct mlx5_flow_meter_info *fm,
1001                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
1002                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
1003 {
1004 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1005         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
1006         uint32_t *attr;
1007         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
1008         int ret;
1009         struct mlx5_aso_mtr *aso_mtr = NULL;
1010         uint32_t cbs_cir, ebs_eir, val;
1011
1012         if (priv->sh->meter_aso_en) {
1013                 fm->is_enable = !!is_enable;
1014                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1015                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1016                 if (ret)
1017                         return ret;
1018                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
1019                 if (ret)
1020                         return ret;
1021         } else {
1022                 /* Fill command parameters. */
1023                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
1024                 mod_attr.flow_meter_parameter = in;
1025                 mod_attr.flow_meter_parameter_sz =
1026                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
1027                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1028                         mod_attr.active = !!active_state;
1029                 else
1030                         mod_attr.active = 0;
1031                 attr = in;
1032                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
1033                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1034                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1035                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1036                                 ASO_DSEG_EXP_MASK;
1037                         MLX5_SET(flow_meter_parameters, attr,
1038                                 cbs_exponent, val);
1039                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1040                                 ASO_DSEG_MAN_MASK;
1041                         MLX5_SET(flow_meter_parameters, attr,
1042                                 cbs_mantissa, val);
1043                 }
1044                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1045                         val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) &
1046                                 ASO_DSEG_EXP_MASK;
1047                         MLX5_SET(flow_meter_parameters, attr,
1048                                 cir_exponent, val);
1049                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1050                         MLX5_SET(flow_meter_parameters, attr,
1051                                 cir_mantissa, val);
1052                 }
1053                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1054                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1055                                 ASO_DSEG_EXP_MASK;
1056                         MLX5_SET(flow_meter_parameters, attr,
1057                                 ebs_exponent, val);
1058                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1059                                 ASO_DSEG_MAN_MASK;
1060                         MLX5_SET(flow_meter_parameters, attr,
1061                                 ebs_mantissa, val);
1062                 }
1063                 /* Apply modifications to meter only if it was created. */
1064                 if (fm->meter_action) {
1065                         ret = mlx5_glue->dv_modify_flow_action_meter
1066                                         (fm->meter_action, &mod_attr,
1067                                         rte_cpu_to_be_64(modify_bits));
1068                         if (ret)
1069                                 return ret;
1070                 }
1071                 /* Update succeedded modify meter parameters. */
1072                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1073                         fm->active_state = !!active_state;
1074         }
1075         return 0;
1076 #else
1077         (void)priv;
1078         (void)fm;
1079         (void)srtcm;
1080         (void)modify_bits;
1081         (void)active_state;
1082         (void)is_enable;
1083         return -ENOTSUP;
1084 #endif
1085 }
1086
1087 static int
1088 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1089                                 struct mlx5_flow_meter_info *fm,
1090                                 uint64_t stats_mask)
1091 {
1092         fm->bytes_dropped =
1093                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1094         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1095         if (fm->bytes_dropped || fm->pkts_dropped) {
1096                 if (!fm->drop_cnt) {
1097                         /* Alloc policer counters. */
1098                         fm->drop_cnt = mlx5_counter_alloc(dev);
1099                         if (!fm->drop_cnt)
1100                                 return -1;
1101                 }
1102         } else {
1103                 if (fm->drop_cnt) {
1104                         mlx5_counter_free(dev, fm->drop_cnt);
1105                         fm->drop_cnt = 0;
1106                 }
1107         }
1108         return 0;
1109 }
1110
1111 /**
1112  * Create meter rules.
1113  *
1114  * @param[in] dev
1115  *   Pointer to Ethernet device.
1116  * @param[in] meter_id
1117  *   Meter id.
1118  * @param[in] params
1119  *   Pointer to rte meter parameters.
1120  * @param[in] shared
1121  *   Meter shared with other flow or not.
1122  * @param[out] error
1123  *   Pointer to rte meter error structure.
1124  *
1125  * @return
1126  *   0 on success, a negative errno value otherwise and rte_errno is set.
1127  */
1128 static int
1129 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1130                        struct rte_mtr_params *params, int shared,
1131                        struct rte_mtr_error *error)
1132 {
1133         struct mlx5_priv *priv = dev->data->dev_private;
1134         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1135         struct mlx5_flow_meter_profile *fmp;
1136         struct mlx5_flow_meter_info *fm;
1137         struct mlx5_legacy_flow_meter *legacy_fm;
1138         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1139         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1140                 .size = 0,
1141                 .trunk_size = 64,
1142                 .need_lock = 1,
1143                 .type = "mlx5_flow_mtr_flow_id_pool",
1144         };
1145         struct mlx5_aso_mtr *aso_mtr;
1146         uint32_t mtr_idx, policy_idx;
1147         union mlx5_l3t_data data;
1148         int ret;
1149         uint8_t domain_bitmap;
1150         uint8_t mtr_id_bits;
1151         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1152                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1153
1154         if (!priv->mtr_en)
1155                 return -rte_mtr_error_set(error, ENOTSUP,
1156                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1157                                         "Meter is not supported");
1158         /* Validate the parameters. */
1159         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1160         if (ret)
1161                 return ret;
1162         /* Meter profile must exist. */
1163         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1164         if (fmp == NULL)
1165                 return -rte_mtr_error_set(error, ENOENT,
1166                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1167                         NULL, "Meter profile id not valid.");
1168         /* Meter policy must exist. */
1169         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1170                 __atomic_add_fetch
1171                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1172                         1, __ATOMIC_RELAXED);
1173                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1174                 if (!priv->config.dv_esw_en)
1175                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1176         } else {
1177                 if (!priv->sh->meter_aso_en)
1178                         return -rte_mtr_error_set(error, ENOTSUP,
1179                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1180                                 "Part of the policies cannot be "
1181                                 "supported without ASO ");
1182                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1183                                 params->meter_policy_id, &policy_idx);
1184                 if (!mtr_policy)
1185                         return -rte_mtr_error_set(error, ENOENT,
1186                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1187                                 NULL, "Meter policy id not valid.");
1188                 domain_bitmap = (mtr_policy->ingress ?
1189                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1190                                 (mtr_policy->egress ?
1191                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1192                                 (mtr_policy->transfer ?
1193                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1194                 if (fmp->g_support && mtr_policy->skip_g)
1195                         return -rte_mtr_error_set(error, ENOTSUP,
1196                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1197                                         NULL, "Meter green policy is empty.");
1198                 if (fmp->y_support && mtr_policy->skip_y)
1199                         return -rte_mtr_error_set(error, ENOTSUP,
1200                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1201                                         NULL, "Meter yellow policy is empty.");
1202         }
1203         /* Allocate the flow meter memory. */
1204         if (priv->sh->meter_aso_en) {
1205                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1206                 if (!mtr_idx)
1207                         return -rte_mtr_error_set(error, ENOMEM,
1208                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1209                                 "Memory alloc failed for meter.");
1210                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1211                 fm = &aso_mtr->fm;
1212         } else {
1213                 legacy_fm = mlx5_ipool_zmalloc
1214                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1215                 if (legacy_fm == NULL)
1216                         return -rte_mtr_error_set(error, ENOMEM,
1217                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1218                                 "Memory alloc failed for meter.");
1219                 legacy_fm->idx = mtr_idx;
1220                 fm = &legacy_fm->fm;
1221         }
1222         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1223         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1224             mtr_reg_bits) {
1225                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1226                 goto error;
1227         }
1228         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1229                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1230         /* Fill the flow meter parameters. */
1231         fm->meter_id = meter_id;
1232         fm->policy_id = params->meter_policy_id;
1233         fm->profile = fmp;
1234         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1235                 goto error;
1236         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1237                 goto error;
1238         /* Add to the flow meter list. */
1239         if (!priv->sh->meter_aso_en)
1240                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1241         /* Add to the flow meter list. */
1242         fm->active_state = 1; /* Config meter starts as active. */
1243         fm->is_enable = 1;
1244         fm->shared = !!shared;
1245         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1246         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1247                 fm->def_policy = 1;
1248                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1249                 if (!fm->flow_ipool)
1250                         goto error;
1251         }
1252         rte_spinlock_init(&fm->sl);
1253         /* If ASO meter supported, update ASO flow meter by wqe. */
1254         if (priv->sh->meter_aso_en) {
1255                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1256                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1257                 if (ret)
1258                         goto error;
1259                 if (!priv->mtr_idx_tbl) {
1260                         priv->mtr_idx_tbl =
1261                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1262                         if (!priv->mtr_idx_tbl)
1263                                 goto error;
1264                 }
1265                 data.dword = mtr_idx;
1266                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1267                         goto error;
1268         }
1269         if (mtr_policy)
1270                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1271         return 0;
1272 error:
1273         mlx5_flow_destroy_mtr_tbls(dev, fm);
1274         /* Free policer counters. */
1275         if (fm->drop_cnt)
1276                 mlx5_counter_free(dev, fm->drop_cnt);
1277         if (priv->sh->meter_aso_en)
1278                 mlx5_flow_mtr_free(dev, mtr_idx);
1279         else
1280                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1281         return -rte_mtr_error_set(error, ENOTSUP,
1282                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1283                 NULL, "Failed to create devx meter.");
1284 }
1285
1286 static int
1287 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1288                         struct mlx5_flow_meter_info *fm,
1289                         uint32_t mtr_idx)
1290 {
1291         struct mlx5_priv *priv = dev->data->dev_private;
1292         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1293         struct mlx5_flow_meter_profile *fmp;
1294         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1295         struct mlx5_flow_meter_policy *mtr_policy;
1296
1297         /* Meter object must not have any owner. */
1298         MLX5_ASSERT(!fm->ref_cnt);
1299         /* Get meter profile. */
1300         fmp = fm->profile;
1301         if (fmp == NULL)
1302                 return -1;
1303         /* Update dependencies. */
1304         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1305         fm->profile = NULL;
1306         /* Remove from list. */
1307         if (!priv->sh->meter_aso_en) {
1308                 legacy_fm = container_of(fm,
1309                         struct mlx5_legacy_flow_meter, fm);
1310                 TAILQ_REMOVE(fms, legacy_fm, next);
1311         }
1312         /* Free drop counters. */
1313         if (fm->drop_cnt)
1314                 mlx5_counter_free(dev, fm->drop_cnt);
1315         /* Free meter flow table. */
1316         if (fm->flow_ipool) {
1317                 mlx5_ipool_destroy(fm->flow_ipool);
1318                 fm->flow_ipool = 0;
1319         }
1320         mlx5_flow_destroy_mtr_tbls(dev, fm);
1321         if (fm->def_policy)
1322                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1323                                 1, __ATOMIC_RELAXED);
1324         if (priv->sh->meter_aso_en) {
1325                 if (!fm->def_policy) {
1326                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1327                                                 fm->policy_id, NULL);
1328                         if (mtr_policy)
1329                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1330                                                 1, __ATOMIC_RELAXED);
1331                         fm->policy_id = 0;
1332                 }
1333                 fm->def_policy = 0;
1334                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1335                         return -1;
1336                 mlx5_flow_mtr_free(dev, mtr_idx);
1337         } else {
1338                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1339                                         legacy_fm->idx);
1340         }
1341         return 0;
1342 }
1343
1344 /**
1345  * Destroy meter rules.
1346  *
1347  * @param[in] dev
1348  *   Pointer to Ethernet device.
1349  * @param[in] meter_id
1350  *   Meter id.
1351  * @param[out] error
1352  *   Pointer to rte meter error structure.
1353  *
1354  * @return
1355  *   0 on success, a negative errno value otherwise and rte_errno is set.
1356  */
1357 static int
1358 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1359                         struct rte_mtr_error *error)
1360 {
1361         struct mlx5_priv *priv = dev->data->dev_private;
1362         struct mlx5_flow_meter_info *fm;
1363         uint32_t mtr_idx = 0;
1364
1365         if (!priv->mtr_en)
1366                 return -rte_mtr_error_set(error, ENOTSUP,
1367                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1368                                           NULL,
1369                                           "Meter is not supported");
1370         /* Meter object must exist. */
1371         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1372         if (fm == NULL)
1373                 return -rte_mtr_error_set(error, ENOENT,
1374                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1375                                           NULL,
1376                                           "Meter object id not valid.");
1377         /* Meter object must not have any owner. */
1378         if (fm->ref_cnt > 0)
1379                 return -rte_mtr_error_set(error, EBUSY,
1380                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1381                                           NULL,
1382                                           "Meter object is being used.");
1383         /* Destroy the meter profile. */
1384         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1385                 return -rte_mtr_error_set(error, EINVAL,
1386                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1387                                         NULL,
1388                                         "MTR object meter profile invalid.");
1389         return 0;
1390 }
1391
1392 /**
1393  * Modify meter state.
1394  *
1395  * @param[in] priv
1396  *   Pointer to mlx5 private data structure.
1397  * @param[in] fm
1398  *   Pointer to flow meter.
1399  * @param[in] new_state
1400  *   New state to update.
1401  * @param[out] error
1402  *   Pointer to rte meter error structure.
1403  *
1404  * @return
1405  *   0 on success, a negative errno value otherwise and rte_errno is set.
1406  */
1407 static int
1408 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1409                              struct mlx5_flow_meter_info *fm,
1410                              uint32_t new_state,
1411                              struct rte_mtr_error *error)
1412 {
1413         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1414                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1415                 .ebs_eir = 0,
1416         };
1417         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1418                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1419         int ret;
1420
1421         if (new_state == MLX5_FLOW_METER_DISABLE)
1422                 ret = mlx5_flow_meter_action_modify(priv, fm,
1423                                 &srtcm, modify_bits, 0, 0);
1424         else
1425                 ret = mlx5_flow_meter_action_modify(priv, fm,
1426                                                     &fm->profile->srtcm_prm,
1427                                                     modify_bits, 0, 1);
1428         if (ret)
1429                 return -rte_mtr_error_set(error, -ret,
1430                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1431                                           NULL,
1432                                           new_state ?
1433                                           "Failed to enable meter." :
1434                                           "Failed to disable meter.");
1435         return 0;
1436 }
1437
1438 /**
1439  * Callback to enable flow meter.
1440  *
1441  * @param[in] dev
1442  *   Pointer to Ethernet device.
1443  * @param[in] meter_id
1444  *   Meter id.
1445  * @param[out] error
1446  *   Pointer to rte meter error structure.
1447  *
1448  * @return
1449  *   0 on success, a negative errno value otherwise and rte_errno is set.
1450  */
1451 static int
1452 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1453                        uint32_t meter_id,
1454                        struct rte_mtr_error *error)
1455 {
1456         struct mlx5_priv *priv = dev->data->dev_private;
1457         struct mlx5_flow_meter_info *fm;
1458         int ret;
1459
1460         if (!priv->mtr_en)
1461                 return -rte_mtr_error_set(error, ENOTSUP,
1462                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1463                                           "Meter is not supported");
1464         /* Meter object must exist. */
1465         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1466         if (fm == NULL)
1467                 return -rte_mtr_error_set(error, ENOENT,
1468                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1469                                           NULL, "Meter not found.");
1470         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1471                 return 0;
1472         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1473                                            error);
1474         if (!ret)
1475                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1476         return ret;
1477 }
1478
1479 /**
1480  * Callback to disable flow meter.
1481  *
1482  * @param[in] dev
1483  *   Pointer to Ethernet device.
1484  * @param[in] meter_id
1485  *   Meter id.
1486  * @param[out] error
1487  *   Pointer to rte meter error structure.
1488  *
1489  * @return
1490  *   0 on success, a negative errno value otherwise and rte_errno is set.
1491  */
1492 static int
1493 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1494                         uint32_t meter_id,
1495                         struct rte_mtr_error *error)
1496 {
1497         struct mlx5_priv *priv = dev->data->dev_private;
1498         struct mlx5_flow_meter_info *fm;
1499         int ret;
1500
1501         if (!priv->mtr_en)
1502                 return -rte_mtr_error_set(error, ENOTSUP,
1503                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1504                                           "Meter is not supported");
1505         /* Meter object must exist. */
1506         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1507         if (fm == NULL)
1508                 return -rte_mtr_error_set(error, ENOENT,
1509                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1510                                           NULL, "Meter not found.");
1511         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1512                 return 0;
1513         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1514                                            error);
1515         if (!ret)
1516                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1517         return ret;
1518 }
1519
1520 /**
1521  * Callback to update meter profile.
1522  *
1523  * @param[in] dev
1524  *   Pointer to Ethernet device.
1525  * @param[in] meter_id
1526  *   Meter id.
1527  * @param[in] meter_profile_id
1528  *   To be updated meter profile id.
1529  * @param[out] error
1530  *   Pointer to rte meter error structure.
1531  *
1532  * @return
1533  *   0 on success, a negative errno value otherwise and rte_errno is set.
1534  */
1535 static int
1536 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1537                                uint32_t meter_id,
1538                                uint32_t meter_profile_id,
1539                                struct rte_mtr_error *error)
1540 {
1541         struct mlx5_priv *priv = dev->data->dev_private;
1542         struct mlx5_flow_meter_profile *fmp;
1543         struct mlx5_flow_meter_profile *old_fmp;
1544         struct mlx5_flow_meter_info *fm;
1545         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1546                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1547         int ret;
1548
1549         if (!priv->mtr_en)
1550                 return -rte_mtr_error_set(error, ENOTSUP,
1551                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1552                                           "Meter is not supported");
1553         /* Meter profile must exist. */
1554         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1555         if (fmp == NULL)
1556                 return -rte_mtr_error_set(error, ENOENT,
1557                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1558                                           NULL, "Meter profile not found.");
1559         /* Meter object must exist. */
1560         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1561         if (fm == NULL)
1562                 return -rte_mtr_error_set(error, ENOENT,
1563                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1564                                           NULL, "Meter not found.");
1565         /* MTR object already set to meter profile id. */
1566         old_fmp = fm->profile;
1567         if (fmp == old_fmp)
1568                 return 0;
1569         /* Update the profile. */
1570         fm->profile = fmp;
1571         /* Update meter params in HW (if not disabled). */
1572         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1573                 return 0;
1574         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1575                                               modify_bits, fm->active_state, 1);
1576         if (ret) {
1577                 fm->profile = old_fmp;
1578                 return -rte_mtr_error_set(error, -ret,
1579                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1580                                           NULL, "Failed to update meter"
1581                                           " parmeters in hardware.");
1582         }
1583         old_fmp->ref_cnt--;
1584         fmp->ref_cnt++;
1585         return 0;
1586 }
1587
1588 /**
1589  * Callback to update meter stats mask.
1590  *
1591  * @param[in] dev
1592  *   Pointer to Ethernet device.
1593  * @param[in] meter_id
1594  *   Meter id.
1595  * @param[in] stats_mask
1596  *   To be updated stats_mask.
1597  * @param[out] error
1598  *   Pointer to rte meter error structure.
1599  *
1600  * @return
1601  *   0 on success, a negative errno value otherwise and rte_errno is set.
1602  */
1603 static int
1604 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1605                              uint32_t meter_id,
1606                              uint64_t stats_mask,
1607                              struct rte_mtr_error *error)
1608 {
1609         struct mlx5_priv *priv = dev->data->dev_private;
1610         struct mlx5_flow_meter_info *fm;
1611
1612         if (!priv->mtr_en)
1613                 return -rte_mtr_error_set(error, ENOTSUP,
1614                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1615                                           "Meter is not supported");
1616         /* Meter object must exist. */
1617         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1618         if (fm == NULL)
1619                 return -rte_mtr_error_set(error, ENOENT,
1620                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1621                                           NULL, "Meter object id not valid.");
1622         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1623                 return -rte_mtr_error_set(error, ENOENT,
1624                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1625                                           NULL, "Fail to allocate "
1626                                           "counter for meter.");
1627         return 0;
1628 }
1629
1630 /**
1631  * Callback to read meter statistics.
1632  *
1633  * @param[in] dev
1634  *   Pointer to Ethernet device.
1635  * @param[in] meter_id
1636  *   Meter id.
1637  * @param[out] stats
1638  *   Pointer to store the statistics.
1639  * @param[out] stats_mask
1640  *   Pointer to store the stats_mask.
1641  * @param[in] clear
1642  *   Statistic to be cleared after read or not.
1643  * @param[out] error
1644  *   Pointer to rte meter error structure.
1645  *
1646  * @return
1647  *   0 on success, a negative errno value otherwise and rte_errno is set.
1648  */
1649 static int
1650 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1651                            uint32_t meter_id,
1652                            struct rte_mtr_stats *stats,
1653                            uint64_t *stats_mask,
1654                            int clear,
1655                            struct rte_mtr_error *error)
1656 {
1657         struct mlx5_priv *priv = dev->data->dev_private;
1658         struct mlx5_flow_meter_info *fm;
1659         uint64_t pkts;
1660         uint64_t bytes;
1661         int ret = 0;
1662
1663         if (!priv->mtr_en)
1664                 return -rte_mtr_error_set(error, ENOTSUP,
1665                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1666                                           "Meter is not supported");
1667         /* Meter object must exist. */
1668         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1669         if (fm == NULL)
1670                 return -rte_mtr_error_set(error, ENOENT,
1671                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1672                                           NULL, "Meter object id not valid.");
1673         *stats_mask = 0;
1674         if (fm->bytes_dropped)
1675                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1676         if (fm->pkts_dropped)
1677                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1678         memset(stats, 0, sizeof(*stats));
1679         if (fm->drop_cnt) {
1680                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1681                                                  &bytes);
1682                 if (ret)
1683                         goto error;
1684                 /* If need to read the packets, set it. */
1685                 if (fm->pkts_dropped)
1686                         stats->n_pkts_dropped = pkts;
1687                 /* If need to read the bytes, set it. */
1688                 if (fm->bytes_dropped)
1689                         stats->n_bytes_dropped = bytes;
1690         }
1691         return 0;
1692 error:
1693         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1694                                  "Failed to read meter drop counters.");
1695 }
1696
1697 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1698         .capabilities_get = mlx5_flow_mtr_cap_get,
1699         .meter_profile_add = mlx5_flow_meter_profile_add,
1700         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1701         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1702         .meter_policy_add = mlx5_flow_meter_policy_add,
1703         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1704         .create = mlx5_flow_meter_create,
1705         .destroy = mlx5_flow_meter_destroy,
1706         .meter_enable = mlx5_flow_meter_enable,
1707         .meter_disable = mlx5_flow_meter_disable,
1708         .meter_profile_update = mlx5_flow_meter_profile_update,
1709         .meter_dscp_table_update = NULL,
1710         .stats_update = mlx5_flow_meter_stats_update,
1711         .stats_read = mlx5_flow_meter_stats_read,
1712 };
1713
1714 /**
1715  * Get meter operations.
1716  *
1717  * @param dev
1718  *   Pointer to Ethernet device structure.
1719  * @param arg
1720  *   Pointer to set the mtr operations.
1721  *
1722  * @return
1723  *   Always 0.
1724  */
1725 int
1726 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1727 {
1728         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1729         return 0;
1730 }
1731
1732 /**
1733  * Find meter by id.
1734  *
1735  * @param priv
1736  *   Pointer to mlx5_priv.
1737  * @param meter_id
1738  *   Meter id.
1739  * @param mtr_idx
1740  *   Pointer to Meter index.
1741  *
1742  * @return
1743  *   Pointer to the meter info found on success, NULL otherwise.
1744  */
1745 struct mlx5_flow_meter_info *
1746 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1747                 uint32_t *mtr_idx)
1748 {
1749         struct mlx5_legacy_flow_meter *legacy_fm;
1750         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1751         struct mlx5_aso_mtr *aso_mtr;
1752         struct mlx5_aso_mtr_pools_mng *pools_mng =
1753                                 &priv->sh->mtrmng->pools_mng;
1754         union mlx5_l3t_data data;
1755
1756         if (priv->sh->meter_aso_en) {
1757                 rte_spinlock_lock(&pools_mng->mtrsl);
1758                 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1759                         rte_spinlock_unlock(&pools_mng->mtrsl);
1760                         return NULL;
1761                 }
1762                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1763                         !data.dword) {
1764                         rte_spinlock_unlock(&pools_mng->mtrsl);
1765                         return NULL;
1766                 }
1767                 if (mtr_idx)
1768                         *mtr_idx = data.dword;
1769                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1770                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1771                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1772                 rte_spinlock_unlock(&pools_mng->mtrsl);
1773                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1774                         return NULL;
1775                 return &aso_mtr->fm;
1776         }
1777         TAILQ_FOREACH(legacy_fm, fms, next)
1778                 if (meter_id == legacy_fm->fm.meter_id) {
1779                         if (mtr_idx)
1780                                 *mtr_idx = legacy_fm->idx;
1781                         return &legacy_fm->fm;
1782                 }
1783         return NULL;
1784 }
1785
1786 /**
1787  * Find meter by index.
1788  *
1789  * @param priv
1790  *   Pointer to mlx5_priv.
1791  * @param idx
1792  *   Meter index.
1793  *
1794  * @return
1795  *   Pointer to the meter info found on success, NULL otherwise.
1796  */
1797 struct mlx5_flow_meter_info *
1798 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1799 {
1800         struct mlx5_aso_mtr *aso_mtr;
1801
1802         if (priv->sh->meter_aso_en) {
1803                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1804                 if (!aso_mtr)
1805                         return NULL;
1806                 return &aso_mtr->fm;
1807         } else {
1808                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1809         }
1810 }
1811
1812 /**
1813  * Attach meter to flow.
1814  * Unidirectional Meter creation can only be done
1815  * when flow direction is known, i.e. when calling meter_attach.
1816  *
1817  * @param [in] priv
1818  *  Pointer to mlx5 private data.
1819  * @param[in] fm
1820  *   Pointer to flow meter.
1821  * @param [in] attr
1822  *  Pointer to flow attributes.
1823  * @param [out] error
1824  *  Pointer to error structure.
1825  *
1826  * @return
1827  *   0 on success, a negative errno value otherwise and rte_errno is set.
1828  */
1829 int
1830 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1831                        struct mlx5_flow_meter_info *fm,
1832                        const struct rte_flow_attr *attr,
1833                        struct rte_flow_error *error)
1834 {
1835         int ret = 0;
1836
1837         if (priv->sh->meter_aso_en) {
1838                 struct mlx5_aso_mtr *aso_mtr;
1839
1840                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1841                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1842                         return rte_flow_error_set(error, ENOENT,
1843                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1844                                         NULL,
1845                                         "Timeout in meter configuration");
1846                 }
1847                 rte_spinlock_lock(&fm->sl);
1848                 if (fm->shared || !fm->ref_cnt) {
1849                         fm->ref_cnt++;
1850                 } else {
1851                         rte_flow_error_set(error, EINVAL,
1852                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1853                                    "Meter cannot be shared");
1854                         ret = -1;
1855                 }
1856                 rte_spinlock_unlock(&fm->sl);
1857         } else {
1858                 rte_spinlock_lock(&fm->sl);
1859                 if (fm->meter_action) {
1860                         if (fm->shared &&
1861                             attr->transfer == fm->transfer &&
1862                             attr->ingress == fm->ingress &&
1863                             attr->egress == fm->egress) {
1864                                 fm->ref_cnt++;
1865                         } else {
1866                                 rte_flow_error_set(error, EINVAL,
1867                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1868                                         fm->shared ?
1869                                         "Meter attr not match." :
1870                                         "Meter cannot be shared.");
1871                                 ret = -1;
1872                         }
1873                 } else {
1874                         fm->ingress = attr->ingress;
1875                         fm->egress = attr->egress;
1876                         fm->transfer = attr->transfer;
1877                         fm->ref_cnt = 1;
1878                         /* This also creates the meter object. */
1879                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1880                                                                          fm);
1881                         if (!fm->meter_action) {
1882                                 fm->ref_cnt = 0;
1883                                 fm->ingress = 0;
1884                                 fm->egress = 0;
1885                                 fm->transfer = 0;
1886                                 rte_flow_error_set(error, EINVAL,
1887                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1888                                         "Meter action create failed.");
1889                                 ret = -1;
1890                         }
1891                 }
1892                 rte_spinlock_unlock(&fm->sl);
1893         }
1894         return ret ? -rte_errno : 0;
1895 }
1896
1897 /**
1898  * Detach meter from flow.
1899  *
1900  * @param [in] priv
1901  *  Pointer to mlx5 private data.
1902  * @param [in] fm
1903  *  Pointer to flow meter.
1904  */
1905 void
1906 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1907                        struct mlx5_flow_meter_info *fm)
1908 {
1909 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1910         rte_spinlock_lock(&fm->sl);
1911         MLX5_ASSERT(fm->ref_cnt);
1912         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1913                 mlx5_glue->destroy_flow_action(fm->meter_action);
1914                 fm->meter_action = NULL;
1915                 fm->ingress = 0;
1916                 fm->egress = 0;
1917                 fm->transfer = 0;
1918         }
1919         rte_spinlock_unlock(&fm->sl);
1920 #else
1921         (void)priv;
1922         (void)fm;
1923 #endif
1924 }
1925
1926 /**
1927  * Flush meter with Rx queue configuration.
1928  *
1929  * @param[in] dev
1930  *   Pointer to Ethernet device.
1931  */
1932 void
1933 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1934 {
1935         struct mlx5_priv *priv = dev->data->dev_private;
1936         struct mlx5_flow_meter_sub_policy *sub_policy;
1937         struct mlx5_flow_meter_policy *mtr_policy;
1938         void *entry;
1939         uint32_t i, policy_idx;
1940
1941         if (!priv->mtr_en)
1942                 return;
1943         if (priv->policy_idx_tbl) {
1944                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1945                         policy_idx = *(uint32_t *)entry;
1946                         sub_policy = mlx5_ipool_get
1947                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1948                                 policy_idx);
1949                         if (!sub_policy || !sub_policy->main_policy)
1950                                 continue;
1951                         mtr_policy = sub_policy->main_policy;
1952                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1953                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1954                                         mtr_policy);
1955                 }
1956         }
1957 }
1958
1959 /**
1960  * Iterate a meter hierarchy and flush all meters and policies if possible.
1961  *
1962  * @param[in] dev
1963  *   Pointer to Ethernet device.
1964  * @param[in] fm
1965  *   Pointer to flow meter.
1966  * @param[in] mtr_idx
1967  *   .Meter's index
1968  * @param[out] error
1969  *   Pointer to rte meter error structure.
1970  *
1971  * @return
1972  *   0 on success, a negative errno value otherwise and rte_errno is set.
1973  */
1974 static int
1975 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
1976                                 struct mlx5_flow_meter_info *fm,
1977                                 uint32_t mtr_idx,
1978                                 struct rte_mtr_error *error)
1979 {
1980         struct mlx5_priv *priv = dev->data->dev_private;
1981         struct mlx5_flow_meter_policy *policy;
1982         uint32_t policy_id;
1983         struct mlx5_flow_meter_info *next_fm;
1984         uint32_t next_mtr_idx;
1985         struct mlx5_flow_meter_policy *next_policy = NULL;
1986
1987         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
1988         MLX5_ASSERT(policy);
1989         while (!fm->ref_cnt && policy->is_hierarchy) {
1990                 policy_id = fm->policy_id;
1991                 next_fm = mlx5_flow_meter_find(priv,
1992                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
1993                                 &next_mtr_idx);
1994                 if (next_fm) {
1995                         next_policy = mlx5_flow_meter_policy_find(dev,
1996                                                         next_fm->policy_id,
1997                                                         NULL);
1998                         MLX5_ASSERT(next_policy);
1999                 }
2000                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
2001                         return -rte_mtr_error_set(error, ENOTSUP,
2002                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
2003                                                 NULL,
2004                                                 "Failed to flush meter.");
2005                 if (policy->ref_cnt)
2006                         break;
2007                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
2008                                                 policy, error, true))
2009                         return -rte_errno;
2010                 mlx5_free(policy);
2011                 if (!next_fm || !next_policy)
2012                         break;
2013                 fm = next_fm;
2014                 mtr_idx = next_mtr_idx;
2015                 policy = next_policy;
2016         }
2017         return 0;
2018 }
2019
2020 /**
2021  * Flush all the hierarchy meters and their policies.
2022  *
2023  * @param[in] dev
2024  *   Pointer to Ethernet device.
2025  * @param[out] error
2026  *   Pointer to rte meter error structure.
2027  *
2028  * @return
2029  *   0 on success, a negative errno value otherwise and rte_errno is set.
2030  */
2031 static int
2032 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
2033                                       struct rte_mtr_error *error)
2034 {
2035         struct mlx5_priv *priv = dev->data->dev_private;
2036         struct mlx5_flow_meter_info *fm;
2037         struct mlx5_flow_meter_policy *policy;
2038         struct mlx5_flow_meter_sub_policy *sub_policy;
2039         struct mlx5_flow_meter_info *next_fm;
2040         struct mlx5_aso_mtr *aso_mtr;
2041         uint32_t mtr_idx = 0;
2042         uint32_t i, policy_idx;
2043         void *entry;
2044
2045         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2046                 return 0;
2047         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2048                 mtr_idx = *(uint32_t *)entry;
2049                 if (!mtr_idx)
2050                         continue;
2051                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2052                 fm = &aso_mtr->fm;
2053                 if (fm->ref_cnt || fm->def_policy)
2054                         continue;
2055                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2056                         return -rte_errno;
2057         }
2058         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2059                 policy_idx = *(uint32_t *)entry;
2060                 sub_policy = mlx5_ipool_get
2061                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2062                                 policy_idx);
2063                 if (!sub_policy)
2064                         return -rte_mtr_error_set(error,
2065                                         EINVAL,
2066                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2067                                         NULL, "Meter policy invalid.");
2068                 policy = sub_policy->main_policy;
2069                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2070                         continue;
2071                 next_fm = mlx5_flow_meter_find(priv,
2072                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2073                                 &mtr_idx);
2074                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2075                                                     error, true))
2076                         return -rte_mtr_error_set(error,
2077                                         EINVAL,
2078                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2079                                         NULL, "Meter policy invalid.");
2080                 mlx5_free(policy);
2081                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2082                         continue;
2083                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2084                                                     mtr_idx, error))
2085                         return -rte_errno;
2086         }
2087         return 0;
2088 }
2089 /**
2090  * Flush meter configuration.
2091  *
2092  * @param[in] dev
2093  *   Pointer to Ethernet device.
2094  * @param[out] error
2095  *   Pointer to rte meter error structure.
2096  *
2097  * @return
2098  *   0 on success, a negative errno value otherwise and rte_errno is set.
2099  */
2100 int
2101 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2102 {
2103         struct mlx5_priv *priv = dev->data->dev_private;
2104         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2105         struct mlx5_flow_meter_profile *fmp;
2106         struct mlx5_legacy_flow_meter *legacy_fm;
2107         struct mlx5_flow_meter_info *fm;
2108         struct mlx5_flow_meter_sub_policy *sub_policy;
2109         void *tmp;
2110         uint32_t i, mtr_idx, policy_idx;
2111         void *entry;
2112         struct mlx5_aso_mtr *aso_mtr;
2113
2114         if (!priv->mtr_en)
2115                 return 0;
2116         if (priv->sh->meter_aso_en) {
2117                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2118                         return -rte_errno;
2119                 if (priv->mtr_idx_tbl) {
2120                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2121                                 mtr_idx = *(uint32_t *)entry;
2122                                 if (mtr_idx) {
2123                                         aso_mtr =
2124                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2125                                         fm = &aso_mtr->fm;
2126                                         (void)mlx5_flow_meter_params_flush(dev,
2127                                                 fm, mtr_idx);
2128                                 }
2129                         }
2130                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2131                         priv->mtr_idx_tbl = NULL;
2132                 }
2133         } else {
2134                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2135                         fm = &legacy_fm->fm;
2136                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2137                                 return -rte_mtr_error_set(error, EINVAL,
2138                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2139                                 NULL, "MTR object meter profile invalid.");
2140                 }
2141         }
2142         if (priv->policy_idx_tbl) {
2143                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2144                         policy_idx = *(uint32_t *)entry;
2145                         sub_policy = mlx5_ipool_get
2146                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2147                                 policy_idx);
2148                         if (!sub_policy)
2149                                 return -rte_mtr_error_set(error,
2150                                                 EINVAL,
2151                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2152                                                 NULL, "MTR object "
2153                                                 "meter policy invalid.");
2154                         if (__mlx5_flow_meter_policy_delete(dev, i,
2155                                                 sub_policy->main_policy,
2156                                                 error, true))
2157                                 return -rte_mtr_error_set(error,
2158                                                 EINVAL,
2159                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2160                                                 NULL, "MTR object "
2161                                                 "meter policy invalid.");
2162                         mlx5_free(sub_policy->main_policy);
2163                 }
2164                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2165                 priv->policy_idx_tbl = NULL;
2166         }
2167         if (priv->mtr_profile_tbl) {
2168                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2169                         fmp = entry;
2170                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2171                                                            error))
2172                                 return -rte_mtr_error_set(error, EINVAL,
2173                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2174                                                 NULL, "Fail to destroy "
2175                                                 "meter profile.");
2176                 }
2177                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2178                 priv->mtr_profile_tbl = NULL;
2179         }
2180         /* Delete default policy table. */
2181         mlx5_flow_destroy_def_policy(dev);
2182         if (priv->sh->refcnt == 1)
2183                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2184         return 0;
2185 }