drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133
 134         /* Profile must not be NULL. */
 135         if (profile == NULL)
 136                 return -rte_mtr_error_set(error, EINVAL,
 137                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 138                                           NULL, "Meter profile is null.");
 139         /* Meter profile ID must be valid. */
 140         if (meter_profile_id == UINT32_MAX)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 143                                           NULL, "Meter profile id not valid.");
 144         /* Meter profile must not exist. */
 145         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 146         if (fmp)
 147                 return -rte_mtr_error_set(error, EEXIST,
 148                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 149                                           NULL,
 150                                           "Meter profile already exists.");
 151         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 152                 if (priv->config.hca_attr.qos.flow_meter_old) {
 153                         /* Verify support for flow meter parameters. */
 154                         if (priv->sh->meter_aso_en && profile->packet_mode) {
 155                                 if (profile->srtcm_rfc2697.cir > 0 &&
 156                                         (profile->srtcm_rfc2697.cir <<
 157                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 158                                         <= MLX5_SRTCM_CIR_MAX &&
 159                                         profile->srtcm_rfc2697.cbs > 0 &&
 160                                         (profile->srtcm_rfc2697.cbs <<
 161                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 162                                         <= MLX5_SRTCM_CBS_MAX &&
 163                                         (profile->srtcm_rfc2697.ebs <<
 164                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 165                                         <= MLX5_SRTCM_EBS_MAX)
 166                                         return 0;
 167                                 return -rte_mtr_error_set
 168                                              (error, ENOTSUP,
 169                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 170                                               NULL,
 171                                               profile->srtcm_rfc2697.ebs ?
 172                                               "Metering value ebs must be 0." :
 173                                               "Invalid metering parameters.");
 174                         }
 175                         if (profile->srtcm_rfc2697.cir > 0 &&
 176                                 profile->srtcm_rfc2697.cir <=
 177                                                 MLX5_SRTCM_CIR_MAX &&
 178                                 profile->srtcm_rfc2697.cbs > 0 &&
 179                                 profile->srtcm_rfc2697.cbs <=
 180                                                 MLX5_SRTCM_CBS_MAX &&
 181                                 profile->srtcm_rfc2697.ebs <=
 182                                                 MLX5_SRTCM_EBS_MAX)
 183                                 return 0;
 184                         return -rte_mtr_error_set(error, ENOTSUP,
 185                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 186                                         NULL,
 187                                         profile->srtcm_rfc2697.ebs ?
 188                                         "Metering value ebs must be 0." :
 189                                         "Invalid metering parameters.");
 190                 }
 191         }
 192         return -rte_mtr_error_set(error, ENOTSUP,
 193                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 194                                   NULL, "Metering algorithm not supported.");
 195 }
 196
 197 /**
 198  * Calculate mantissa and exponent for cir.
 199  *
 200  * @param[in] cir
 201  *   Value to be calculated.
 202  * @param[out] man
 203  *   Pointer to the mantissa.
 204  * @param[out] exp
 205  *   Pointer to the exp.
 206  */
 207 static void
 208 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 209 {
 210         int64_t _cir;
 211         int64_t delta = INT64_MAX;
 212         uint8_t _man = 0;
 213         uint8_t _exp = 0;
 214         uint64_t m, e;
 215
 216         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 217                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 218                         _cir = (1000000000ULL * m) >> e;
 219                         if (llabs(cir - _cir) <= delta) {
 220                                 delta = llabs(cir - _cir);
 221                                 _man = m;
 222                                 _exp = e;
 223                         }
 224                 }
 225         }
 226         *man = _man;
 227         *exp = _exp;
 228 }
 229
 230 /**
 231  * Calculate mantissa and exponent for xbs.
 232  *
 233  * @param[in] xbs
 234  *   Value to be calculated.
 235  * @param[out] man
 236  *   Pointer to the mantissa.
 237  * @param[out] exp
 238  *   Pointer to the exp.
 239  */
 240 static void
 241 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 242 {
 243         int _exp;
 244         double _man;
 245
 246         /* Special case xbs == 0 ? both exp and matissa are 0. */
 247         if (xbs == 0) {
 248                 *man = 0;
 249                 *exp = 0;
 250                 return;
 251         }
 252         /* xbs = xbs_mantissa * 2^xbs_exponent */
 253         _man = frexp(xbs, &_exp);
 254         _man = _man * pow(2, MLX5_MAN_WIDTH);
 255         _exp = _exp - MLX5_MAN_WIDTH;
 256         *man = (uint8_t)ceil(_man);
 257         *exp = _exp;
 258 }
 259
 260 /**
 261  * Fill the prm meter parameter.
 262  *
 263  * @param[in,out] fmp
 264  *   Pointer to meter profie to be converted.
 265  * @param[out] error
 266  *   Pointer to the error structure.
 267  *
 268  * @return
 269  *   0 on success, a negative errno value otherwise and rte_errno is set.
 270  */
 271 static int
 272 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 273                         struct mlx5_priv *priv, struct rte_mtr_error *error)
 274 {
 275         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 276         uint8_t man, exp;
 277         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 278         uint32_t ebs_exp, ebs_man;
 279         uint64_t cir, cbs, ebs;
 280
 281         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 282                 return -rte_mtr_error_set(error, ENOTSUP,
 283                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 284                                 NULL, "Metering algorithm not supported.");
 285         if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
 286                 return -rte_mtr_error_set(error, ENOTSUP,
 287                         RTE_MTR_ERROR_TYPE_METER_PROFILE,
 288                         NULL, "Metering algorithm packet mode not supported.");
 289         if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
 290                 cir = fmp->profile.srtcm_rfc2697.cir <<
 291                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 292                 cbs = fmp->profile.srtcm_rfc2697.cbs <<
 293                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 294                 ebs = fmp->profile.srtcm_rfc2697.ebs <<
 295                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 296         } else {
 297                 cir = fmp->profile.srtcm_rfc2697.cir;
 298                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 299                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 300         }
 301         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 302         mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
 303         /* Check if cir mantissa is too large. */
 304         if (exp > ASO_DSEG_CIR_EXP_MASK)
 305                 return -rte_mtr_error_set(error, ENOTSUP,
 306                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 307                                           "meter profile parameter cir is"
 308                                           " not supported.");
 309         cir_man = man;
 310         cir_exp = exp;
 311          /* cbs = cbs_mantissa * 2^cbs_exponent */
 312         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 313         /* Check if cbs mantissa is too large. */
 314         if (exp > ASO_DSEG_EXP_MASK)
 315                 return -rte_mtr_error_set(error, ENOTSUP,
 316                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 317                                           "meter profile parameter cbs is"
 318                                           " not supported.");
 319         cbs_man = man;
 320         cbs_exp = exp;
 321         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 322                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 323                                           cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 324                                           cir_man);
 325         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 326         /* Check if ebs mantissa is too large. */
 327         if (exp > ASO_DSEG_EXP_MASK)
 328                 return -rte_mtr_error_set(error, ENOTSUP,
 329                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 330                                           "meter profile parameter ebs is"
 331                                           " not supported.");
 332         ebs_man = man;
 333         ebs_exp = exp;
 334         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 335                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 336         return 0;
 337 }
 338
 339 /**
 340  * Callback to get MTR capabilities.
 341  *
 342  * @param[in] dev
 343  *   Pointer to Ethernet device.
 344  * @param[out] cap
 345  *   Pointer to save MTR capabilities.
 346  * @param[out] error
 347  *   Pointer to the error structure.
 348  *
 349  * @return
 350  *   0 on success, a negative errno value otherwise and rte_errno is set.
 351  */
 352 static int
 353 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 354                  struct rte_mtr_capabilities *cap,
 355                  struct rte_mtr_error *error __rte_unused)
 356 {
 357         struct mlx5_priv *priv = dev->data->dev_private;
 358         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 359
 360         if (!priv->mtr_en)
 361                 return -rte_mtr_error_set(error, ENOTSUP,
 362                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 363                                           "Meter is not supported");
 364         memset(cap, 0, sizeof(*cap));
 365         if (priv->sh->meter_aso_en) {
 366                 /* 2 meters per one ASO cache line. */
 367                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 368                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 369         } else {
 370                 cap->n_max = 1 << qattr->log_max_flow_meter;
 371                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 372         }
 373         cap->srtcm_rfc2697_byte_mode_supported = 1;
 374         cap->n_shared_max = cap->n_max;
 375         cap->identical = 1;
 376         cap->shared_identical = 1;
 377         cap->shared_n_flows_per_mtr_max = 4 << 20;
 378         /* 2M flows can share the same meter. */
 379         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 380         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 381         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 382         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 383                           RTE_MTR_STATS_N_PKTS_DROPPED;
 384         return 0;
 385 }
 386
 387 /**
 388  * Callback to add MTR profile.
 389  *
 390  * @param[in] dev
 391  *   Pointer to Ethernet device.
 392  * @param[in] meter_profile_id
 393  *   Meter profile id.
 394  * @param[in] profile
 395  *   Pointer to meter profile detail.
 396  * @param[out] error
 397  *   Pointer to the error structure.
 398  *
 399  * @return
 400  *   0 on success, a negative errno value otherwise and rte_errno is set.
 401  */
 402 static int
 403 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 404                        uint32_t meter_profile_id,
 405                        struct rte_mtr_meter_profile *profile,
 406                        struct rte_mtr_error *error)
 407 {
 408         struct mlx5_priv *priv = dev->data->dev_private;
 409         struct mlx5_flow_meter_profile *fmp;
 410         union mlx5_l3t_data data;
 411         int ret;
 412
 413         if (!priv->mtr_en)
 414                 return -rte_mtr_error_set(error, ENOTSUP,
 415                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 416                                           "Meter is not supported");
 417         /* Check input params. */
 418         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 419                                                profile, error);
 420         if (ret)
 421                 return ret;
 422         /* Meter profile memory allocation. */
 423         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 424                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 425         if (fmp == NULL)
 426                 return -rte_mtr_error_set(error, ENOMEM,
 427                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 428                                           NULL, "Meter profile memory "
 429                                           "alloc failed.");
 430         /* Fill profile info. */
 431         fmp->id = meter_profile_id;
 432         fmp->profile = *profile;
 433         /* Fill the flow meter parameters for the PRM. */
 434         ret = mlx5_flow_meter_param_fill(fmp, priv, error);
 435         if (ret)
 436                 goto error;
 437         data.ptr = fmp;
 438         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 439                                  meter_profile_id, &data);
 440         if (ret)
 441                 return -rte_mtr_error_set(error, ENOTSUP,
 442                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 443                                           NULL, "Meter profile insert fail.");
 444         return 0;
 445 error:
 446         mlx5_free(fmp);
 447         return ret;
 448 }
 449
 450 /**
 451  * Callback to delete MTR profile.
 452  *
 453  * @param[in] dev
 454  *   Pointer to Ethernet device.
 455  * @param[in] meter_profile_id
 456  *   Meter profile id.
 457  * @param[out] error
 458  *   Pointer to the error structure.
 459  *
 460  * @return
 461  *   0 on success, a negative errno value otherwise and rte_errno is set.
 462  */
 463 static int
 464 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 465                           uint32_t meter_profile_id,
 466                           struct rte_mtr_error *error)
 467 {
 468         struct mlx5_priv *priv = dev->data->dev_private;
 469         struct mlx5_flow_meter_profile *fmp;
 470
 471         if (!priv->mtr_en)
 472                 return -rte_mtr_error_set(error, ENOTSUP,
 473                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 474                                           "Meter is not supported");
 475         /* Meter profile must exist. */
 476         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 477         if (fmp == NULL)
 478                 return -rte_mtr_error_set(error, ENOENT,
 479                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 480                                           &meter_profile_id,
 481                                           "Meter profile id is invalid.");
 482         /* Check profile is unused. */
 483         if (fmp->ref_cnt)
 484                 return -rte_mtr_error_set(error, EBUSY,
 485                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 486                                           NULL, "Meter profile is in use.");
 487         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 488                 return -rte_mtr_error_set(error, EBUSY,
 489                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 490                                           NULL, "Meter profile remove fail.");
 491         mlx5_free(fmp);
 492         return 0;
 493 }
 494
 495 /**
 496  * Find policy by id.
 497  *
 498  * @param[in] dev
 499  *   Pointer to Ethernet device.
 500  * @param policy_id
 501  *   Policy id.
 502  *
 503  * @return
 504  *   Pointer to the policy found on success, NULL otherwise.
 505  */
 506 struct mlx5_flow_meter_policy *
 507 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 508                             uint32_t policy_id,
 509                             uint32_t *policy_idx)
 510 {
 511         struct mlx5_priv *priv = dev->data->dev_private;
 512         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 513         union mlx5_l3t_data data;
 514
 515         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 516                 return NULL;
 517         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 518                                 !data.dword)
 519                 return NULL;
 520         if (policy_idx)
 521                 *policy_idx = data.dword;
 522         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 523                                         data.dword);
 524         /* Remove reference taken by the mlx5_l3t_get_entry. */
 525         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 526         if (sub_policy)
 527                 if (sub_policy->main_policy_id)
 528                         return sub_policy->main_policy;
 529         return NULL;
 530 }
 531
 532 /**
 533  * Get the last meter's policy from one meter's policy in hierarchy.
 534  *
 535  * @param[in] dev
 536  *   Pointer to Ethernet device.
 537  * @param[in] policy
 538  *   Pointer to flow meter policy.
 539  *
 540  * @return
 541  *   Pointer to the final meter's policy, or NULL when fail.
 542  */
 543 struct mlx5_flow_meter_policy *
 544 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 545                                         struct mlx5_flow_meter_policy *policy)
 546 {
 547         struct mlx5_priv *priv = dev->data->dev_private;
 548         struct mlx5_flow_meter_info *next_fm;
 549         struct mlx5_flow_meter_policy *next_policy = policy;
 550
 551         while (next_policy->is_hierarchy) {
 552                 next_fm = mlx5_flow_meter_find(priv,
 553                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 554                 if (!next_fm || next_fm->def_policy)
 555                         return NULL;
 556                 next_policy = mlx5_flow_meter_policy_find(dev,
 557                                                 next_fm->policy_id, NULL);
 558                 MLX5_ASSERT(next_policy);
 559         }
 560         return next_policy;
 561 }
 562
 563 /**
 564  * Callback to check MTR policy action validate
 565  *
 566  * @param[in] dev
 567  *   Pointer to Ethernet device.
 568  * @param[in] actions
 569  *   Pointer to meter policy action detail.
 570  * @param[out] error
 571  *   Pointer to the error structure.
 572  *
 573  * @return
 574  *   0 on success, a negative errno value otherwise and rte_errno is set.
 575  */
 576 static int
 577 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 578         struct rte_mtr_meter_policy_params *policy,
 579         struct rte_mtr_error *error)
 580 {
 581         struct mlx5_priv *priv = dev->data->dev_private;
 582         struct rte_flow_attr attr = { .transfer =
 583                         priv->config.dv_esw_en ? 1 : 0};
 584         bool is_rss = false;
 585         bool is_def_policy = false;
 586         uint8_t domain_bitmap;
 587         int ret;
 588
 589         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 590                 return -rte_mtr_error_set(error, ENOTSUP,
 591                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 592                                 NULL, "meter policy unsupported.");
 593         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 594                         &is_rss, &domain_bitmap, &is_def_policy, error);
 595         if (ret)
 596                 return ret;
 597         return 0;
 598 }
 599
 600 static int
 601 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 602                         uint32_t policy_id,
 603                         struct mlx5_flow_meter_policy *mtr_policy,
 604                         struct rte_mtr_error *error,
 605                         bool clear_l3t)
 606 {
 607         struct mlx5_priv *priv = dev->data->dev_private;
 608         struct mlx5_flow_meter_sub_policy *sub_policy;
 609         uint32_t i, j;
 610         uint16_t sub_policy_num;
 611
 612         rte_spinlock_lock(&mtr_policy->sl);
 613         if (mtr_policy->ref_cnt) {
 614                 rte_spinlock_unlock(&mtr_policy->sl);
 615                 return -rte_mtr_error_set(error, EBUSY,
 616                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 617                                  NULL,
 618                                 "Meter policy object is being used.");
 619         }
 620         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 621         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 622         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 623                 sub_policy_num = (mtr_policy->sub_policy_num >>
 624                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 625                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 626                 if (sub_policy_num) {
 627                         for (j = 0; j < sub_policy_num; j++) {
 628                                 sub_policy = mtr_policy->sub_policys[i][j];
 629                                 if (sub_policy)
 630                                         mlx5_ipool_free
 631                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 632                                         sub_policy->idx);
 633                         }
 634                 }
 635         }
 636         if (priv->policy_idx_tbl && clear_l3t) {
 637                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 638                         rte_spinlock_unlock(&mtr_policy->sl);
 639                         return -rte_mtr_error_set(error, ENOTSUP,
 640                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 641                                 "Fail to delete policy in index table.");
 642                 }
 643         }
 644         rte_spinlock_unlock(&mtr_policy->sl);
 645         return 0;
 646 }
 647
 648 /**
 649  * Callback to add MTR policy.
 650  *
 651  * @param[in] dev
 652  *   Pointer to Ethernet device.
 653  * @param[out] policy_id
 654  *   Pointer to policy id
 655  * @param[in] actions
 656  *   Pointer to meter policy action detail.
 657  * @param[out] error
 658  *   Pointer to the error structure.
 659  *
 660  * @return
 661  *   0 on success, a negative errno value otherwise and rte_errno is set.
 662  */
 663 static int
 664 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 665                         uint32_t policy_id,
 666                         struct rte_mtr_meter_policy_params *policy,
 667                         struct rte_mtr_error *error)
 668 {
 669         struct mlx5_priv *priv = dev->data->dev_private;
 670         struct rte_flow_attr attr = { .transfer =
 671                         priv->config.dv_esw_en ? 1 : 0};
 672         uint32_t sub_policy_idx = 0;
 673         uint32_t policy_idx = 0;
 674         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 675         struct mlx5_flow_meter_sub_policy *sub_policy;
 676         bool is_rss = false;
 677         bool is_def_policy = false;
 678         uint32_t i;
 679         int ret;
 680         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 681         uint16_t sub_policy_num;
 682         uint8_t domain_bitmap = 0;
 683         union mlx5_l3t_data data;
 684         bool skip_rule = false;
 685
 686         if (!priv->mtr_en)
 687                 return -rte_mtr_error_set(error, ENOTSUP,
 688                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 689                                           NULL, "meter policy unsupported.");
 690         if (policy_id == MLX5_INVALID_POLICY_ID)
 691                 return -rte_mtr_error_set(error, ENOTSUP,
 692                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 693                         "policy ID is invalid. ");
 694         if (policy_id == priv->sh->mtrmng->def_policy_id)
 695                 return -rte_mtr_error_set(error, EEXIST,
 696                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 697                         "policy ID exists. ");
 698         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id,
 699                                 &policy_idx);
 700         if (mtr_policy)
 701                 return -rte_mtr_error_set(error, EEXIST,
 702                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 703                         "policy ID exists. ");
 704         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 705                         &is_rss, &domain_bitmap, &is_def_policy, error);
 706         if (ret)
 707                 return ret;
 708         if (!domain_bitmap)
 709                 return -rte_mtr_error_set(error, ENOTSUP,
 710                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 711                                 NULL, "fail to find policy domain.");
 712         if (is_def_policy) {
 713                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 714                         return -rte_mtr_error_set(error, EEXIST,
 715                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 716                                 NULL, "a policy with similar actions "
 717                                 "is already configured");
 718                 if (mlx5_flow_create_def_policy(dev))
 719                         return -rte_mtr_error_set(error, ENOTSUP,
 720                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 721                                 NULL,
 722                                 "fail to create non-terminated policy.");
 723                 priv->sh->mtrmng->def_policy_id = policy_id;
 724                 return 0;
 725         }
 726         if (!priv->sh->meter_aso_en)
 727                 return -rte_mtr_error_set(error, ENOTSUP,
 728                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 729                         "no ASO capability to support the policy ");
 730         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 731                 if (!(domain_bitmap & (1 << i)))
 732                         continue;
 733                 if (is_rss) {
 734                         policy_size +=
 735                         sizeof(struct mlx5_flow_meter_sub_policy *) *
 736                         MLX5_MTR_RSS_MAX_SUB_POLICY;
 737                         break;
 738                 }
 739                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 740         }
 741         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 742                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 743         if (!mtr_policy)
 744                 return -rte_mtr_error_set(error, ENOMEM,
 745                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 746                                 "Memory alloc failed for meter policy.");
 747         policy_size = sizeof(struct mlx5_flow_meter_policy);
 748         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 749                 if (!(domain_bitmap & (1 << i)))
 750                         continue;
 751                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 752                         mtr_policy->ingress = 1;
 753                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 754                         mtr_policy->egress = 1;
 755                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 756                         mtr_policy->transfer = 1;
 757                 sub_policy = mlx5_ipool_zmalloc
 758                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 759                                 &sub_policy_idx);
 760                 if (!sub_policy)
 761                         goto policy_add_err;
 762                 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 763                         goto policy_add_err;
 764                 sub_policy->idx = sub_policy_idx;
 765                 sub_policy->main_policy = mtr_policy;
 766                 if (!policy_idx) {
 767                         policy_idx = sub_policy_idx;
 768                         sub_policy->main_policy_id = 1;
 769                 }
 770                 mtr_policy->sub_policys[i] =
 771                 (struct mlx5_flow_meter_sub_policy **)
 772                         ((uint8_t *)mtr_policy + policy_size);
 773                 mtr_policy->sub_policys[i][0] = sub_policy;
 774                 sub_policy_num = (mtr_policy->sub_policy_num >>
 775                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 776                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 777                 sub_policy_num++;
 778                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 779                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 780                 mtr_policy->sub_policy_num |=
 781                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 782                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 783                 if (is_rss) {
 784                         mtr_policy->is_rss = 1;
 785                         break;
 786                 }
 787                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy  *);
 788         }
 789         rte_spinlock_init(&mtr_policy->sl);
 790         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 791                                         policy->actions, error);
 792         if (ret)
 793                 goto policy_add_err;
 794         if (mtr_policy->is_hierarchy) {
 795                 struct mlx5_flow_meter_policy *final_policy;
 796
 797                 final_policy =
 798                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 799                 if (!final_policy)
 800                         goto policy_add_err;
 801                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 802         }
 803         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 804                 /* Create policy rules in HW. */
 805                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 806                 if (ret)
 807                         goto policy_add_err;
 808         }
 809         data.dword = policy_idx;
 810         if (!priv->policy_idx_tbl) {
 811                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 812                 if (!priv->policy_idx_tbl)
 813                         goto policy_add_err;
 814         }
 815         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 816                 goto policy_add_err;
 817         return 0;
 818 policy_add_err:
 819         if (mtr_policy) {
 820                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 821                         mtr_policy, error, false);
 822                 mlx5_free(mtr_policy);
 823                 if (ret)
 824                         return ret;
 825         }
 826         return -rte_mtr_error_set(error, ENOTSUP,
 827                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 828                                   NULL, "Failed to create devx policy.");
 829 }
 830
 831 /**
 832  * Callback to delete MTR policy.
 833  *
 834  * @param[in] dev
 835  *   Pointer to Ethernet device.
 836  * @param[in] policy_id
 837  *   Meter policy id.
 838  * @param[out] error
 839  *   Pointer to the error structure.
 840  *
 841  * @return
 842  *   0 on success, a negative errno value otherwise and rte_errno is set.
 843  */
 844 static int
 845 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 846                           uint32_t policy_id,
 847                           struct rte_mtr_error *error)
 848 {
 849         struct mlx5_priv *priv = dev->data->dev_private;
 850         struct mlx5_flow_meter_policy *mtr_policy;
 851         uint32_t policy_idx;
 852         int ret;
 853
 854         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 855                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 856                         return -rte_mtr_error_set(error, ENOTSUP,
 857                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 858                                 "Meter policy object is being used.");
 859                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 860                 return 0;
 861         }
 862         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 863         if (!mtr_policy)
 864                 return -rte_mtr_error_set(error, ENOTSUP,
 865                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 866                         "Meter policy id is invalid. ");
 867         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 868                                                 error, true);
 869         if (ret)
 870                 return ret;
 871         mlx5_free(mtr_policy);
 872         return 0;
 873 }
 874
 875 /**
 876  * Check meter validation.
 877  *
 878  * @param[in] priv
 879  *   Pointer to mlx5 private data structure.
 880  * @param[in] meter_id
 881  *   Meter id.
 882  * @param[in] params
 883  *   Pointer to rte meter parameters.
 884  * @param[out] error
 885  *   Pointer to rte meter error structure.
 886  *
 887  * @return
 888  *   0 on success, a negative errno value otherwise and rte_errno is set.
 889  */
 890 static int
 891 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 892                          struct rte_mtr_params *params,
 893                          struct rte_mtr_error *error)
 894 {
 895         /* Meter must use global drop action. */
 896         if (!priv->sh->dr_drop_action)
 897                 return -rte_mtr_error_set(error, ENOTSUP,
 898                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 899                                           NULL,
 900                                           "No drop action ready for meter.");
 901         /* Meter params must not be NULL. */
 902         if (params == NULL)
 903                 return -rte_mtr_error_set(error, EINVAL,
 904                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 905                                           NULL, "Meter object params null.");
 906         /* Previous meter color is not supported. */
 907         if (params->use_prev_mtr_color)
 908                 return -rte_mtr_error_set(error, ENOTSUP,
 909                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 910                                           NULL,
 911                                           "Previous meter color "
 912                                           "not supported.");
 913         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 914                 return -rte_mtr_error_set(error, ENOENT,
 915                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 916                                 NULL, "Meter policy id not valid.");
 917         /* Validate meter id. */
 918         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 919                 return -rte_mtr_error_set(error, EEXIST,
 920                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 921                         "Meter object already exists.");
 922         return 0;
 923 }
 924
 925 /**
 926  * Modify the flow meter action.
 927  *
 928  * @param[in] priv
 929  *   Pointer to mlx5 private data structure.
 930  * @param[in] fm
 931  *   Pointer to flow meter to be modified.
 932  * @param[in] srtcm
 933  *   Pointer to meter srtcm description parameter.
 934  * @param[in] modify_bits
 935  *   The bit in srtcm to be updated.
 936  * @param[in] active_state
 937  *   The state to be updated.
 938  * @return
 939  *   0 on success, o negative value otherwise.
 940  */
 941 static int
 942 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 943                 struct mlx5_flow_meter_info *fm,
 944                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 945                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 946 {
 947 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 948         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 949         uint32_t *attr;
 950         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 951         int ret;
 952         struct mlx5_aso_mtr *aso_mtr = NULL;
 953         uint32_t cbs_cir, ebs_eir, val;
 954
 955         if (priv->sh->meter_aso_en) {
 956                 fm->is_enable = !!is_enable;
 957                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 958                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 959                 if (ret)
 960                         return ret;
 961                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 962                 if (ret)
 963                         return ret;
 964         } else {
 965                 /* Fill command parameters. */
 966                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 967                 mod_attr.flow_meter_parameter = in;
 968                 mod_attr.flow_meter_parameter_sz =
 969                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 970                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 971                         mod_attr.active = !!active_state;
 972                 else
 973                         mod_attr.active = 0;
 974                 attr = in;
 975                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 976                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 977                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 978                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 979                                 ASO_DSEG_EXP_MASK;
 980                         MLX5_SET(flow_meter_parameters, attr,
 981                                 cbs_exponent, val);
 982                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 983                                 ASO_DSEG_MAN_MASK;
 984                         MLX5_SET(flow_meter_parameters, attr,
 985                                 cbs_mantissa, val);
 986                 }
 987                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
 988                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
 989                                 ASO_DSEG_EXP_MASK;
 990                         MLX5_SET(flow_meter_parameters, attr,
 991                                 cir_exponent, val);
 992                         val = cbs_cir & ASO_DSEG_MAN_MASK;
 993                         MLX5_SET(flow_meter_parameters, attr,
 994                                 cir_mantissa, val);
 995                 }
 996                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
 997                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
 998                                 ASO_DSEG_EXP_MASK;
 999                         MLX5_SET(flow_meter_parameters, attr,
1000                                 ebs_exponent, val);
1001                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1002                                 ASO_DSEG_MAN_MASK;
1003                         MLX5_SET(flow_meter_parameters, attr,
1004                                 ebs_mantissa, val);
1005                 }
1006                 /* Apply modifications to meter only if it was created. */
1007                 if (fm->meter_action) {
1008                         ret = mlx5_glue->dv_modify_flow_action_meter
1009                                         (fm->meter_action, &mod_attr,
1010                                         rte_cpu_to_be_64(modify_bits));
1011                         if (ret)
1012                                 return ret;
1013                 }
1014                 /* Update succeedded modify meter parameters. */
1015                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1016                         fm->active_state = !!active_state;
1017         }
1018         return 0;
1019 #else
1020         (void)priv;
1021         (void)fm;
1022         (void)srtcm;
1023         (void)modify_bits;
1024         (void)active_state;
1025         (void)is_enable;
1026         return -ENOTSUP;
1027 #endif
1028 }
1029
1030 static int
1031 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1032                                 struct mlx5_flow_meter_info *fm,
1033                                 uint64_t stats_mask)
1034 {
1035         fm->bytes_dropped =
1036                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1037         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1038         if (fm->bytes_dropped || fm->pkts_dropped) {
1039                 if (!fm->drop_cnt) {
1040                         /* Alloc policer counters. */
1041                         fm->drop_cnt = mlx5_counter_alloc(dev);
1042                         if (!fm->drop_cnt)
1043                                 return -1;
1044                 }
1045         } else {
1046                 if (fm->drop_cnt) {
1047                         mlx5_counter_free(dev, fm->drop_cnt);
1048                         fm->drop_cnt = 0;
1049                 }
1050         }
1051         return 0;
1052 }
1053
1054 /**
1055  * Create meter rules.
1056  *
1057  * @param[in] dev
1058  *   Pointer to Ethernet device.
1059  * @param[in] meter_id
1060  *   Meter id.
1061  * @param[in] params
1062  *   Pointer to rte meter parameters.
1063  * @param[in] shared
1064  *   Meter shared with other flow or not.
1065  * @param[out] error
1066  *   Pointer to rte meter error structure.
1067  *
1068  * @return
1069  *   0 on success, a negative errno value otherwise and rte_errno is set.
1070  */
1071 static int
1072 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1073                        struct rte_mtr_params *params, int shared,
1074                        struct rte_mtr_error *error)
1075 {
1076         struct mlx5_priv *priv = dev->data->dev_private;
1077         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1078         struct mlx5_flow_meter_profile *fmp;
1079         struct mlx5_flow_meter_info *fm;
1080         struct mlx5_legacy_flow_meter *legacy_fm;
1081         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1082         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1083                 .size = 0,
1084                 .trunk_size = 64,
1085                 .need_lock = 1,
1086                 .type = "mlx5_flow_mtr_flow_id_pool",
1087         };
1088         struct mlx5_aso_mtr *aso_mtr;
1089         uint32_t mtr_idx, policy_idx;
1090         union mlx5_l3t_data data;
1091         int ret;
1092         uint8_t domain_bitmap;
1093         uint8_t mtr_id_bits;
1094         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1095                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1096
1097         if (!priv->mtr_en)
1098                 return -rte_mtr_error_set(error, ENOTSUP,
1099                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1100                                         "Meter is not supported");
1101         /* Validate the parameters. */
1102         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1103         if (ret)
1104                 return ret;
1105         /* Meter profile must exist. */
1106         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1107         if (fmp == NULL)
1108                 return -rte_mtr_error_set(error, ENOENT,
1109                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1110                         NULL, "Meter profile id not valid.");
1111         /* Meter policy must exist. */
1112         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1113                 __atomic_add_fetch
1114                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1115                         1, __ATOMIC_RELAXED);
1116                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1117                 if (!priv->config.dv_esw_en)
1118                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1119         } else {
1120                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1121                                 params->meter_policy_id, &policy_idx);
1122                 if (!priv->sh->meter_aso_en)
1123                         return -rte_mtr_error_set(error, ENOTSUP,
1124                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1125                                 "Part of the policies cannot be "
1126                                 "supported without ASO ");
1127                 if (!mtr_policy)
1128                         return -rte_mtr_error_set(error, ENOENT,
1129                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1130                                 NULL, "Meter policy id not valid.");
1131                 domain_bitmap = (mtr_policy->ingress ?
1132                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1133                                 (mtr_policy->egress ?
1134                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1135                                 (mtr_policy->transfer ?
1136                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1137         }
1138         /* Allocate the flow meter memory. */
1139         if (priv->sh->meter_aso_en) {
1140                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1141                 if (!mtr_idx)
1142                         return -rte_mtr_error_set(error, ENOMEM,
1143                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1144                                 "Memory alloc failed for meter.");
1145                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1146                 fm = &aso_mtr->fm;
1147         } else {
1148                 legacy_fm = mlx5_ipool_zmalloc
1149                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1150                 if (legacy_fm == NULL)
1151                         return -rte_mtr_error_set(error, ENOMEM,
1152                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1153                                 "Memory alloc failed for meter.");
1154                 legacy_fm->idx = mtr_idx;
1155                 fm = &legacy_fm->fm;
1156         }
1157         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1158         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1159             mtr_reg_bits) {
1160                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1161                 goto error;
1162         }
1163         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1164                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1165         /* Fill the flow meter parameters. */
1166         fm->meter_id = meter_id;
1167         fm->policy_id = params->meter_policy_id;
1168         fm->profile = fmp;
1169         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1170                 goto error;
1171         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1172                 goto error;
1173         /* Add to the flow meter list. */
1174         if (!priv->sh->meter_aso_en)
1175                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1176         /* Add to the flow meter list. */
1177         fm->active_state = 1; /* Config meter starts as active. */
1178         fm->is_enable = 1;
1179         fm->shared = !!shared;
1180         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1181         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1182                 fm->def_policy = 1;
1183                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1184                 if (!fm->flow_ipool)
1185                         goto error;
1186         }
1187         rte_spinlock_init(&fm->sl);
1188         /* If ASO meter supported, update ASO flow meter by wqe. */
1189         if (priv->sh->meter_aso_en) {
1190                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1191                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1192                 if (ret)
1193                         goto error;
1194                 if (!priv->mtr_idx_tbl) {
1195                         priv->mtr_idx_tbl =
1196                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1197                         if (!priv->mtr_idx_tbl)
1198                                 goto error;
1199                 }
1200                 data.dword = mtr_idx;
1201                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1202                         goto error;
1203         }
1204         if (mtr_policy)
1205                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1206         return 0;
1207 error:
1208         mlx5_flow_destroy_mtr_tbls(dev, fm);
1209         /* Free policer counters. */
1210         if (fm->drop_cnt)
1211                 mlx5_counter_free(dev, fm->drop_cnt);
1212         if (priv->sh->meter_aso_en)
1213                 mlx5_flow_mtr_free(dev, mtr_idx);
1214         else
1215                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1216         return -rte_mtr_error_set(error, ENOTSUP,
1217                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1218                 NULL, "Failed to create devx meter.");
1219 }
1220
1221 static int
1222 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1223                         struct mlx5_flow_meter_info *fm,
1224                         uint32_t mtr_idx)
1225 {
1226         struct mlx5_priv *priv = dev->data->dev_private;
1227         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1228         struct mlx5_flow_meter_profile *fmp;
1229         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1230         struct mlx5_flow_meter_policy *mtr_policy;
1231
1232         /* Meter object must not have any owner. */
1233         MLX5_ASSERT(!fm->ref_cnt);
1234         /* Get meter profile. */
1235         fmp = fm->profile;
1236         if (fmp == NULL)
1237                 return -1;
1238         /* Update dependencies. */
1239         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1240         fm->profile = NULL;
1241         /* Remove from list. */
1242         if (!priv->sh->meter_aso_en) {
1243                 legacy_fm = container_of(fm,
1244                         struct mlx5_legacy_flow_meter, fm);
1245                 TAILQ_REMOVE(fms, legacy_fm, next);
1246         }
1247         /* Free drop counters. */
1248         if (fm->drop_cnt)
1249                 mlx5_counter_free(dev, fm->drop_cnt);
1250         /* Free meter flow table. */
1251         if (fm->flow_ipool) {
1252                 mlx5_ipool_destroy(fm->flow_ipool);
1253                 fm->flow_ipool = 0;
1254         }
1255         mlx5_flow_destroy_mtr_tbls(dev, fm);
1256         if (fm->def_policy)
1257                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1258                                 1, __ATOMIC_RELAXED);
1259         if (priv->sh->meter_aso_en) {
1260                 if (!fm->def_policy) {
1261                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1262                                                 fm->policy_id, NULL);
1263                         if (mtr_policy)
1264                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1265                                                 1, __ATOMIC_RELAXED);
1266                         fm->policy_id = 0;
1267                 }
1268                 fm->def_policy = 0;
1269                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1270                         return -1;
1271                 mlx5_flow_mtr_free(dev, mtr_idx);
1272         } else {
1273                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1274                                         legacy_fm->idx);
1275         }
1276         return 0;
1277 }
1278
1279 /**
1280  * Destroy meter rules.
1281  *
1282  * @param[in] dev
1283  *   Pointer to Ethernet device.
1284  * @param[in] meter_id
1285  *   Meter id.
1286  * @param[out] error
1287  *   Pointer to rte meter error structure.
1288  *
1289  * @return
1290  *   0 on success, a negative errno value otherwise and rte_errno is set.
1291  */
1292 static int
1293 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1294                         struct rte_mtr_error *error)
1295 {
1296         struct mlx5_priv *priv = dev->data->dev_private;
1297         struct mlx5_flow_meter_info *fm;
1298         uint32_t mtr_idx = 0;
1299
1300         if (!priv->mtr_en)
1301                 return -rte_mtr_error_set(error, ENOTSUP,
1302                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1303                                           NULL,
1304                                           "Meter is not supported");
1305         /* Meter object must exist. */
1306         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1307         if (fm == NULL)
1308                 return -rte_mtr_error_set(error, ENOENT,
1309                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1310                                           NULL,
1311                                           "Meter object id not valid.");
1312         /* Meter object must not have any owner. */
1313         if (fm->ref_cnt > 0)
1314                 return -rte_mtr_error_set(error, EBUSY,
1315                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1316                                           NULL,
1317                                           "Meter object is being used.");
1318         /* Destroy the meter profile. */
1319         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1320                 return -rte_mtr_error_set(error, EINVAL,
1321                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1322                                         NULL,
1323                                         "MTR object meter profile invalid.");
1324         return 0;
1325 }
1326
1327 /**
1328  * Modify meter state.
1329  *
1330  * @param[in] priv
1331  *   Pointer to mlx5 private data structure.
1332  * @param[in] fm
1333  *   Pointer to flow meter.
1334  * @param[in] new_state
1335  *   New state to update.
1336  * @param[out] error
1337  *   Pointer to rte meter error structure.
1338  *
1339  * @return
1340  *   0 on success, a negative errno value otherwise and rte_errno is set.
1341  */
1342 static int
1343 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1344                              struct mlx5_flow_meter_info *fm,
1345                              uint32_t new_state,
1346                              struct rte_mtr_error *error)
1347 {
1348         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1349                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1350                 .ebs_eir = 0,
1351         };
1352         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1353                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1354         int ret;
1355
1356         if (new_state == MLX5_FLOW_METER_DISABLE)
1357                 ret = mlx5_flow_meter_action_modify(priv, fm,
1358                                 &srtcm, modify_bits, 0, 0);
1359         else
1360                 ret = mlx5_flow_meter_action_modify(priv, fm,
1361                                                    &fm->profile->srtcm_prm,
1362                                                     modify_bits, 0, 1);
1363         if (ret)
1364                 return -rte_mtr_error_set(error, -ret,
1365                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1366                                           NULL,
1367                                           new_state ?
1368                                           "Failed to enable meter." :
1369                                           "Failed to disable meter.");
1370         return 0;
1371 }
1372
1373 /**
1374  * Callback to enable flow meter.
1375  *
1376  * @param[in] dev
1377  *   Pointer to Ethernet device.
1378  * @param[in] meter_id
1379  *   Meter id.
1380  * @param[out] error
1381  *   Pointer to rte meter error structure.
1382  *
1383  * @return
1384  *   0 on success, a negative errno value otherwise and rte_errno is set.
1385  */
1386 static int
1387 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1388                        uint32_t meter_id,
1389                        struct rte_mtr_error *error)
1390 {
1391         struct mlx5_priv *priv = dev->data->dev_private;
1392         struct mlx5_flow_meter_info *fm;
1393         int ret;
1394
1395         if (!priv->mtr_en)
1396                 return -rte_mtr_error_set(error, ENOTSUP,
1397                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1398                                           "Meter is not supported");
1399         /* Meter object must exist. */
1400         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1401         if (fm == NULL)
1402                 return -rte_mtr_error_set(error, ENOENT,
1403                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1404                                           NULL, "Meter not found.");
1405         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1406                 return 0;
1407         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1408                                            error);
1409         if (!ret)
1410                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1411         return ret;
1412 }
1413
1414 /**
1415  * Callback to disable flow meter.
1416  *
1417  * @param[in] dev
1418  *   Pointer to Ethernet device.
1419  * @param[in] meter_id
1420  *   Meter id.
1421  * @param[out] error
1422  *   Pointer to rte meter error structure.
1423  *
1424  * @return
1425  *   0 on success, a negative errno value otherwise and rte_errno is set.
1426  */
1427 static int
1428 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1429                         uint32_t meter_id,
1430                         struct rte_mtr_error *error)
1431 {
1432         struct mlx5_priv *priv = dev->data->dev_private;
1433         struct mlx5_flow_meter_info *fm;
1434         int ret;
1435
1436         if (!priv->mtr_en)
1437                 return -rte_mtr_error_set(error, ENOTSUP,
1438                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1439                                           "Meter is not supported");
1440         /* Meter object must exist. */
1441         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1442         if (fm == NULL)
1443                 return -rte_mtr_error_set(error, ENOENT,
1444                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1445                                           NULL, "Meter not found.");
1446         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1447                 return 0;
1448         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1449                                            error);
1450         if (!ret)
1451                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1452         return ret;
1453 }
1454
1455 /**
1456  * Callback to update meter profile.
1457  *
1458  * @param[in] dev
1459  *   Pointer to Ethernet device.
1460  * @param[in] meter_id
1461  *   Meter id.
1462  * @param[in] meter_profile_id
1463  *   To be updated meter profile id.
1464  * @param[out] error
1465  *   Pointer to rte meter error structure.
1466  *
1467  * @return
1468  *   0 on success, a negative errno value otherwise and rte_errno is set.
1469  */
1470 static int
1471 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1472                                uint32_t meter_id,
1473                                uint32_t meter_profile_id,
1474                                struct rte_mtr_error *error)
1475 {
1476         struct mlx5_priv *priv = dev->data->dev_private;
1477         struct mlx5_flow_meter_profile *fmp;
1478         struct mlx5_flow_meter_profile *old_fmp;
1479         struct mlx5_flow_meter_info *fm;
1480         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1481                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1482         int ret;
1483
1484         if (!priv->mtr_en)
1485                 return -rte_mtr_error_set(error, ENOTSUP,
1486                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1487                                           "Meter is not supported");
1488         /* Meter profile must exist. */
1489         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1490         if (fmp == NULL)
1491                 return -rte_mtr_error_set(error, ENOENT,
1492                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1493                                           NULL, "Meter profile not found.");
1494         /* Meter object must exist. */
1495         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1496         if (fm == NULL)
1497                 return -rte_mtr_error_set(error, ENOENT,
1498                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1499                                           NULL, "Meter not found.");
1500         /* MTR object already set to meter profile id. */
1501         old_fmp = fm->profile;
1502         if (fmp == old_fmp)
1503                 return 0;
1504         /* Update the profile. */
1505         fm->profile = fmp;
1506         /* Update meter params in HW (if not disabled). */
1507         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1508                 return 0;
1509         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1510                                               modify_bits, fm->active_state, 1);
1511         if (ret) {
1512                 fm->profile = old_fmp;
1513                 return -rte_mtr_error_set(error, -ret,
1514                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1515                                           NULL, "Failed to update meter"
1516                                           " parmeters in hardware.");
1517         }
1518         old_fmp->ref_cnt--;
1519         fmp->ref_cnt++;
1520         return 0;
1521 }
1522
1523 /**
1524  * Callback to update meter stats mask.
1525  *
1526  * @param[in] dev
1527  *   Pointer to Ethernet device.
1528  * @param[in] meter_id
1529  *   Meter id.
1530  * @param[in] stats_mask
1531  *   To be updated stats_mask.
1532  * @param[out] error
1533  *   Pointer to rte meter error structure.
1534  *
1535  * @return
1536  *   0 on success, a negative errno value otherwise and rte_errno is set.
1537  */
1538 static int
1539 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1540                              uint32_t meter_id,
1541                              uint64_t stats_mask,
1542                              struct rte_mtr_error *error)
1543 {
1544         struct mlx5_priv *priv = dev->data->dev_private;
1545         struct mlx5_flow_meter_info *fm;
1546
1547         if (!priv->mtr_en)
1548                 return -rte_mtr_error_set(error, ENOTSUP,
1549                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1550                                           "Meter is not supported");
1551         /* Meter object must exist. */
1552         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1553         if (fm == NULL)
1554                 return -rte_mtr_error_set(error, ENOENT,
1555                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1556                                           NULL, "Meter object id not valid.");
1557         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1558                 return -rte_mtr_error_set(error, ENOENT,
1559                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1560                                           NULL, "Fail to allocate "
1561                                           "counter for meter.");
1562         return 0;
1563 }
1564
1565 /**
1566  * Callback to read meter statistics.
1567  *
1568  * @param[in] dev
1569  *   Pointer to Ethernet device.
1570  * @param[in] meter_id
1571  *   Meter id.
1572  * @param[out] stats
1573  *   Pointer to store the statistics.
1574  * @param[out] stats_mask
1575  *   Pointer to store the stats_mask.
1576  * @param[in] clear
1577  *   Statistic to be cleared after read or not.
1578  * @param[out] error
1579  *   Pointer to rte meter error structure.
1580  *
1581  * @return
1582  *   0 on success, a negative errno value otherwise and rte_errno is set.
1583  */
1584 static int
1585 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1586                            uint32_t meter_id,
1587                            struct rte_mtr_stats *stats,
1588                            uint64_t *stats_mask,
1589                            int clear,
1590                            struct rte_mtr_error *error)
1591 {
1592         struct mlx5_priv *priv = dev->data->dev_private;
1593         struct mlx5_flow_meter_info *fm;
1594         uint64_t pkts;
1595         uint64_t bytes;
1596         int ret = 0;
1597
1598         if (!priv->mtr_en)
1599                 return -rte_mtr_error_set(error, ENOTSUP,
1600                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1601                                           "Meter is not supported");
1602         /* Meter object must exist. */
1603         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1604         if (fm == NULL)
1605                 return -rte_mtr_error_set(error, ENOENT,
1606                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1607                                           NULL, "Meter object id not valid.");
1608         *stats_mask = 0;
1609         if (fm->bytes_dropped)
1610                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1611         if (fm->pkts_dropped)
1612                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1613         memset(stats, 0, sizeof(*stats));
1614         if (fm->drop_cnt) {
1615                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1616                                                  &bytes);
1617                 if (ret)
1618                         goto error;
1619                 /* If need to read the packets, set it. */
1620                 if (fm->pkts_dropped)
1621                         stats->n_pkts_dropped = pkts;
1622                 /* If need to read the bytes, set it. */
1623                 if (fm->bytes_dropped)
1624                         stats->n_bytes_dropped = bytes;
1625         }
1626         return 0;
1627 error:
1628         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1629                                  "Failed to read meter drop counters.");
1630 }
1631
1632 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1633         .capabilities_get = mlx5_flow_mtr_cap_get,
1634         .meter_profile_add = mlx5_flow_meter_profile_add,
1635         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1636         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1637         .meter_policy_add = mlx5_flow_meter_policy_add,
1638         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1639         .create = mlx5_flow_meter_create,
1640         .destroy = mlx5_flow_meter_destroy,
1641         .meter_enable = mlx5_flow_meter_enable,
1642         .meter_disable = mlx5_flow_meter_disable,
1643         .meter_profile_update = mlx5_flow_meter_profile_update,
1644         .meter_dscp_table_update = NULL,
1645         .stats_update = mlx5_flow_meter_stats_update,
1646         .stats_read = mlx5_flow_meter_stats_read,
1647 };
1648
1649 /**
1650  * Get meter operations.
1651  *
1652  * @param dev
1653  *   Pointer to Ethernet device structure.
1654  * @param arg
1655  *   Pointer to set the mtr operations.
1656  *
1657  * @return
1658  *   Always 0.
1659  */
1660 int
1661 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1662 {
1663         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1664         return 0;
1665 }
1666
1667 /**
1668  * Find meter by id.
1669  *
1670  * @param priv
1671  *   Pointer to mlx5_priv.
1672  * @param meter_id
1673  *   Meter id.
1674  * @param mtr_idx
1675  *   Pointer to Meter index.
1676  *
1677  * @return
1678  *   Pointer to the meter info found on success, NULL otherwise.
1679  */
1680 struct mlx5_flow_meter_info *
1681 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1682                 uint32_t *mtr_idx)
1683 {
1684         struct mlx5_legacy_flow_meter *legacy_fm;
1685         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1686         struct mlx5_aso_mtr *aso_mtr;
1687         struct mlx5_aso_mtr_pools_mng *pools_mng =
1688                                 &priv->sh->mtrmng->pools_mng;
1689         union mlx5_l3t_data data;
1690
1691         if (priv->sh->meter_aso_en) {
1692                 rte_spinlock_lock(&pools_mng->mtrsl);
1693                 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1694                         rte_spinlock_unlock(&pools_mng->mtrsl);
1695                         return NULL;
1696                 }
1697                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1698                         !data.dword) {
1699                         rte_spinlock_unlock(&pools_mng->mtrsl);
1700                         return NULL;
1701                 }
1702                 if (mtr_idx)
1703                         *mtr_idx = data.dword;
1704                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1705                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1706                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1707                 rte_spinlock_unlock(&pools_mng->mtrsl);
1708                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1709                         return NULL;
1710                 return &aso_mtr->fm;
1711         }
1712         TAILQ_FOREACH(legacy_fm, fms, next)
1713                 if (meter_id == legacy_fm->fm.meter_id) {
1714                         if (mtr_idx)
1715                                 *mtr_idx = legacy_fm->idx;
1716                         return &legacy_fm->fm;
1717                 }
1718         return NULL;
1719 }
1720
1721 /**
1722  * Find meter by index.
1723  *
1724  * @param priv
1725  *   Pointer to mlx5_priv.
1726  * @param idx
1727  *   Meter index.
1728  *
1729  * @return
1730  *   Pointer to the meter info found on success, NULL otherwise.
1731  */
1732 struct mlx5_flow_meter_info *
1733 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1734 {
1735         struct mlx5_aso_mtr *aso_mtr;
1736
1737         if (priv->sh->meter_aso_en) {
1738                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1739                 if (!aso_mtr)
1740                         return NULL;
1741                 return &aso_mtr->fm;
1742         } else {
1743                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1744         }
1745 }
1746
1747 /**
1748  * Attach meter to flow.
1749  * Unidirectional Meter creation can only be done
1750  * when flow direction is known, i.e. when calling meter_attach.
1751  *
1752  * @param [in] priv
1753  *  Pointer to mlx5 private data.
1754  * @param[in] fm
1755  *   Pointer to flow meter.
1756  * @param [in] attr
1757  *  Pointer to flow attributes.
1758  * @param [out] error
1759  *  Pointer to error structure.
1760  *
1761  * @return
1762  *   0 on success, a negative errno value otherwise and rte_errno is set.
1763  */
1764 int
1765 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1766                        struct mlx5_flow_meter_info *fm,
1767                        const struct rte_flow_attr *attr,
1768                        struct rte_flow_error *error)
1769 {
1770         int ret = 0;
1771
1772         if (priv->sh->meter_aso_en) {
1773                 struct mlx5_aso_mtr *aso_mtr;
1774
1775                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1776                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1777                         return rte_flow_error_set(error, ENOENT,
1778                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1779                                         NULL,
1780                                         "Timeout in meter configuration");
1781                 }
1782                 rte_spinlock_lock(&fm->sl);
1783                 if (fm->shared || !fm->ref_cnt) {
1784                         fm->ref_cnt++;
1785                 } else {
1786                         rte_flow_error_set(error, EINVAL,
1787                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1788                                    "Meter cannot be shared");
1789                         ret = -1;
1790                 }
1791                 rte_spinlock_unlock(&fm->sl);
1792         } else {
1793                 rte_spinlock_lock(&fm->sl);
1794                 if (fm->meter_action) {
1795                         if (fm->shared &&
1796                             attr->transfer == fm->transfer &&
1797                             attr->ingress == fm->ingress &&
1798                             attr->egress == fm->egress) {
1799                                 fm->ref_cnt++;
1800                         } else {
1801                                 rte_flow_error_set(error, EINVAL,
1802                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1803                                         fm->shared ?
1804                                         "Meter attr not match." :
1805                                         "Meter cannot be shared.");
1806                                 ret = -1;
1807                         }
1808                 } else {
1809                         fm->ingress = attr->ingress;
1810                         fm->egress = attr->egress;
1811                         fm->transfer = attr->transfer;
1812                         fm->ref_cnt = 1;
1813                         /* This also creates the meter object. */
1814                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1815                                                                          fm);
1816                         if (!fm->meter_action) {
1817                                 fm->ref_cnt = 0;
1818                                 fm->ingress = 0;
1819                                 fm->egress = 0;
1820                                 fm->transfer = 0;
1821                                 rte_flow_error_set(error, EINVAL,
1822                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1823                                         "Meter action create failed.");
1824                                 ret = -1;
1825                         }
1826                 }
1827                 rte_spinlock_unlock(&fm->sl);
1828         }
1829         return ret ? -rte_errno : 0;
1830 }
1831
1832 /**
1833  * Detach meter from flow.
1834  *
1835  * @param [in] priv
1836  *  Pointer to mlx5 private data.
1837  * @param [in] fm
1838  *  Pointer to flow meter.
1839  */
1840 void
1841 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1842                        struct mlx5_flow_meter_info *fm)
1843 {
1844 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1845         rte_spinlock_lock(&fm->sl);
1846         MLX5_ASSERT(fm->ref_cnt);
1847         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1848                 mlx5_glue->destroy_flow_action(fm->meter_action);
1849                 fm->meter_action = NULL;
1850                 fm->ingress = 0;
1851                 fm->egress = 0;
1852                 fm->transfer = 0;
1853         }
1854         rte_spinlock_unlock(&fm->sl);
1855 #else
1856         (void)priv;
1857         (void)fm;
1858 #endif
1859 }
1860
1861 /**
1862  * Flush meter with Rx queue configuration.
1863  *
1864  * @param[in] dev
1865  *   Pointer to Ethernet device.
1866  */
1867 void
1868 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1869 {
1870         struct mlx5_priv *priv = dev->data->dev_private;
1871         struct mlx5_flow_meter_sub_policy *sub_policy;
1872         struct mlx5_flow_meter_policy *mtr_policy;
1873         void *entry;
1874         uint32_t i, policy_idx;
1875
1876         if (!priv->mtr_en)
1877                 return;
1878         if (priv->policy_idx_tbl) {
1879                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1880                         policy_idx = *(uint32_t *)entry;
1881                         sub_policy = mlx5_ipool_get
1882                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1883                                 policy_idx);
1884                         if (!sub_policy || !sub_policy->main_policy)
1885                                 continue;
1886                         mtr_policy = sub_policy->main_policy;
1887                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1888                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1889                                         mtr_policy);
1890                 }
1891         }
1892 }
1893
1894 /**
1895  * Iterate a meter hierarchy and flush all meters and policies if possible.
1896  *
1897  * @param[in] dev
1898  *   Pointer to Ethernet device.
1899  * @param[in] fm
1900  *   Pointer to flow meter.
1901  * @param[in] mtr_idx
1902  *   .Meter's index
1903  * @param[out] error
1904  *   Pointer to rte meter error structure.
1905  *
1906  * @return
1907  *   0 on success, a negative errno value otherwise and rte_errno is set.
1908  */
1909 static int
1910 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
1911                                 struct mlx5_flow_meter_info *fm,
1912                                 uint32_t mtr_idx,
1913                                 struct rte_mtr_error *error)
1914 {
1915         struct mlx5_priv *priv = dev->data->dev_private;
1916         struct mlx5_flow_meter_policy *policy;
1917         uint32_t policy_id;
1918         struct mlx5_flow_meter_info *next_fm;
1919         uint32_t next_mtr_idx;
1920         struct mlx5_flow_meter_policy *next_policy = NULL;
1921
1922         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
1923         MLX5_ASSERT(policy);
1924         while (!fm->ref_cnt && policy->is_hierarchy) {
1925                 policy_id = fm->policy_id;
1926                 next_fm = mlx5_flow_meter_find(priv,
1927                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
1928                                 &next_mtr_idx);
1929                 if (next_fm) {
1930                         next_policy = mlx5_flow_meter_policy_find(dev,
1931                                                         next_fm->policy_id,
1932                                                         NULL);
1933                         MLX5_ASSERT(next_policy);
1934                 }
1935                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1936                         return -rte_mtr_error_set(error, ENOTSUP,
1937                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
1938                                                 NULL,
1939                                                 "Failed to flush meter.");
1940                 if (policy->ref_cnt)
1941                         break;
1942                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
1943                                                 policy, error, true))
1944                         return -rte_errno;
1945                 mlx5_free(policy);
1946                 if (!next_fm || !next_policy)
1947                         break;
1948                 fm = next_fm;
1949                 mtr_idx = next_mtr_idx;
1950                 policy = next_policy;
1951         }
1952         return 0;
1953 }
1954
1955 /**
1956  * Flush all the hierarchy meters and their policies.
1957  *
1958  * @param[in] dev
1959  *   Pointer to Ethernet device.
1960  * @param[out] error
1961  *   Pointer to rte meter error structure.
1962  *
1963  * @return
1964  *   0 on success, a negative errno value otherwise and rte_errno is set.
1965  */
1966 static int
1967 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
1968                                       struct rte_mtr_error *error)
1969 {
1970         struct mlx5_priv *priv = dev->data->dev_private;
1971         struct mlx5_flow_meter_info *fm;
1972         struct mlx5_flow_meter_policy *policy;
1973         struct mlx5_flow_meter_sub_policy *sub_policy;
1974         struct mlx5_flow_meter_info *next_fm;
1975         struct mlx5_aso_mtr *aso_mtr;
1976         uint32_t mtr_idx = 0;
1977         uint32_t i, policy_idx;
1978         void *entry;
1979
1980         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
1981                 return 0;
1982         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
1983                 mtr_idx = *(uint32_t *)entry;
1984                 if (!mtr_idx)
1985                         continue;
1986                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1987                 fm = &aso_mtr->fm;
1988                 if (fm->ref_cnt || fm->def_policy)
1989                         continue;
1990                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
1991                         return -rte_errno;
1992         }
1993         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1994                 policy_idx = *(uint32_t *)entry;
1995                 sub_policy = mlx5_ipool_get
1996                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1997                                 policy_idx);
1998                 if (!sub_policy)
1999                         return -rte_mtr_error_set(error,
2000                                         EINVAL,
2001                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2002                                         NULL, "Meter policy invalid.");
2003                 policy = sub_policy->main_policy;
2004                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2005                         continue;
2006                 next_fm = mlx5_flow_meter_find(priv,
2007                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2008                                 &mtr_idx);
2009                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2010                                                     error, true))
2011                         return -rte_mtr_error_set(error,
2012                                         EINVAL,
2013                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2014                                         NULL, "Meter policy invalid.");
2015                 mlx5_free(policy);
2016                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2017                         continue;
2018                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2019                                                     mtr_idx, error))
2020                         return -rte_errno;
2021         }
2022         return 0;
2023 }
2024 /**
2025  * Flush meter configuration.
2026  *
2027  * @param[in] dev
2028  *   Pointer to Ethernet device.
2029  * @param[out] error
2030  *   Pointer to rte meter error structure.
2031  *
2032  * @return
2033  *   0 on success, a negative errno value otherwise and rte_errno is set.
2034  */
2035 int
2036 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2037 {
2038         struct mlx5_priv *priv = dev->data->dev_private;
2039         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2040         struct mlx5_flow_meter_profile *fmp;
2041         struct mlx5_legacy_flow_meter *legacy_fm;
2042         struct mlx5_flow_meter_info *fm;
2043         struct mlx5_flow_meter_sub_policy *sub_policy;
2044         void *tmp;
2045         uint32_t i, mtr_idx, policy_idx;
2046         void *entry;
2047         struct mlx5_aso_mtr *aso_mtr;
2048
2049         if (!priv->mtr_en)
2050                 return 0;
2051         if (priv->sh->meter_aso_en) {
2052                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2053                         return -rte_errno;
2054                 if (priv->mtr_idx_tbl) {
2055                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2056                                 mtr_idx = *(uint32_t *)entry;
2057                                 if (mtr_idx) {
2058                                         aso_mtr =
2059                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2060                                         fm = &aso_mtr->fm;
2061                                         (void)mlx5_flow_meter_params_flush(dev,
2062                                                 fm, mtr_idx);
2063                                 }
2064                         }
2065                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2066                         priv->mtr_idx_tbl = NULL;
2067                 }
2068         } else {
2069                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2070                         fm = &legacy_fm->fm;
2071                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2072                                 return -rte_mtr_error_set(error, EINVAL,
2073                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2074                                 NULL, "MTR object meter profile invalid.");
2075                 }
2076         }
2077         if (priv->policy_idx_tbl) {
2078                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2079                         policy_idx = *(uint32_t *)entry;
2080                         sub_policy = mlx5_ipool_get
2081                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2082                                 policy_idx);
2083                         if (!sub_policy)
2084                                 return -rte_mtr_error_set(error,
2085                                                 EINVAL,
2086                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2087                                                 NULL, "MTR object "
2088                                                 "meter policy invalid.");
2089                         if (__mlx5_flow_meter_policy_delete(dev, i,
2090                                                 sub_policy->main_policy,
2091                                                 error, true))
2092                                 return -rte_mtr_error_set(error,
2093                                                 EINVAL,
2094                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2095                                                 NULL, "MTR object "
2096                                                 "meter policy invalid.");
2097                         mlx5_free(sub_policy->main_policy);
2098                 }
2099                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2100                 priv->policy_idx_tbl = NULL;
2101         }
2102         if (priv->mtr_profile_tbl) {
2103                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2104                         fmp = entry;
2105                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2106                                                            error))
2107                                 return -rte_mtr_error_set(error, EINVAL,
2108                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2109                                                 NULL, "Fail to destroy "
2110                                                 "meter profile.");
2111                 }
2112                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2113                 priv->mtr_profile_tbl = NULL;
2114         }
2115         /* Delete default policy table. */
2116         mlx5_flow_destroy_def_policy(dev);
2117         if (priv->sh->refcnt == 1)
2118                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2119         return 0;
2120 }