drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133
 134         /* Profile must not be NULL. */
 135         if (profile == NULL)
 136                 return -rte_mtr_error_set(error, EINVAL,
 137                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 138                                           NULL, "Meter profile is null.");
 139         /* Meter profile ID must be valid. */
 140         if (meter_profile_id == UINT32_MAX)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 143                                           NULL, "Meter profile id not valid.");
 144         /* Meter profile must not exist. */
 145         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 146         if (fmp)
 147                 return -rte_mtr_error_set(error, EEXIST,
 148                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 149                                           NULL,
 150                                           "Meter profile already exists.");
 151         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 152                 if (priv->config.hca_attr.qos.flow_meter_old) {
 153                         /* Verify support for flow meter parameters. */
 154                         if (priv->sh->meter_aso_en && profile->packet_mode) {
 155                                 if (profile->srtcm_rfc2697.cir > 0 &&
 156                                         (profile->srtcm_rfc2697.cir <<
 157                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 158                                         <= MLX5_SRTCM_CIR_MAX &&
 159                                         profile->srtcm_rfc2697.cbs > 0 &&
 160                                         (profile->srtcm_rfc2697.cbs <<
 161                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 162                                         <= MLX5_SRTCM_CBS_MAX &&
 163                                         (profile->srtcm_rfc2697.ebs <<
 164                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 165                                         <= MLX5_SRTCM_EBS_MAX)
 166                                         return 0;
 167                                 return -rte_mtr_error_set
 168                                              (error, ENOTSUP,
 169                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 170                                               NULL,
 171                                               profile->srtcm_rfc2697.ebs ?
 172                                               "Metering value ebs must be 0." :
 173                                               "Invalid metering parameters.");
 174                         }
 175                         if (profile->srtcm_rfc2697.cir > 0 &&
 176                                 profile->srtcm_rfc2697.cir <=
 177                                                 MLX5_SRTCM_CIR_MAX &&
 178                                 profile->srtcm_rfc2697.cbs > 0 &&
 179                                 profile->srtcm_rfc2697.cbs <=
 180                                                 MLX5_SRTCM_CBS_MAX &&
 181                                 profile->srtcm_rfc2697.ebs <=
 182                                                 MLX5_SRTCM_EBS_MAX)
 183                                 return 0;
 184                         return -rte_mtr_error_set(error, ENOTSUP,
 185                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 186                                         NULL,
 187                                         profile->srtcm_rfc2697.ebs ?
 188                                         "Metering value ebs must be 0." :
 189                                         "Invalid metering parameters.");
 190                 }
 191         }
 192         return -rte_mtr_error_set(error, ENOTSUP,
 193                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 194                                   NULL, "Metering algorithm not supported.");
 195 }
 196
 197 /**
 198  * Calculate mantissa and exponent for cir.
 199  *
 200  * @param[in] cir
 201  *   Value to be calculated.
 202  * @param[out] man
 203  *   Pointer to the mantissa.
 204  * @param[out] exp
 205  *   Pointer to the exp.
 206  */
 207 static void
 208 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 209 {
 210         int64_t _cir;
 211         int64_t delta = INT64_MAX;
 212         uint8_t _man = 0;
 213         uint8_t _exp = 0;
 214         uint64_t m, e;
 215
 216         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 217                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 218                         _cir = (1000000000ULL * m) >> e;
 219                         if (llabs(cir - _cir) <= delta) {
 220                                 delta = llabs(cir - _cir);
 221                                 _man = m;
 222                                 _exp = e;
 223                         }
 224                 }
 225         }
 226         *man = _man;
 227         *exp = _exp;
 228 }
 229
 230 /**
 231  * Calculate mantissa and exponent for xbs.
 232  *
 233  * @param[in] xbs
 234  *   Value to be calculated.
 235  * @param[out] man
 236  *   Pointer to the mantissa.
 237  * @param[out] exp
 238  *   Pointer to the exp.
 239  */
 240 static void
 241 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 242 {
 243         int _exp;
 244         double _man;
 245
 246         /* Special case xbs == 0 ? both exp and matissa are 0. */
 247         if (xbs == 0) {
 248                 *man = 0;
 249                 *exp = 0;
 250                 return;
 251         }
 252         /* xbs = xbs_mantissa * 2^xbs_exponent */
 253         _man = frexp(xbs, &_exp);
 254         _man = _man * pow(2, MLX5_MAN_WIDTH);
 255         _exp = _exp - MLX5_MAN_WIDTH;
 256         *man = (uint8_t)ceil(_man);
 257         *exp = _exp;
 258 }
 259
 260 /**
 261  * Fill the prm meter parameter.
 262  *
 263  * @param[in,out] fmp
 264  *   Pointer to meter profie to be converted.
 265  * @param[out] error
 266  *   Pointer to the error structure.
 267  *
 268  * @return
 269  *   0 on success, a negative errno value otherwise and rte_errno is set.
 270  */
 271 static int
 272 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 273                         struct mlx5_priv *priv, struct rte_mtr_error *error)
 274 {
 275         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 276         uint8_t man, exp;
 277         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 278         uint32_t ebs_exp, ebs_man;
 279         uint64_t cir, cbs, ebs;
 280
 281         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 282                 return -rte_mtr_error_set(error, ENOTSUP,
 283                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 284                                 NULL, "Metering algorithm not supported.");
 285         if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
 286                 return -rte_mtr_error_set(error, ENOTSUP,
 287                         RTE_MTR_ERROR_TYPE_METER_PROFILE,
 288                         NULL, "Metering algorithm packet mode not supported.");
 289         if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
 290                 cir = fmp->profile.srtcm_rfc2697.cir <<
 291                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 292                 cbs = fmp->profile.srtcm_rfc2697.cbs <<
 293                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 294                 ebs = fmp->profile.srtcm_rfc2697.ebs <<
 295                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 296         } else {
 297                 cir = fmp->profile.srtcm_rfc2697.cir;
 298                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 299                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 300         }
 301         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 302         mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
 303         /* Check if cir mantissa is too large. */
 304         if (exp > ASO_DSEG_CIR_EXP_MASK)
 305                 return -rte_mtr_error_set(error, ENOTSUP,
 306                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 307                                           "meter profile parameter cir is"
 308                                           " not supported.");
 309         cir_man = man;
 310         cir_exp = exp;
 311          /* cbs = cbs_mantissa * 2^cbs_exponent */
 312         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 313         /* Check if cbs mantissa is too large. */
 314         if (exp > ASO_DSEG_EXP_MASK)
 315                 return -rte_mtr_error_set(error, ENOTSUP,
 316                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 317                                           "meter profile parameter cbs is"
 318                                           " not supported.");
 319         cbs_man = man;
 320         cbs_exp = exp;
 321         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 322                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 323                                           cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 324                                           cir_man);
 325         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 326         /* Check if ebs mantissa is too large. */
 327         if (exp > ASO_DSEG_EXP_MASK)
 328                 return -rte_mtr_error_set(error, ENOTSUP,
 329                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 330                                           "meter profile parameter ebs is"
 331                                           " not supported.");
 332         ebs_man = man;
 333         ebs_exp = exp;
 334         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 335                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 336         return 0;
 337 }
 338
 339 /**
 340  * Callback to get MTR capabilities.
 341  *
 342  * @param[in] dev
 343  *   Pointer to Ethernet device.
 344  * @param[out] cap
 345  *   Pointer to save MTR capabilities.
 346  * @param[out] error
 347  *   Pointer to the error structure.
 348  *
 349  * @return
 350  *   0 on success, a negative errno value otherwise and rte_errno is set.
 351  */
 352 static int
 353 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 354                  struct rte_mtr_capabilities *cap,
 355                  struct rte_mtr_error *error __rte_unused)
 356 {
 357         struct mlx5_priv *priv = dev->data->dev_private;
 358         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 359
 360         if (!priv->mtr_en)
 361                 return -rte_mtr_error_set(error, ENOTSUP,
 362                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 363                                           "Meter is not supported");
 364         memset(cap, 0, sizeof(*cap));
 365         if (priv->sh->meter_aso_en) {
 366                 /* 2 meters per one ASO cache line. */
 367                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 368                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 369         } else {
 370                 cap->n_max = 1 << qattr->log_max_flow_meter;
 371                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 372         }
 373         cap->srtcm_rfc2697_byte_mode_supported = 1;
 374         cap->n_shared_max = cap->n_max;
 375         cap->identical = 1;
 376         cap->shared_identical = 1;
 377         cap->shared_n_flows_per_mtr_max = 4 << 20;
 378         /* 2M flows can share the same meter. */
 379         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 380         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 381         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 382         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 383                           RTE_MTR_STATS_N_PKTS_DROPPED;
 384         return 0;
 385 }
 386
 387 /**
 388  * Callback to add MTR profile.
 389  *
 390  * @param[in] dev
 391  *   Pointer to Ethernet device.
 392  * @param[in] meter_profile_id
 393  *   Meter profile id.
 394  * @param[in] profile
 395  *   Pointer to meter profile detail.
 396  * @param[out] error
 397  *   Pointer to the error structure.
 398  *
 399  * @return
 400  *   0 on success, a negative errno value otherwise and rte_errno is set.
 401  */
 402 static int
 403 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 404                        uint32_t meter_profile_id,
 405                        struct rte_mtr_meter_profile *profile,
 406                        struct rte_mtr_error *error)
 407 {
 408         struct mlx5_priv *priv = dev->data->dev_private;
 409         struct mlx5_flow_meter_profile *fmp;
 410         union mlx5_l3t_data data;
 411         int ret;
 412
 413         if (!priv->mtr_en)
 414                 return -rte_mtr_error_set(error, ENOTSUP,
 415                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 416                                           "Meter is not supported");
 417         /* Check input params. */
 418         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 419                                                profile, error);
 420         if (ret)
 421                 return ret;
 422         /* Meter profile memory allocation. */
 423         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 424                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 425         if (fmp == NULL)
 426                 return -rte_mtr_error_set(error, ENOMEM,
 427                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 428                                           NULL, "Meter profile memory "
 429                                           "alloc failed.");
 430         /* Fill profile info. */
 431         fmp->id = meter_profile_id;
 432         fmp->profile = *profile;
 433         /* Fill the flow meter parameters for the PRM. */
 434         ret = mlx5_flow_meter_param_fill(fmp, priv, error);
 435         if (ret)
 436                 goto error;
 437         data.ptr = fmp;
 438         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 439                                  meter_profile_id, &data);
 440         if (ret)
 441                 return -rte_mtr_error_set(error, ENOTSUP,
 442                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 443                                           NULL, "Meter profile insert fail.");
 444         return 0;
 445 error:
 446         mlx5_free(fmp);
 447         return ret;
 448 }
 449
 450 /**
 451  * Callback to delete MTR profile.
 452  *
 453  * @param[in] dev
 454  *   Pointer to Ethernet device.
 455  * @param[in] meter_profile_id
 456  *   Meter profile id.
 457  * @param[out] error
 458  *   Pointer to the error structure.
 459  *
 460  * @return
 461  *   0 on success, a negative errno value otherwise and rte_errno is set.
 462  */
 463 static int
 464 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 465                           uint32_t meter_profile_id,
 466                           struct rte_mtr_error *error)
 467 {
 468         struct mlx5_priv *priv = dev->data->dev_private;
 469         struct mlx5_flow_meter_profile *fmp;
 470
 471         if (!priv->mtr_en)
 472                 return -rte_mtr_error_set(error, ENOTSUP,
 473                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 474                                           "Meter is not supported");
 475         /* Meter profile must exist. */
 476         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 477         if (fmp == NULL)
 478                 return -rte_mtr_error_set(error, ENOENT,
 479                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 480                                           &meter_profile_id,
 481                                           "Meter profile id is invalid.");
 482         /* Check profile is unused. */
 483         if (fmp->ref_cnt)
 484                 return -rte_mtr_error_set(error, EBUSY,
 485                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 486                                           NULL, "Meter profile is in use.");
 487         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 488                 return -rte_mtr_error_set(error, EBUSY,
 489                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 490                                           NULL, "Meter profile remove fail.");
 491         mlx5_free(fmp);
 492         return 0;
 493 }
 494
 495 /**
 496  * Find policy by id.
 497  *
 498  * @param[in] dev
 499  *   Pointer to Ethernet device.
 500  * @param policy_id
 501  *   Policy id.
 502  *
 503  * @return
 504  *   Pointer to the policy found on success, NULL otherwise.
 505  */
 506 struct mlx5_flow_meter_policy *
 507 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 508                             uint32_t policy_id,
 509                             uint32_t *policy_idx)
 510 {
 511         struct mlx5_priv *priv = dev->data->dev_private;
 512         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 513         union mlx5_l3t_data data;
 514
 515         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 516                 return NULL;
 517         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 518                                 !data.dword)
 519                 return NULL;
 520         if (policy_idx)
 521                 *policy_idx = data.dword;
 522         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 523                                         data.dword);
 524         /* Remove reference taken by the mlx5_l3t_get_entry. */
 525         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 526         if (sub_policy)
 527                 if (sub_policy->main_policy_id)
 528                         return sub_policy->main_policy;
 529         return NULL;
 530 }
 531
 532 /**
 533  * Get the last meter's policy from one meter's policy in hierarchy.
 534  *
 535  * @param[in] dev
 536  *   Pointer to Ethernet device.
 537  * @param[in] policy
 538  *   Pointer to flow meter policy.
 539  *
 540  * @return
 541  *   Pointer to the final meter's policy, or NULL when fail.
 542  */
 543 struct mlx5_flow_meter_policy *
 544 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 545                                         struct mlx5_flow_meter_policy *policy)
 546 {
 547         struct mlx5_priv *priv = dev->data->dev_private;
 548         struct mlx5_flow_meter_info *next_fm;
 549         struct mlx5_flow_meter_policy *next_policy = policy;
 550
 551         while (next_policy->is_hierarchy) {
 552                 next_fm = mlx5_flow_meter_find(priv,
 553                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 554                 if (!next_fm || next_fm->def_policy)
 555                         return NULL;
 556                 next_policy = mlx5_flow_meter_policy_find(dev,
 557                                                 next_fm->policy_id, NULL);
 558                 MLX5_ASSERT(next_policy);
 559         }
 560         return next_policy;
 561 }
 562
 563 /**
 564  * Callback to check MTR policy action validate
 565  *
 566  * @param[in] dev
 567  *   Pointer to Ethernet device.
 568  * @param[in] actions
 569  *   Pointer to meter policy action detail.
 570  * @param[out] error
 571  *   Pointer to the error structure.
 572  *
 573  * @return
 574  *   0 on success, a negative errno value otherwise and rte_errno is set.
 575  */
 576 static int
 577 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 578         struct rte_mtr_meter_policy_params *policy,
 579         struct rte_mtr_error *error)
 580 {
 581         struct mlx5_priv *priv = dev->data->dev_private;
 582         struct rte_flow_attr attr = { .transfer =
 583                         priv->config.dv_esw_en ? 1 : 0};
 584         bool is_rss = false;
 585         bool is_def_policy = false;
 586         uint8_t domain_bitmap;
 587         int ret;
 588
 589         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 590                 return -rte_mtr_error_set(error, ENOTSUP,
 591                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 592                                 NULL, "meter policy unsupported.");
 593         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 594                         &is_rss, &domain_bitmap, &is_def_policy, error);
 595         if (ret)
 596                 return ret;
 597         return 0;
 598 }
 599
 600 static int
 601 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 602                         uint32_t policy_id,
 603                         struct mlx5_flow_meter_policy *mtr_policy,
 604                         struct rte_mtr_error *error,
 605                         bool clear_l3t)
 606 {
 607         struct mlx5_priv *priv = dev->data->dev_private;
 608         struct mlx5_flow_meter_sub_policy *sub_policy;
 609         uint32_t i, j;
 610         uint16_t sub_policy_num;
 611
 612         rte_spinlock_lock(&mtr_policy->sl);
 613         if (mtr_policy->ref_cnt) {
 614                 rte_spinlock_unlock(&mtr_policy->sl);
 615                 return -rte_mtr_error_set(error, EBUSY,
 616                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 617                                  NULL,
 618                                 "Meter policy object is being used.");
 619         }
 620         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 621         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 622         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 623                 sub_policy_num = (mtr_policy->sub_policy_num >>
 624                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 625                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 626                 if (sub_policy_num) {
 627                         for (j = 0; j < sub_policy_num; j++) {
 628                                 sub_policy = mtr_policy->sub_policys[i][j];
 629                                 if (sub_policy)
 630                                         mlx5_ipool_free
 631                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 632                                         sub_policy->idx);
 633                         }
 634                 }
 635         }
 636         if (priv->policy_idx_tbl && clear_l3t) {
 637                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 638                         rte_spinlock_unlock(&mtr_policy->sl);
 639                         return -rte_mtr_error_set(error, ENOTSUP,
 640                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 641                                 "Fail to delete policy in index table.");
 642                 }
 643         }
 644         rte_spinlock_unlock(&mtr_policy->sl);
 645         return 0;
 646 }
 647
 648 /**
 649  * Callback to add MTR policy.
 650  *
 651  * @param[in] dev
 652  *   Pointer to Ethernet device.
 653  * @param[out] policy_id
 654  *   Pointer to policy id
 655  * @param[in] actions
 656  *   Pointer to meter policy action detail.
 657  * @param[out] error
 658  *   Pointer to the error structure.
 659  *
 660  * @return
 661  *   0 on success, a negative errno value otherwise and rte_errno is set.
 662  */
 663 static int
 664 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 665                         uint32_t policy_id,
 666                         struct rte_mtr_meter_policy_params *policy,
 667                         struct rte_mtr_error *error)
 668 {
 669         struct mlx5_priv *priv = dev->data->dev_private;
 670         struct rte_flow_attr attr = { .transfer =
 671                         priv->config.dv_esw_en ? 1 : 0};
 672         uint32_t sub_policy_idx = 0;
 673         uint32_t policy_idx = 0;
 674         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 675         struct mlx5_flow_meter_sub_policy *sub_policy;
 676         bool is_rss = false;
 677         bool is_def_policy = false;
 678         uint32_t i;
 679         int ret;
 680         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 681         uint16_t sub_policy_num;
 682         uint8_t domain_bitmap = 0;
 683         union mlx5_l3t_data data;
 684         bool skip_rule = false;
 685
 686         if (!priv->mtr_en)
 687                 return -rte_mtr_error_set(error, ENOTSUP,
 688                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 689                                           NULL, "meter policy unsupported. ");
 690         if (policy_id == MLX5_INVALID_POLICY_ID)
 691                 return -rte_mtr_error_set(error, ENOTSUP,
 692                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 693                                           NULL, "policy ID is invalid. ");
 694         if (policy_id == priv->sh->mtrmng->def_policy_id)
 695                 return -rte_mtr_error_set(error, EEXIST,
 696                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 697                                           NULL, "default policy ID exists. ");
 698         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 699         if (mtr_policy)
 700                 return -rte_mtr_error_set(error, EEXIST,
 701                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 702                                           NULL, "policy ID exists. ");
 703         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 704                         &is_rss, &domain_bitmap, &is_def_policy, error);
 705         if (ret)
 706                 return ret;
 707         if (!domain_bitmap)
 708                 return -rte_mtr_error_set(error, ENOTSUP,
 709                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 710                                 NULL, "fail to find policy domain.");
 711         if (is_def_policy) {
 712                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 713                         return -rte_mtr_error_set(error, EEXIST,
 714                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 715                                 NULL, "a policy with similar actions "
 716                                 "is already configured");
 717                 if (mlx5_flow_create_def_policy(dev))
 718                         return -rte_mtr_error_set(error, ENOTSUP,
 719                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 720                                 NULL,
 721                                 "fail to create non-terminated policy.");
 722                 priv->sh->mtrmng->def_policy_id = policy_id;
 723                 return 0;
 724         }
 725         if (!priv->sh->meter_aso_en)
 726                 return -rte_mtr_error_set(error, ENOTSUP,
 727                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 728                         "no ASO capability to support the policy ");
 729         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 730                 if (!(domain_bitmap & (1 << i)))
 731                         continue;
 732                 /*
 733                  * If RSS is found, it means that only the ingress domain can
 734                  * be supported. It is invalid to support RSS for one color
 735                  * and egress / transfer domain actions for another. Drop and
 736                  * jump action should have no impact.
 737                  */
 738                 if (is_rss) {
 739                         policy_size +=
 740                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 741                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 742                         break;
 743                 }
 744                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 745         }
 746         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 747                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 748         if (!mtr_policy)
 749                 return -rte_mtr_error_set(error, ENOMEM,
 750                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 751                                 "Memory alloc failed for meter policy.");
 752         policy_size = sizeof(struct mlx5_flow_meter_policy);
 753         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 754                 if (!(domain_bitmap & (1 << i)))
 755                         continue;
 756                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 757                         mtr_policy->ingress = 1;
 758                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 759                         mtr_policy->egress = 1;
 760                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 761                         mtr_policy->transfer = 1;
 762                 sub_policy = mlx5_ipool_zmalloc
 763                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 764                                  &sub_policy_idx);
 765                 if (!sub_policy ||
 766                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 767                         goto policy_add_err;
 768                 sub_policy->idx = sub_policy_idx;
 769                 sub_policy->main_policy = mtr_policy;
 770                 if (!policy_idx) {
 771                         policy_idx = sub_policy_idx;
 772                         sub_policy->main_policy_id = 1;
 773                 }
 774                 mtr_policy->sub_policys[i] =
 775                         (struct mlx5_flow_meter_sub_policy **)
 776                         ((uint8_t *)mtr_policy + policy_size);
 777                 mtr_policy->sub_policys[i][0] = sub_policy;
 778                 sub_policy_num = (mtr_policy->sub_policy_num >>
 779                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 780                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 781                 sub_policy_num++;
 782                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 783                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 784                 mtr_policy->sub_policy_num |=
 785                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 786                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 787                 /*
 788                  * If RSS is found, it means that only the ingress domain can
 789                  * be supported. It is invalid to support RSS for one color
 790                  * and egress / transfer domain actions for another. Drop and
 791                  * jump action should have no impact.
 792                  */
 793                 if (is_rss) {
 794                         mtr_policy->is_rss = 1;
 795                         break;
 796                 }
 797                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 798         }
 799         rte_spinlock_init(&mtr_policy->sl);
 800         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 801                                         policy->actions, error);
 802         if (ret)
 803                 goto policy_add_err;
 804         if (mtr_policy->is_hierarchy) {
 805                 struct mlx5_flow_meter_policy *final_policy;
 806
 807                 final_policy =
 808                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 809                 if (!final_policy)
 810                         goto policy_add_err;
 811                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 812         }
 813         /*
 814          * If either Green or Yellow has queue / RSS action, all the policy
 815          * rules will be created later in the flow splitting stage.
 816          */
 817         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 818                 /* Create policy rules in HW. */
 819                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 820                 if (ret)
 821                         goto policy_add_err;
 822         }
 823         data.dword = policy_idx;
 824         if (!priv->policy_idx_tbl) {
 825                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 826                 if (!priv->policy_idx_tbl)
 827                         goto policy_add_err;
 828         }
 829         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 830                 goto policy_add_err;
 831         return 0;
 832 policy_add_err:
 833         if (mtr_policy) {
 834                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 835                         mtr_policy, error, false);
 836                 mlx5_free(mtr_policy);
 837                 if (ret)
 838                         return ret;
 839         }
 840         return -rte_mtr_error_set(error, ENOTSUP,
 841                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 842                                   NULL, "Failed to create devx policy.");
 843 }
 844
 845 /**
 846  * Callback to delete MTR policy.
 847  *
 848  * @param[in] dev
 849  *   Pointer to Ethernet device.
 850  * @param[in] policy_id
 851  *   Meter policy id.
 852  * @param[out] error
 853  *   Pointer to the error structure.
 854  *
 855  * @return
 856  *   0 on success, a negative errno value otherwise and rte_errno is set.
 857  */
 858 static int
 859 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 860                           uint32_t policy_id,
 861                           struct rte_mtr_error *error)
 862 {
 863         struct mlx5_priv *priv = dev->data->dev_private;
 864         struct mlx5_flow_meter_policy *mtr_policy;
 865         uint32_t policy_idx;
 866         int ret;
 867
 868         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 869                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 870                         return -rte_mtr_error_set(error, ENOTSUP,
 871                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 872                                 "Meter policy object is being used.");
 873                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 874                 return 0;
 875         }
 876         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 877         if (!mtr_policy)
 878                 return -rte_mtr_error_set(error, ENOTSUP,
 879                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 880                         "Meter policy id is invalid. ");
 881         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 882                                                 error, true);
 883         if (ret)
 884                 return ret;
 885         mlx5_free(mtr_policy);
 886         return 0;
 887 }
 888
 889 /**
 890  * Check meter validation.
 891  *
 892  * @param[in] priv
 893  *   Pointer to mlx5 private data structure.
 894  * @param[in] meter_id
 895  *   Meter id.
 896  * @param[in] params
 897  *   Pointer to rte meter parameters.
 898  * @param[out] error
 899  *   Pointer to rte meter error structure.
 900  *
 901  * @return
 902  *   0 on success, a negative errno value otherwise and rte_errno is set.
 903  */
 904 static int
 905 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 906                          struct rte_mtr_params *params,
 907                          struct rte_mtr_error *error)
 908 {
 909         /* Meter must use global drop action. */
 910         if (!priv->sh->dr_drop_action)
 911                 return -rte_mtr_error_set(error, ENOTSUP,
 912                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 913                                           NULL,
 914                                           "No drop action ready for meter.");
 915         /* Meter params must not be NULL. */
 916         if (params == NULL)
 917                 return -rte_mtr_error_set(error, EINVAL,
 918                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 919                                           NULL, "Meter object params null.");
 920         /* Previous meter color is not supported. */
 921         if (params->use_prev_mtr_color)
 922                 return -rte_mtr_error_set(error, ENOTSUP,
 923                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 924                                           NULL,
 925                                           "Previous meter color "
 926                                           "not supported.");
 927         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 928                 return -rte_mtr_error_set(error, ENOENT,
 929                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 930                                 NULL, "Meter policy id not valid.");
 931         /* Validate meter id. */
 932         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 933                 return -rte_mtr_error_set(error, EEXIST,
 934                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 935                         "Meter object already exists.");
 936         return 0;
 937 }
 938
 939 /**
 940  * Modify the flow meter action.
 941  *
 942  * @param[in] priv
 943  *   Pointer to mlx5 private data structure.
 944  * @param[in] fm
 945  *   Pointer to flow meter to be modified.
 946  * @param[in] srtcm
 947  *   Pointer to meter srtcm description parameter.
 948  * @param[in] modify_bits
 949  *   The bit in srtcm to be updated.
 950  * @param[in] active_state
 951  *   The state to be updated.
 952  * @return
 953  *   0 on success, o negative value otherwise.
 954  */
 955 static int
 956 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 957                 struct mlx5_flow_meter_info *fm,
 958                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 959                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 960 {
 961 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 962         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 963         uint32_t *attr;
 964         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 965         int ret;
 966         struct mlx5_aso_mtr *aso_mtr = NULL;
 967         uint32_t cbs_cir, ebs_eir, val;
 968
 969         if (priv->sh->meter_aso_en) {
 970                 fm->is_enable = !!is_enable;
 971                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 972                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 973                 if (ret)
 974                         return ret;
 975                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 976                 if (ret)
 977                         return ret;
 978         } else {
 979                 /* Fill command parameters. */
 980                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 981                 mod_attr.flow_meter_parameter = in;
 982                 mod_attr.flow_meter_parameter_sz =
 983                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 984                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 985                         mod_attr.active = !!active_state;
 986                 else
 987                         mod_attr.active = 0;
 988                 attr = in;
 989                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 990                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 991                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 992                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 993                                 ASO_DSEG_EXP_MASK;
 994                         MLX5_SET(flow_meter_parameters, attr,
 995                                 cbs_exponent, val);
 996                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 997                                 ASO_DSEG_MAN_MASK;
 998                         MLX5_SET(flow_meter_parameters, attr,
 999                                 cbs_mantissa, val);
1000                 }
1001                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1002                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
1003                                 ASO_DSEG_EXP_MASK;
1004                         MLX5_SET(flow_meter_parameters, attr,
1005                                 cir_exponent, val);
1006                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1007                         MLX5_SET(flow_meter_parameters, attr,
1008                                 cir_mantissa, val);
1009                 }
1010                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1011                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1012                                 ASO_DSEG_EXP_MASK;
1013                         MLX5_SET(flow_meter_parameters, attr,
1014                                 ebs_exponent, val);
1015                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1016                                 ASO_DSEG_MAN_MASK;
1017                         MLX5_SET(flow_meter_parameters, attr,
1018                                 ebs_mantissa, val);
1019                 }
1020                 /* Apply modifications to meter only if it was created. */
1021                 if (fm->meter_action) {
1022                         ret = mlx5_glue->dv_modify_flow_action_meter
1023                                         (fm->meter_action, &mod_attr,
1024                                         rte_cpu_to_be_64(modify_bits));
1025                         if (ret)
1026                                 return ret;
1027                 }
1028                 /* Update succeedded modify meter parameters. */
1029                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1030                         fm->active_state = !!active_state;
1031         }
1032         return 0;
1033 #else
1034         (void)priv;
1035         (void)fm;
1036         (void)srtcm;
1037         (void)modify_bits;
1038         (void)active_state;
1039         (void)is_enable;
1040         return -ENOTSUP;
1041 #endif
1042 }
1043
1044 static int
1045 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1046                                 struct mlx5_flow_meter_info *fm,
1047                                 uint64_t stats_mask)
1048 {
1049         fm->bytes_dropped =
1050                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1051         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1052         if (fm->bytes_dropped || fm->pkts_dropped) {
1053                 if (!fm->drop_cnt) {
1054                         /* Alloc policer counters. */
1055                         fm->drop_cnt = mlx5_counter_alloc(dev);
1056                         if (!fm->drop_cnt)
1057                                 return -1;
1058                 }
1059         } else {
1060                 if (fm->drop_cnt) {
1061                         mlx5_counter_free(dev, fm->drop_cnt);
1062                         fm->drop_cnt = 0;
1063                 }
1064         }
1065         return 0;
1066 }
1067
1068 /**
1069  * Create meter rules.
1070  *
1071  * @param[in] dev
1072  *   Pointer to Ethernet device.
1073  * @param[in] meter_id
1074  *   Meter id.
1075  * @param[in] params
1076  *   Pointer to rte meter parameters.
1077  * @param[in] shared
1078  *   Meter shared with other flow or not.
1079  * @param[out] error
1080  *   Pointer to rte meter error structure.
1081  *
1082  * @return
1083  *   0 on success, a negative errno value otherwise and rte_errno is set.
1084  */
1085 static int
1086 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1087                        struct rte_mtr_params *params, int shared,
1088                        struct rte_mtr_error *error)
1089 {
1090         struct mlx5_priv *priv = dev->data->dev_private;
1091         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1092         struct mlx5_flow_meter_profile *fmp;
1093         struct mlx5_flow_meter_info *fm;
1094         struct mlx5_legacy_flow_meter *legacy_fm;
1095         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1096         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1097                 .size = 0,
1098                 .trunk_size = 64,
1099                 .need_lock = 1,
1100                 .type = "mlx5_flow_mtr_flow_id_pool",
1101         };
1102         struct mlx5_aso_mtr *aso_mtr;
1103         uint32_t mtr_idx, policy_idx;
1104         union mlx5_l3t_data data;
1105         int ret;
1106         uint8_t domain_bitmap;
1107         uint8_t mtr_id_bits;
1108         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1109                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1110
1111         if (!priv->mtr_en)
1112                 return -rte_mtr_error_set(error, ENOTSUP,
1113                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1114                                         "Meter is not supported");
1115         /* Validate the parameters. */
1116         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1117         if (ret)
1118                 return ret;
1119         /* Meter profile must exist. */
1120         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1121         if (fmp == NULL)
1122                 return -rte_mtr_error_set(error, ENOENT,
1123                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1124                         NULL, "Meter profile id not valid.");
1125         /* Meter policy must exist. */
1126         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1127                 __atomic_add_fetch
1128                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1129                         1, __ATOMIC_RELAXED);
1130                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1131                 if (!priv->config.dv_esw_en)
1132                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1133         } else {
1134                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1135                                 params->meter_policy_id, &policy_idx);
1136                 if (!priv->sh->meter_aso_en)
1137                         return -rte_mtr_error_set(error, ENOTSUP,
1138                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1139                                 "Part of the policies cannot be "
1140                                 "supported without ASO ");
1141                 if (!mtr_policy)
1142                         return -rte_mtr_error_set(error, ENOENT,
1143                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1144                                 NULL, "Meter policy id not valid.");
1145                 domain_bitmap = (mtr_policy->ingress ?
1146                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1147                                 (mtr_policy->egress ?
1148                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1149                                 (mtr_policy->transfer ?
1150                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1151         }
1152         /* Allocate the flow meter memory. */
1153         if (priv->sh->meter_aso_en) {
1154                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1155                 if (!mtr_idx)
1156                         return -rte_mtr_error_set(error, ENOMEM,
1157                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1158                                 "Memory alloc failed for meter.");
1159                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1160                 fm = &aso_mtr->fm;
1161         } else {
1162                 legacy_fm = mlx5_ipool_zmalloc
1163                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1164                 if (legacy_fm == NULL)
1165                         return -rte_mtr_error_set(error, ENOMEM,
1166                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1167                                 "Memory alloc failed for meter.");
1168                 legacy_fm->idx = mtr_idx;
1169                 fm = &legacy_fm->fm;
1170         }
1171         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1172         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1173             mtr_reg_bits) {
1174                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1175                 goto error;
1176         }
1177         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1178                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1179         /* Fill the flow meter parameters. */
1180         fm->meter_id = meter_id;
1181         fm->policy_id = params->meter_policy_id;
1182         fm->profile = fmp;
1183         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1184                 goto error;
1185         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1186                 goto error;
1187         /* Add to the flow meter list. */
1188         if (!priv->sh->meter_aso_en)
1189                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1190         /* Add to the flow meter list. */
1191         fm->active_state = 1; /* Config meter starts as active. */
1192         fm->is_enable = 1;
1193         fm->shared = !!shared;
1194         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1195         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1196                 fm->def_policy = 1;
1197                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1198                 if (!fm->flow_ipool)
1199                         goto error;
1200         }
1201         rte_spinlock_init(&fm->sl);
1202         /* If ASO meter supported, update ASO flow meter by wqe. */
1203         if (priv->sh->meter_aso_en) {
1204                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1205                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1206                 if (ret)
1207                         goto error;
1208                 if (!priv->mtr_idx_tbl) {
1209                         priv->mtr_idx_tbl =
1210                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1211                         if (!priv->mtr_idx_tbl)
1212                                 goto error;
1213                 }
1214                 data.dword = mtr_idx;
1215                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1216                         goto error;
1217         }
1218         if (mtr_policy)
1219                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1220         return 0;
1221 error:
1222         mlx5_flow_destroy_mtr_tbls(dev, fm);
1223         /* Free policer counters. */
1224         if (fm->drop_cnt)
1225                 mlx5_counter_free(dev, fm->drop_cnt);
1226         if (priv->sh->meter_aso_en)
1227                 mlx5_flow_mtr_free(dev, mtr_idx);
1228         else
1229                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1230         return -rte_mtr_error_set(error, ENOTSUP,
1231                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1232                 NULL, "Failed to create devx meter.");
1233 }
1234
1235 static int
1236 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1237                         struct mlx5_flow_meter_info *fm,
1238                         uint32_t mtr_idx)
1239 {
1240         struct mlx5_priv *priv = dev->data->dev_private;
1241         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1242         struct mlx5_flow_meter_profile *fmp;
1243         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1244         struct mlx5_flow_meter_policy *mtr_policy;
1245
1246         /* Meter object must not have any owner. */
1247         MLX5_ASSERT(!fm->ref_cnt);
1248         /* Get meter profile. */
1249         fmp = fm->profile;
1250         if (fmp == NULL)
1251                 return -1;
1252         /* Update dependencies. */
1253         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1254         fm->profile = NULL;
1255         /* Remove from list. */
1256         if (!priv->sh->meter_aso_en) {
1257                 legacy_fm = container_of(fm,
1258                         struct mlx5_legacy_flow_meter, fm);
1259                 TAILQ_REMOVE(fms, legacy_fm, next);
1260         }
1261         /* Free drop counters. */
1262         if (fm->drop_cnt)
1263                 mlx5_counter_free(dev, fm->drop_cnt);
1264         /* Free meter flow table. */
1265         if (fm->flow_ipool) {
1266                 mlx5_ipool_destroy(fm->flow_ipool);
1267                 fm->flow_ipool = 0;
1268         }
1269         mlx5_flow_destroy_mtr_tbls(dev, fm);
1270         if (fm->def_policy)
1271                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1272                                 1, __ATOMIC_RELAXED);
1273         if (priv->sh->meter_aso_en) {
1274                 if (!fm->def_policy) {
1275                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1276                                                 fm->policy_id, NULL);
1277                         if (mtr_policy)
1278                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1279                                                 1, __ATOMIC_RELAXED);
1280                         fm->policy_id = 0;
1281                 }
1282                 fm->def_policy = 0;
1283                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1284                         return -1;
1285                 mlx5_flow_mtr_free(dev, mtr_idx);
1286         } else {
1287                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1288                                         legacy_fm->idx);
1289         }
1290         return 0;
1291 }
1292
1293 /**
1294  * Destroy meter rules.
1295  *
1296  * @param[in] dev
1297  *   Pointer to Ethernet device.
1298  * @param[in] meter_id
1299  *   Meter id.
1300  * @param[out] error
1301  *   Pointer to rte meter error structure.
1302  *
1303  * @return
1304  *   0 on success, a negative errno value otherwise and rte_errno is set.
1305  */
1306 static int
1307 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1308                         struct rte_mtr_error *error)
1309 {
1310         struct mlx5_priv *priv = dev->data->dev_private;
1311         struct mlx5_flow_meter_info *fm;
1312         uint32_t mtr_idx = 0;
1313
1314         if (!priv->mtr_en)
1315                 return -rte_mtr_error_set(error, ENOTSUP,
1316                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1317                                           NULL,
1318                                           "Meter is not supported");
1319         /* Meter object must exist. */
1320         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1321         if (fm == NULL)
1322                 return -rte_mtr_error_set(error, ENOENT,
1323                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1324                                           NULL,
1325                                           "Meter object id not valid.");
1326         /* Meter object must not have any owner. */
1327         if (fm->ref_cnt > 0)
1328                 return -rte_mtr_error_set(error, EBUSY,
1329                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1330                                           NULL,
1331                                           "Meter object is being used.");
1332         /* Destroy the meter profile. */
1333         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1334                 return -rte_mtr_error_set(error, EINVAL,
1335                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1336                                         NULL,
1337                                         "MTR object meter profile invalid.");
1338         return 0;
1339 }
1340
1341 /**
1342  * Modify meter state.
1343  *
1344  * @param[in] priv
1345  *   Pointer to mlx5 private data structure.
1346  * @param[in] fm
1347  *   Pointer to flow meter.
1348  * @param[in] new_state
1349  *   New state to update.
1350  * @param[out] error
1351  *   Pointer to rte meter error structure.
1352  *
1353  * @return
1354  *   0 on success, a negative errno value otherwise and rte_errno is set.
1355  */
1356 static int
1357 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1358                              struct mlx5_flow_meter_info *fm,
1359                              uint32_t new_state,
1360                              struct rte_mtr_error *error)
1361 {
1362         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1363                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1364                 .ebs_eir = 0,
1365         };
1366         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1367                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1368         int ret;
1369
1370         if (new_state == MLX5_FLOW_METER_DISABLE)
1371                 ret = mlx5_flow_meter_action_modify(priv, fm,
1372                                 &srtcm, modify_bits, 0, 0);
1373         else
1374                 ret = mlx5_flow_meter_action_modify(priv, fm,
1375                                                    &fm->profile->srtcm_prm,
1376                                                     modify_bits, 0, 1);
1377         if (ret)
1378                 return -rte_mtr_error_set(error, -ret,
1379                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1380                                           NULL,
1381                                           new_state ?
1382                                           "Failed to enable meter." :
1383                                           "Failed to disable meter.");
1384         return 0;
1385 }
1386
1387 /**
1388  * Callback to enable flow meter.
1389  *
1390  * @param[in] dev
1391  *   Pointer to Ethernet device.
1392  * @param[in] meter_id
1393  *   Meter id.
1394  * @param[out] error
1395  *   Pointer to rte meter error structure.
1396  *
1397  * @return
1398  *   0 on success, a negative errno value otherwise and rte_errno is set.
1399  */
1400 static int
1401 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1402                        uint32_t meter_id,
1403                        struct rte_mtr_error *error)
1404 {
1405         struct mlx5_priv *priv = dev->data->dev_private;
1406         struct mlx5_flow_meter_info *fm;
1407         int ret;
1408
1409         if (!priv->mtr_en)
1410                 return -rte_mtr_error_set(error, ENOTSUP,
1411                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1412                                           "Meter is not supported");
1413         /* Meter object must exist. */
1414         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1415         if (fm == NULL)
1416                 return -rte_mtr_error_set(error, ENOENT,
1417                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1418                                           NULL, "Meter not found.");
1419         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1420                 return 0;
1421         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1422                                            error);
1423         if (!ret)
1424                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1425         return ret;
1426 }
1427
1428 /**
1429  * Callback to disable flow meter.
1430  *
1431  * @param[in] dev
1432  *   Pointer to Ethernet device.
1433  * @param[in] meter_id
1434  *   Meter id.
1435  * @param[out] error
1436  *   Pointer to rte meter error structure.
1437  *
1438  * @return
1439  *   0 on success, a negative errno value otherwise and rte_errno is set.
1440  */
1441 static int
1442 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1443                         uint32_t meter_id,
1444                         struct rte_mtr_error *error)
1445 {
1446         struct mlx5_priv *priv = dev->data->dev_private;
1447         struct mlx5_flow_meter_info *fm;
1448         int ret;
1449
1450         if (!priv->mtr_en)
1451                 return -rte_mtr_error_set(error, ENOTSUP,
1452                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1453                                           "Meter is not supported");
1454         /* Meter object must exist. */
1455         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1456         if (fm == NULL)
1457                 return -rte_mtr_error_set(error, ENOENT,
1458                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1459                                           NULL, "Meter not found.");
1460         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1461                 return 0;
1462         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1463                                            error);
1464         if (!ret)
1465                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1466         return ret;
1467 }
1468
1469 /**
1470  * Callback to update meter profile.
1471  *
1472  * @param[in] dev
1473  *   Pointer to Ethernet device.
1474  * @param[in] meter_id
1475  *   Meter id.
1476  * @param[in] meter_profile_id
1477  *   To be updated meter profile id.
1478  * @param[out] error
1479  *   Pointer to rte meter error structure.
1480  *
1481  * @return
1482  *   0 on success, a negative errno value otherwise and rte_errno is set.
1483  */
1484 static int
1485 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1486                                uint32_t meter_id,
1487                                uint32_t meter_profile_id,
1488                                struct rte_mtr_error *error)
1489 {
1490         struct mlx5_priv *priv = dev->data->dev_private;
1491         struct mlx5_flow_meter_profile *fmp;
1492         struct mlx5_flow_meter_profile *old_fmp;
1493         struct mlx5_flow_meter_info *fm;
1494         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1495                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1496         int ret;
1497
1498         if (!priv->mtr_en)
1499                 return -rte_mtr_error_set(error, ENOTSUP,
1500                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1501                                           "Meter is not supported");
1502         /* Meter profile must exist. */
1503         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1504         if (fmp == NULL)
1505                 return -rte_mtr_error_set(error, ENOENT,
1506                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1507                                           NULL, "Meter profile not found.");
1508         /* Meter object must exist. */
1509         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1510         if (fm == NULL)
1511                 return -rte_mtr_error_set(error, ENOENT,
1512                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1513                                           NULL, "Meter not found.");
1514         /* MTR object already set to meter profile id. */
1515         old_fmp = fm->profile;
1516         if (fmp == old_fmp)
1517                 return 0;
1518         /* Update the profile. */
1519         fm->profile = fmp;
1520         /* Update meter params in HW (if not disabled). */
1521         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1522                 return 0;
1523         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1524                                               modify_bits, fm->active_state, 1);
1525         if (ret) {
1526                 fm->profile = old_fmp;
1527                 return -rte_mtr_error_set(error, -ret,
1528                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1529                                           NULL, "Failed to update meter"
1530                                           " parmeters in hardware.");
1531         }
1532         old_fmp->ref_cnt--;
1533         fmp->ref_cnt++;
1534         return 0;
1535 }
1536
1537 /**
1538  * Callback to update meter stats mask.
1539  *
1540  * @param[in] dev
1541  *   Pointer to Ethernet device.
1542  * @param[in] meter_id
1543  *   Meter id.
1544  * @param[in] stats_mask
1545  *   To be updated stats_mask.
1546  * @param[out] error
1547  *   Pointer to rte meter error structure.
1548  *
1549  * @return
1550  *   0 on success, a negative errno value otherwise and rte_errno is set.
1551  */
1552 static int
1553 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1554                              uint32_t meter_id,
1555                              uint64_t stats_mask,
1556                              struct rte_mtr_error *error)
1557 {
1558         struct mlx5_priv *priv = dev->data->dev_private;
1559         struct mlx5_flow_meter_info *fm;
1560
1561         if (!priv->mtr_en)
1562                 return -rte_mtr_error_set(error, ENOTSUP,
1563                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1564                                           "Meter is not supported");
1565         /* Meter object must exist. */
1566         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1567         if (fm == NULL)
1568                 return -rte_mtr_error_set(error, ENOENT,
1569                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1570                                           NULL, "Meter object id not valid.");
1571         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1572                 return -rte_mtr_error_set(error, ENOENT,
1573                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1574                                           NULL, "Fail to allocate "
1575                                           "counter for meter.");
1576         return 0;
1577 }
1578
1579 /**
1580  * Callback to read meter statistics.
1581  *
1582  * @param[in] dev
1583  *   Pointer to Ethernet device.
1584  * @param[in] meter_id
1585  *   Meter id.
1586  * @param[out] stats
1587  *   Pointer to store the statistics.
1588  * @param[out] stats_mask
1589  *   Pointer to store the stats_mask.
1590  * @param[in] clear
1591  *   Statistic to be cleared after read or not.
1592  * @param[out] error
1593  *   Pointer to rte meter error structure.
1594  *
1595  * @return
1596  *   0 on success, a negative errno value otherwise and rte_errno is set.
1597  */
1598 static int
1599 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1600                            uint32_t meter_id,
1601                            struct rte_mtr_stats *stats,
1602                            uint64_t *stats_mask,
1603                            int clear,
1604                            struct rte_mtr_error *error)
1605 {
1606         struct mlx5_priv *priv = dev->data->dev_private;
1607         struct mlx5_flow_meter_info *fm;
1608         uint64_t pkts;
1609         uint64_t bytes;
1610         int ret = 0;
1611
1612         if (!priv->mtr_en)
1613                 return -rte_mtr_error_set(error, ENOTSUP,
1614                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1615                                           "Meter is not supported");
1616         /* Meter object must exist. */
1617         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1618         if (fm == NULL)
1619                 return -rte_mtr_error_set(error, ENOENT,
1620                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1621                                           NULL, "Meter object id not valid.");
1622         *stats_mask = 0;
1623         if (fm->bytes_dropped)
1624                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1625         if (fm->pkts_dropped)
1626                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1627         memset(stats, 0, sizeof(*stats));
1628         if (fm->drop_cnt) {
1629                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1630                                                  &bytes);
1631                 if (ret)
1632                         goto error;
1633                 /* If need to read the packets, set it. */
1634                 if (fm->pkts_dropped)
1635                         stats->n_pkts_dropped = pkts;
1636                 /* If need to read the bytes, set it. */
1637                 if (fm->bytes_dropped)
1638                         stats->n_bytes_dropped = bytes;
1639         }
1640         return 0;
1641 error:
1642         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1643                                  "Failed to read meter drop counters.");
1644 }
1645
1646 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1647         .capabilities_get = mlx5_flow_mtr_cap_get,
1648         .meter_profile_add = mlx5_flow_meter_profile_add,
1649         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1650         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1651         .meter_policy_add = mlx5_flow_meter_policy_add,
1652         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1653         .create = mlx5_flow_meter_create,
1654         .destroy = mlx5_flow_meter_destroy,
1655         .meter_enable = mlx5_flow_meter_enable,
1656         .meter_disable = mlx5_flow_meter_disable,
1657         .meter_profile_update = mlx5_flow_meter_profile_update,
1658         .meter_dscp_table_update = NULL,
1659         .stats_update = mlx5_flow_meter_stats_update,
1660         .stats_read = mlx5_flow_meter_stats_read,
1661 };
1662
1663 /**
1664  * Get meter operations.
1665  *
1666  * @param dev
1667  *   Pointer to Ethernet device structure.
1668  * @param arg
1669  *   Pointer to set the mtr operations.
1670  *
1671  * @return
1672  *   Always 0.
1673  */
1674 int
1675 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1676 {
1677         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1678         return 0;
1679 }
1680
1681 /**
1682  * Find meter by id.
1683  *
1684  * @param priv
1685  *   Pointer to mlx5_priv.
1686  * @param meter_id
1687  *   Meter id.
1688  * @param mtr_idx
1689  *   Pointer to Meter index.
1690  *
1691  * @return
1692  *   Pointer to the meter info found on success, NULL otherwise.
1693  */
1694 struct mlx5_flow_meter_info *
1695 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1696                 uint32_t *mtr_idx)
1697 {
1698         struct mlx5_legacy_flow_meter *legacy_fm;
1699         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1700         struct mlx5_aso_mtr *aso_mtr;
1701         struct mlx5_aso_mtr_pools_mng *pools_mng =
1702                                 &priv->sh->mtrmng->pools_mng;
1703         union mlx5_l3t_data data;
1704
1705         if (priv->sh->meter_aso_en) {
1706                 rte_spinlock_lock(&pools_mng->mtrsl);
1707                 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1708                         rte_spinlock_unlock(&pools_mng->mtrsl);
1709                         return NULL;
1710                 }
1711                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1712                         !data.dword) {
1713                         rte_spinlock_unlock(&pools_mng->mtrsl);
1714                         return NULL;
1715                 }
1716                 if (mtr_idx)
1717                         *mtr_idx = data.dword;
1718                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1719                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1720                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1721                 rte_spinlock_unlock(&pools_mng->mtrsl);
1722                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1723                         return NULL;
1724                 return &aso_mtr->fm;
1725         }
1726         TAILQ_FOREACH(legacy_fm, fms, next)
1727                 if (meter_id == legacy_fm->fm.meter_id) {
1728                         if (mtr_idx)
1729                                 *mtr_idx = legacy_fm->idx;
1730                         return &legacy_fm->fm;
1731                 }
1732         return NULL;
1733 }
1734
1735 /**
1736  * Find meter by index.
1737  *
1738  * @param priv
1739  *   Pointer to mlx5_priv.
1740  * @param idx
1741  *   Meter index.
1742  *
1743  * @return
1744  *   Pointer to the meter info found on success, NULL otherwise.
1745  */
1746 struct mlx5_flow_meter_info *
1747 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1748 {
1749         struct mlx5_aso_mtr *aso_mtr;
1750
1751         if (priv->sh->meter_aso_en) {
1752                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1753                 if (!aso_mtr)
1754                         return NULL;
1755                 return &aso_mtr->fm;
1756         } else {
1757                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1758         }
1759 }
1760
1761 /**
1762  * Attach meter to flow.
1763  * Unidirectional Meter creation can only be done
1764  * when flow direction is known, i.e. when calling meter_attach.
1765  *
1766  * @param [in] priv
1767  *  Pointer to mlx5 private data.
1768  * @param[in] fm
1769  *   Pointer to flow meter.
1770  * @param [in] attr
1771  *  Pointer to flow attributes.
1772  * @param [out] error
1773  *  Pointer to error structure.
1774  *
1775  * @return
1776  *   0 on success, a negative errno value otherwise and rte_errno is set.
1777  */
1778 int
1779 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1780                        struct mlx5_flow_meter_info *fm,
1781                        const struct rte_flow_attr *attr,
1782                        struct rte_flow_error *error)
1783 {
1784         int ret = 0;
1785
1786         if (priv->sh->meter_aso_en) {
1787                 struct mlx5_aso_mtr *aso_mtr;
1788
1789                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1790                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1791                         return rte_flow_error_set(error, ENOENT,
1792                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1793                                         NULL,
1794                                         "Timeout in meter configuration");
1795                 }
1796                 rte_spinlock_lock(&fm->sl);
1797                 if (fm->shared || !fm->ref_cnt) {
1798                         fm->ref_cnt++;
1799                 } else {
1800                         rte_flow_error_set(error, EINVAL,
1801                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1802                                    "Meter cannot be shared");
1803                         ret = -1;
1804                 }
1805                 rte_spinlock_unlock(&fm->sl);
1806         } else {
1807                 rte_spinlock_lock(&fm->sl);
1808                 if (fm->meter_action) {
1809                         if (fm->shared &&
1810                             attr->transfer == fm->transfer &&
1811                             attr->ingress == fm->ingress &&
1812                             attr->egress == fm->egress) {
1813                                 fm->ref_cnt++;
1814                         } else {
1815                                 rte_flow_error_set(error, EINVAL,
1816                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1817                                         fm->shared ?
1818                                         "Meter attr not match." :
1819                                         "Meter cannot be shared.");
1820                                 ret = -1;
1821                         }
1822                 } else {
1823                         fm->ingress = attr->ingress;
1824                         fm->egress = attr->egress;
1825                         fm->transfer = attr->transfer;
1826                         fm->ref_cnt = 1;
1827                         /* This also creates the meter object. */
1828                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1829                                                                          fm);
1830                         if (!fm->meter_action) {
1831                                 fm->ref_cnt = 0;
1832                                 fm->ingress = 0;
1833                                 fm->egress = 0;
1834                                 fm->transfer = 0;
1835                                 rte_flow_error_set(error, EINVAL,
1836                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1837                                         "Meter action create failed.");
1838                                 ret = -1;
1839                         }
1840                 }
1841                 rte_spinlock_unlock(&fm->sl);
1842         }
1843         return ret ? -rte_errno : 0;
1844 }
1845
1846 /**
1847  * Detach meter from flow.
1848  *
1849  * @param [in] priv
1850  *  Pointer to mlx5 private data.
1851  * @param [in] fm
1852  *  Pointer to flow meter.
1853  */
1854 void
1855 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1856                        struct mlx5_flow_meter_info *fm)
1857 {
1858 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1859         rte_spinlock_lock(&fm->sl);
1860         MLX5_ASSERT(fm->ref_cnt);
1861         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1862                 mlx5_glue->destroy_flow_action(fm->meter_action);
1863                 fm->meter_action = NULL;
1864                 fm->ingress = 0;
1865                 fm->egress = 0;
1866                 fm->transfer = 0;
1867         }
1868         rte_spinlock_unlock(&fm->sl);
1869 #else
1870         (void)priv;
1871         (void)fm;
1872 #endif
1873 }
1874
1875 /**
1876  * Flush meter with Rx queue configuration.
1877  *
1878  * @param[in] dev
1879  *   Pointer to Ethernet device.
1880  */
1881 void
1882 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1883 {
1884         struct mlx5_priv *priv = dev->data->dev_private;
1885         struct mlx5_flow_meter_sub_policy *sub_policy;
1886         struct mlx5_flow_meter_policy *mtr_policy;
1887         void *entry;
1888         uint32_t i, policy_idx;
1889
1890         if (!priv->mtr_en)
1891                 return;
1892         if (priv->policy_idx_tbl) {
1893                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1894                         policy_idx = *(uint32_t *)entry;
1895                         sub_policy = mlx5_ipool_get
1896                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1897                                 policy_idx);
1898                         if (!sub_policy || !sub_policy->main_policy)
1899                                 continue;
1900                         mtr_policy = sub_policy->main_policy;
1901                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1902                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1903                                         mtr_policy);
1904                 }
1905         }
1906 }
1907
1908 /**
1909  * Iterate a meter hierarchy and flush all meters and policies if possible.
1910  *
1911  * @param[in] dev
1912  *   Pointer to Ethernet device.
1913  * @param[in] fm
1914  *   Pointer to flow meter.
1915  * @param[in] mtr_idx
1916  *   .Meter's index
1917  * @param[out] error
1918  *   Pointer to rte meter error structure.
1919  *
1920  * @return
1921  *   0 on success, a negative errno value otherwise and rte_errno is set.
1922  */
1923 static int
1924 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
1925                                 struct mlx5_flow_meter_info *fm,
1926                                 uint32_t mtr_idx,
1927                                 struct rte_mtr_error *error)
1928 {
1929         struct mlx5_priv *priv = dev->data->dev_private;
1930         struct mlx5_flow_meter_policy *policy;
1931         uint32_t policy_id;
1932         struct mlx5_flow_meter_info *next_fm;
1933         uint32_t next_mtr_idx;
1934         struct mlx5_flow_meter_policy *next_policy = NULL;
1935
1936         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
1937         MLX5_ASSERT(policy);
1938         while (!fm->ref_cnt && policy->is_hierarchy) {
1939                 policy_id = fm->policy_id;
1940                 next_fm = mlx5_flow_meter_find(priv,
1941                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
1942                                 &next_mtr_idx);
1943                 if (next_fm) {
1944                         next_policy = mlx5_flow_meter_policy_find(dev,
1945                                                         next_fm->policy_id,
1946                                                         NULL);
1947                         MLX5_ASSERT(next_policy);
1948                 }
1949                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1950                         return -rte_mtr_error_set(error, ENOTSUP,
1951                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
1952                                                 NULL,
1953                                                 "Failed to flush meter.");
1954                 if (policy->ref_cnt)
1955                         break;
1956                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
1957                                                 policy, error, true))
1958                         return -rte_errno;
1959                 mlx5_free(policy);
1960                 if (!next_fm || !next_policy)
1961                         break;
1962                 fm = next_fm;
1963                 mtr_idx = next_mtr_idx;
1964                 policy = next_policy;
1965         }
1966         return 0;
1967 }
1968
1969 /**
1970  * Flush all the hierarchy meters and their policies.
1971  *
1972  * @param[in] dev
1973  *   Pointer to Ethernet device.
1974  * @param[out] error
1975  *   Pointer to rte meter error structure.
1976  *
1977  * @return
1978  *   0 on success, a negative errno value otherwise and rte_errno is set.
1979  */
1980 static int
1981 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
1982                                       struct rte_mtr_error *error)
1983 {
1984         struct mlx5_priv *priv = dev->data->dev_private;
1985         struct mlx5_flow_meter_info *fm;
1986         struct mlx5_flow_meter_policy *policy;
1987         struct mlx5_flow_meter_sub_policy *sub_policy;
1988         struct mlx5_flow_meter_info *next_fm;
1989         struct mlx5_aso_mtr *aso_mtr;
1990         uint32_t mtr_idx = 0;
1991         uint32_t i, policy_idx;
1992         void *entry;
1993
1994         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
1995                 return 0;
1996         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
1997                 mtr_idx = *(uint32_t *)entry;
1998                 if (!mtr_idx)
1999                         continue;
2000                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2001                 fm = &aso_mtr->fm;
2002                 if (fm->ref_cnt || fm->def_policy)
2003                         continue;
2004                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2005                         return -rte_errno;
2006         }
2007         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2008                 policy_idx = *(uint32_t *)entry;
2009                 sub_policy = mlx5_ipool_get
2010                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2011                                 policy_idx);
2012                 if (!sub_policy)
2013                         return -rte_mtr_error_set(error,
2014                                         EINVAL,
2015                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2016                                         NULL, "Meter policy invalid.");
2017                 policy = sub_policy->main_policy;
2018                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2019                         continue;
2020                 next_fm = mlx5_flow_meter_find(priv,
2021                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2022                                 &mtr_idx);
2023                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2024                                                     error, true))
2025                         return -rte_mtr_error_set(error,
2026                                         EINVAL,
2027                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2028                                         NULL, "Meter policy invalid.");
2029                 mlx5_free(policy);
2030                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2031                         continue;
2032                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2033                                                     mtr_idx, error))
2034                         return -rte_errno;
2035         }
2036         return 0;
2037 }
2038 /**
2039  * Flush meter configuration.
2040  *
2041  * @param[in] dev
2042  *   Pointer to Ethernet device.
2043  * @param[out] error
2044  *   Pointer to rte meter error structure.
2045  *
2046  * @return
2047  *   0 on success, a negative errno value otherwise and rte_errno is set.
2048  */
2049 int
2050 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2051 {
2052         struct mlx5_priv *priv = dev->data->dev_private;
2053         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2054         struct mlx5_flow_meter_profile *fmp;
2055         struct mlx5_legacy_flow_meter *legacy_fm;
2056         struct mlx5_flow_meter_info *fm;
2057         struct mlx5_flow_meter_sub_policy *sub_policy;
2058         void *tmp;
2059         uint32_t i, mtr_idx, policy_idx;
2060         void *entry;
2061         struct mlx5_aso_mtr *aso_mtr;
2062
2063         if (!priv->mtr_en)
2064                 return 0;
2065         if (priv->sh->meter_aso_en) {
2066                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2067                         return -rte_errno;
2068                 if (priv->mtr_idx_tbl) {
2069                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2070                                 mtr_idx = *(uint32_t *)entry;
2071                                 if (mtr_idx) {
2072                                         aso_mtr =
2073                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2074                                         fm = &aso_mtr->fm;
2075                                         (void)mlx5_flow_meter_params_flush(dev,
2076                                                 fm, mtr_idx);
2077                                 }
2078                         }
2079                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2080                         priv->mtr_idx_tbl = NULL;
2081                 }
2082         } else {
2083                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2084                         fm = &legacy_fm->fm;
2085                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2086                                 return -rte_mtr_error_set(error, EINVAL,
2087                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2088                                 NULL, "MTR object meter profile invalid.");
2089                 }
2090         }
2091         if (priv->policy_idx_tbl) {
2092                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2093                         policy_idx = *(uint32_t *)entry;
2094                         sub_policy = mlx5_ipool_get
2095                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2096                                 policy_idx);
2097                         if (!sub_policy)
2098                                 return -rte_mtr_error_set(error,
2099                                                 EINVAL,
2100                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2101                                                 NULL, "MTR object "
2102                                                 "meter policy invalid.");
2103                         if (__mlx5_flow_meter_policy_delete(dev, i,
2104                                                 sub_policy->main_policy,
2105                                                 error, true))
2106                                 return -rte_mtr_error_set(error,
2107                                                 EINVAL,
2108                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2109                                                 NULL, "MTR object "
2110                                                 "meter policy invalid.");
2111                         mlx5_free(sub_policy->main_policy);
2112                 }
2113                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2114                 priv->policy_idx_tbl = NULL;
2115         }
2116         if (priv->mtr_profile_tbl) {
2117                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2118                         fmp = entry;
2119                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2120                                                            error))
2121                                 return -rte_mtr_error_set(error, EINVAL,
2122                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2123                                                 NULL, "Fail to destroy "
2124                                                 "meter profile.");
2125                 }
2126                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2127                 priv->mtr_profile_tbl = NULL;
2128         }
2129         /* Delete default policy table. */
2130         mlx5_flow_destroy_def_policy(dev);
2131         if (priv->sh->refcnt == 1)
2132                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2133         return 0;
2134 }