drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133
 134         /* Profile must not be NULL. */
 135         if (profile == NULL)
 136                 return -rte_mtr_error_set(error, EINVAL,
 137                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 138                                           NULL, "Meter profile is null.");
 139         /* Meter profile ID must be valid. */
 140         if (meter_profile_id == UINT32_MAX)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 143                                           NULL, "Meter profile id not valid.");
 144         /* Meter profile must not exist. */
 145         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 146         if (fmp)
 147                 return -rte_mtr_error_set(error, EEXIST,
 148                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 149                                           NULL,
 150                                           "Meter profile already exists.");
 151         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 152                 if (priv->config.hca_attr.qos.flow_meter_old) {
 153                         /* Verify support for flow meter parameters. */
 154                         if (priv->sh->meter_aso_en && profile->packet_mode) {
 155                                 if (profile->srtcm_rfc2697.cir > 0 &&
 156                                         (profile->srtcm_rfc2697.cir <<
 157                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 158                                         <= MLX5_SRTCM_CIR_MAX &&
 159                                         profile->srtcm_rfc2697.cbs > 0 &&
 160                                         (profile->srtcm_rfc2697.cbs <<
 161                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 162                                         <= MLX5_SRTCM_CBS_MAX &&
 163                                         (profile->srtcm_rfc2697.ebs <<
 164                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 165                                         <= MLX5_SRTCM_EBS_MAX)
 166                                         return 0;
 167                                 return -rte_mtr_error_set
 168                                              (error, ENOTSUP,
 169                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 170                                               NULL,
 171                                               profile->srtcm_rfc2697.ebs ?
 172                                               "Metering value ebs must be 0." :
 173                                               "Invalid metering parameters.");
 174                         }
 175                         if (profile->srtcm_rfc2697.cir > 0 &&
 176                                 profile->srtcm_rfc2697.cir <=
 177                                                 MLX5_SRTCM_CIR_MAX &&
 178                                 profile->srtcm_rfc2697.cbs > 0 &&
 179                                 profile->srtcm_rfc2697.cbs <=
 180                                                 MLX5_SRTCM_CBS_MAX &&
 181                                 profile->srtcm_rfc2697.ebs <=
 182                                                 MLX5_SRTCM_EBS_MAX)
 183                                 return 0;
 184                         return -rte_mtr_error_set(error, ENOTSUP,
 185                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 186                                         NULL,
 187                                         profile->srtcm_rfc2697.ebs ?
 188                                         "Metering value ebs must be 0." :
 189                                         "Invalid metering parameters.");
 190                 }
 191         }
 192         return -rte_mtr_error_set(error, ENOTSUP,
 193                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 194                                   NULL, "Metering algorithm not supported.");
 195 }
 196
 197 /**
 198  * Calculate mantissa and exponent for cir.
 199  *
 200  * @param[in] cir
 201  *   Value to be calculated.
 202  * @param[out] man
 203  *   Pointer to the mantissa.
 204  * @param[out] exp
 205  *   Pointer to the exp.
 206  */
 207 static void
 208 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 209 {
 210         int64_t _cir;
 211         int64_t delta = INT64_MAX;
 212         uint8_t _man = 0;
 213         uint8_t _exp = 0;
 214         uint64_t m, e;
 215
 216         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 217                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 218                         _cir = (1000000000ULL * m) >> e;
 219                         if (llabs(cir - _cir) <= delta) {
 220                                 delta = llabs(cir - _cir);
 221                                 _man = m;
 222                                 _exp = e;
 223                         }
 224                 }
 225         }
 226         *man = _man;
 227         *exp = _exp;
 228 }
 229
 230 /**
 231  * Calculate mantissa and exponent for xbs.
 232  *
 233  * @param[in] xbs
 234  *   Value to be calculated.
 235  * @param[out] man
 236  *   Pointer to the mantissa.
 237  * @param[out] exp
 238  *   Pointer to the exp.
 239  */
 240 static void
 241 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 242 {
 243         int _exp;
 244         double _man;
 245
 246         /* Special case xbs == 0 ? both exp and matissa are 0. */
 247         if (xbs == 0) {
 248                 *man = 0;
 249                 *exp = 0;
 250                 return;
 251         }
 252         /* xbs = xbs_mantissa * 2^xbs_exponent */
 253         _man = frexp(xbs, &_exp);
 254         _man = _man * pow(2, MLX5_MAN_WIDTH);
 255         _exp = _exp - MLX5_MAN_WIDTH;
 256         *man = (uint8_t)ceil(_man);
 257         *exp = _exp;
 258 }
 259
 260 /**
 261  * Fill the prm meter parameter.
 262  *
 263  * @param[in,out] fmp
 264  *   Pointer to meter profie to be converted.
 265  * @param[out] error
 266  *   Pointer to the error structure.
 267  *
 268  * @return
 269  *   0 on success, a negative errno value otherwise and rte_errno is set.
 270  */
 271 static int
 272 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 273                         struct mlx5_priv *priv, struct rte_mtr_error *error)
 274 {
 275         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 276         uint8_t man, exp;
 277         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 278         uint32_t ebs_exp, ebs_man;
 279         uint64_t cir, cbs, ebs;
 280
 281         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 282                 return -rte_mtr_error_set(error, ENOTSUP,
 283                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 284                                 NULL, "Metering algorithm not supported.");
 285         if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
 286                 return -rte_mtr_error_set(error, ENOTSUP,
 287                         RTE_MTR_ERROR_TYPE_METER_PROFILE,
 288                         NULL, "Metering algorithm packet mode not supported.");
 289         if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
 290                 cir = fmp->profile.srtcm_rfc2697.cir <<
 291                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 292                 cbs = fmp->profile.srtcm_rfc2697.cbs <<
 293                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 294                 ebs = fmp->profile.srtcm_rfc2697.ebs <<
 295                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 296         } else {
 297                 cir = fmp->profile.srtcm_rfc2697.cir;
 298                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 299                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 300         }
 301         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 302         mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
 303         /* Check if cir mantissa is too large. */
 304         if (exp > ASO_DSEG_CIR_EXP_MASK)
 305                 return -rte_mtr_error_set(error, ENOTSUP,
 306                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 307                                           "meter profile parameter cir is"
 308                                           " not supported.");
 309         cir_man = man;
 310         cir_exp = exp;
 311          /* cbs = cbs_mantissa * 2^cbs_exponent */
 312         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 313         /* Check if cbs mantissa is too large. */
 314         if (exp > ASO_DSEG_EXP_MASK)
 315                 return -rte_mtr_error_set(error, ENOTSUP,
 316                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 317                                           "meter profile parameter cbs is"
 318                                           " not supported.");
 319         cbs_man = man;
 320         cbs_exp = exp;
 321         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 322                                 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 323                                 cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 324                                 cir_man);
 325         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 326         /* Check if ebs mantissa is too large. */
 327         if (exp > ASO_DSEG_EXP_MASK)
 328                 return -rte_mtr_error_set(error, ENOTSUP,
 329                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 330                                           "meter profile parameter ebs is"
 331                                           " not supported.");
 332         ebs_man = man;
 333         ebs_exp = exp;
 334         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 335                                         ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 336         return 0;
 337 }
 338
 339 /**
 340  * Callback to get MTR capabilities.
 341  *
 342  * @param[in] dev
 343  *   Pointer to Ethernet device.
 344  * @param[out] cap
 345  *   Pointer to save MTR capabilities.
 346  * @param[out] error
 347  *   Pointer to the error structure.
 348  *
 349  * @return
 350  *   0 on success, a negative errno value otherwise and rte_errno is set.
 351  */
 352 static int
 353 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 354                  struct rte_mtr_capabilities *cap,
 355                  struct rte_mtr_error *error __rte_unused)
 356 {
 357         struct mlx5_priv *priv = dev->data->dev_private;
 358         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 359
 360         if (!priv->mtr_en)
 361                 return -rte_mtr_error_set(error, ENOTSUP,
 362                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 363                                           "Meter is not supported");
 364         memset(cap, 0, sizeof(*cap));
 365         if (priv->sh->meter_aso_en) {
 366                 /* 2 meters per one ASO cache line. */
 367                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 368                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 369         } else {
 370                 cap->n_max = 1 << qattr->log_max_flow_meter;
 371                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 372         }
 373         cap->srtcm_rfc2697_byte_mode_supported = 1;
 374         cap->n_shared_max = cap->n_max;
 375         cap->identical = 1;
 376         cap->shared_identical = 1;
 377         cap->shared_n_flows_per_mtr_max = 4 << 20;
 378         /* 2M flows can share the same meter. */
 379         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 380         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 381         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 382         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 383                           RTE_MTR_STATS_N_PKTS_DROPPED;
 384         return 0;
 385 }
 386
 387 /**
 388  * Callback to add MTR profile.
 389  *
 390  * @param[in] dev
 391  *   Pointer to Ethernet device.
 392  * @param[in] meter_profile_id
 393  *   Meter profile id.
 394  * @param[in] profile
 395  *   Pointer to meter profile detail.
 396  * @param[out] error
 397  *   Pointer to the error structure.
 398  *
 399  * @return
 400  *   0 on success, a negative errno value otherwise and rte_errno is set.
 401  */
 402 static int
 403 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 404                        uint32_t meter_profile_id,
 405                        struct rte_mtr_meter_profile *profile,
 406                        struct rte_mtr_error *error)
 407 {
 408         struct mlx5_priv *priv = dev->data->dev_private;
 409         struct mlx5_flow_meter_profile *fmp;
 410         union mlx5_l3t_data data;
 411         int ret;
 412
 413         if (!priv->mtr_en)
 414                 return -rte_mtr_error_set(error, ENOTSUP,
 415                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 416                                           "Meter is not supported");
 417         /* Check input params. */
 418         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 419                                                profile, error);
 420         if (ret)
 421                 return ret;
 422         /* Meter profile memory allocation. */
 423         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 424                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 425         if (fmp == NULL)
 426                 return -rte_mtr_error_set(error, ENOMEM,
 427                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 428                                           NULL, "Meter profile memory "
 429                                           "alloc failed.");
 430         /* Fill profile info. */
 431         fmp->id = meter_profile_id;
 432         fmp->profile = *profile;
 433         /* Fill the flow meter parameters for the PRM. */
 434         ret = mlx5_flow_meter_param_fill(fmp, priv, error);
 435         if (ret)
 436                 goto error;
 437         data.ptr = fmp;
 438         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 439                                  meter_profile_id, &data);
 440         if (ret)
 441                 return -rte_mtr_error_set(error, ENOTSUP,
 442                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 443                                           NULL, "Meter profile insert fail.");
 444         return 0;
 445 error:
 446         mlx5_free(fmp);
 447         return ret;
 448 }
 449
 450 /**
 451  * Callback to delete MTR profile.
 452  *
 453  * @param[in] dev
 454  *   Pointer to Ethernet device.
 455  * @param[in] meter_profile_id
 456  *   Meter profile id.
 457  * @param[out] error
 458  *   Pointer to the error structure.
 459  *
 460  * @return
 461  *   0 on success, a negative errno value otherwise and rte_errno is set.
 462  */
 463 static int
 464 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 465                           uint32_t meter_profile_id,
 466                           struct rte_mtr_error *error)
 467 {
 468         struct mlx5_priv *priv = dev->data->dev_private;
 469         struct mlx5_flow_meter_profile *fmp;
 470
 471         if (!priv->mtr_en)
 472                 return -rte_mtr_error_set(error, ENOTSUP,
 473                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 474                                           "Meter is not supported");
 475         /* Meter profile must exist. */
 476         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 477         if (fmp == NULL)
 478                 return -rte_mtr_error_set(error, ENOENT,
 479                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 480                                           &meter_profile_id,
 481                                           "Meter profile id is invalid.");
 482         /* Check profile is unused. */
 483         if (fmp->ref_cnt)
 484                 return -rte_mtr_error_set(error, EBUSY,
 485                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 486                                           NULL, "Meter profile is in use.");
 487         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 488                 return -rte_mtr_error_set(error, EBUSY,
 489                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 490                                           NULL, "Meter profile remove fail.");
 491         mlx5_free(fmp);
 492         return 0;
 493 }
 494
 495 /**
 496  * Find policy by id.
 497  *
 498  * @param[in] dev
 499  *   Pointer to Ethernet device.
 500  * @param policy_id
 501  *   Policy id.
 502  *
 503  * @return
 504  *   Pointer to the policy found on success, NULL otherwise.
 505  */
 506 struct mlx5_flow_meter_policy *
 507 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 508                             uint32_t policy_id,
 509                             uint32_t *policy_idx)
 510 {
 511         struct mlx5_priv *priv = dev->data->dev_private;
 512         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 513         union mlx5_l3t_data data;
 514
 515         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM ||
 516                 !priv->sh->mtrmng->policy_idx_tbl)
 517                 return NULL;
 518         if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl,
 519                                 policy_id, &data) ||
 520                                 !data.dword)
 521                 return NULL;
 522         if (policy_idx)
 523                 *policy_idx = data.dword;
 524         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 525                                         data.dword);
 526         /* Remove reference taken by the mlx5_l3t_get_entry. */
 527         mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 528                                 policy_id);
 529         if (sub_policy)
 530                 if (sub_policy->main_policy_id)
 531                         return sub_policy->main_policy;
 532         return NULL;
 533 }
 534
 535 /**
 536  * Callback to check MTR policy action validate
 537  *
 538  * @param[in] dev
 539  *   Pointer to Ethernet device.
 540  * @param[in] actions
 541  *   Pointer to meter policy action detail.
 542  * @param[out] error
 543  *   Pointer to the error structure.
 544  *
 545  * @return
 546  *   0 on success, a negative errno value otherwise and rte_errno is set.
 547  */
 548 static int
 549 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 550         struct rte_mtr_meter_policy_params *policy,
 551         struct rte_mtr_error *error)
 552 {
 553         struct mlx5_priv *priv = dev->data->dev_private;
 554         struct rte_flow_attr attr = { .transfer =
 555                         priv->config.dv_esw_en ? 1 : 0};
 556         bool is_rss = false;
 557         bool is_def_policy = false;
 558         uint8_t domain_bitmap;
 559         int ret;
 560
 561         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 562                 return -rte_mtr_error_set(error, ENOTSUP,
 563                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 564                                 NULL, "meter policy unsupported.");
 565         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 566                         &is_rss, &domain_bitmap, &is_def_policy, error);
 567         if (ret)
 568                 return ret;
 569         return 0;
 570 }
 571
 572 static int
 573 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 574                         uint32_t policy_id,
 575                         struct mlx5_flow_meter_policy *mtr_policy,
 576                         struct rte_mtr_error *error,
 577                         bool clear_l3t)
 578 {
 579         struct mlx5_priv *priv = dev->data->dev_private;
 580         struct mlx5_flow_meter_sub_policy *sub_policy;
 581         uint32_t i, j;
 582         uint16_t sub_policy_num;
 583
 584         rte_spinlock_lock(&mtr_policy->sl);
 585         if (mtr_policy->ref_cnt) {
 586                 rte_spinlock_unlock(&mtr_policy->sl);
 587                 return -rte_mtr_error_set(error, EBUSY,
 588                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 589                                  NULL,
 590                                 "Meter policy object is being used.");
 591         }
 592         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 593         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 594         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 595                 sub_policy_num = (mtr_policy->sub_policy_num >>
 596                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 597                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 598                 if (sub_policy_num) {
 599                         for (j = 0; j < sub_policy_num; j++) {
 600                                 sub_policy = mtr_policy->sub_policys[i][j];
 601                                 if (sub_policy)
 602                                         mlx5_ipool_free
 603                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 604                                         sub_policy->idx);
 605                         }
 606                 }
 607         }
 608         if (priv->sh->mtrmng->policy_idx_tbl && clear_l3t) {
 609                 if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 610                                         policy_id)) {
 611                         rte_spinlock_unlock(&mtr_policy->sl);
 612                         return -rte_mtr_error_set(error, ENOTSUP,
 613                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 614                                 "Fail to delete policy in index table.");
 615                 }
 616         }
 617         rte_spinlock_unlock(&mtr_policy->sl);
 618         return 0;
 619 }
 620
 621 /**
 622  * Callback to add MTR policy.
 623  *
 624  * @param[in] dev
 625  *   Pointer to Ethernet device.
 626  * @param[out] policy_id
 627  *   Pointer to policy id
 628  * @param[in] actions
 629  *   Pointer to meter policy action detail.
 630  * @param[out] error
 631  *   Pointer to the error structure.
 632  *
 633  * @return
 634  *   0 on success, a negative errno value otherwise and rte_errno is set.
 635  */
 636 static int
 637 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 638                         uint32_t policy_id,
 639                         struct rte_mtr_meter_policy_params *policy,
 640                         struct rte_mtr_error *error)
 641 {
 642         struct mlx5_priv *priv = dev->data->dev_private;
 643         struct rte_flow_attr attr = { .transfer =
 644                         priv->config.dv_esw_en ? 1 : 0};
 645         uint32_t sub_policy_idx = 0;
 646         uint32_t policy_idx = 0;
 647         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 648         struct mlx5_flow_meter_sub_policy *sub_policy;
 649         bool is_rss = false;
 650         bool is_def_policy = false;
 651         uint32_t i;
 652         int ret;
 653         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 654         uint16_t sub_policy_num;
 655         uint8_t domain_bitmap = 0;
 656         union mlx5_l3t_data data;
 657
 658         if (!priv->mtr_en)
 659                 return -rte_mtr_error_set(error, ENOTSUP,
 660                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 661                                           NULL, "meter policy unsupported.");
 662         if (policy_id == MLX5_INVALID_POLICY_ID)
 663                 return -rte_mtr_error_set(error, ENOTSUP,
 664                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 665                         "policy ID is invalid. ");
 666         if (policy_id == priv->sh->mtrmng->def_policy_id)
 667                 return -rte_mtr_error_set(error, EEXIST,
 668                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 669                         "policy ID exists. ");
 670         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id,
 671                                 &policy_idx);
 672         if (mtr_policy)
 673                 return -rte_mtr_error_set(error, EEXIST,
 674                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 675                         "policy ID exists. ");
 676         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 677                         &is_rss, &domain_bitmap, &is_def_policy, error);
 678         if (ret)
 679                 return ret;
 680         if (!domain_bitmap)
 681                 return -rte_mtr_error_set(error, ENOTSUP,
 682                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 683                                 NULL, "fail to find policy domain.");
 684         if (is_def_policy) {
 685                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 686                         return -rte_mtr_error_set(error, EEXIST,
 687                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 688                                 NULL, "a policy with similar actions "
 689                                 "is already configured");
 690                 if (mlx5_flow_create_def_policy(dev))
 691                         return -rte_mtr_error_set(error, ENOTSUP,
 692                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 693                                 NULL,
 694                                 "fail to create non-terminated policy.");
 695                 priv->sh->mtrmng->def_policy_id = policy_id;
 696                 return 0;
 697         }
 698         if (!priv->sh->meter_aso_en)
 699                 return -rte_mtr_error_set(error, ENOTSUP,
 700                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 701                         "no ASO capability to support the policy ");
 702         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 703                 if (!(domain_bitmap & (1 << i)))
 704                         continue;
 705                 if (is_rss) {
 706                         policy_size +=
 707                         sizeof(struct mlx5_flow_meter_sub_policy *) *
 708                         MLX5_MTR_RSS_MAX_SUB_POLICY;
 709                         break;
 710                 }
 711                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 712         }
 713         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 714                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 715         if (!mtr_policy)
 716                 return -rte_mtr_error_set(error, ENOMEM,
 717                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 718                                 "Memory alloc failed for meter policy.");
 719         policy_size = sizeof(struct mlx5_flow_meter_policy);
 720         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 721                 if (!(domain_bitmap & (1 << i)))
 722                         continue;
 723                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 724                         mtr_policy->ingress = 1;
 725                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 726                         mtr_policy->egress = 1;
 727                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 728                         mtr_policy->transfer = 1;
 729                 sub_policy = mlx5_ipool_zmalloc
 730                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 731                                 &sub_policy_idx);
 732                 if (!sub_policy)
 733                         goto policy_add_err;
 734                 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 735                         goto policy_add_err;
 736                 sub_policy->idx = sub_policy_idx;
 737                 sub_policy->main_policy = mtr_policy;
 738                 if (!policy_idx) {
 739                         policy_idx = sub_policy_idx;
 740                         sub_policy->main_policy_id = 1;
 741                 }
 742                 mtr_policy->sub_policys[i] =
 743                 (struct mlx5_flow_meter_sub_policy **)
 744                         ((uint8_t *)mtr_policy + policy_size);
 745                 mtr_policy->sub_policys[i][0] = sub_policy;
 746                 sub_policy_num = (mtr_policy->sub_policy_num >>
 747                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 748                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 749                 sub_policy_num++;
 750                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 751                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 752                 mtr_policy->sub_policy_num |=
 753                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 754                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 755                 if (is_rss) {
 756                         mtr_policy->is_rss = 1;
 757                         break;
 758                 }
 759                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy  *);
 760         }
 761         rte_spinlock_init(&mtr_policy->sl);
 762         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 763                                         policy->actions, error);
 764         if (ret)
 765                 goto policy_add_err;
 766         if (!is_rss && !mtr_policy->is_queue) {
 767                 /* Create policy rules in HW. */
 768                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 769                 if (ret)
 770                         goto policy_add_err;
 771         }
 772         data.dword = policy_idx;
 773         if (!priv->sh->mtrmng->policy_idx_tbl) {
 774                 priv->sh->mtrmng->policy_idx_tbl =
 775                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 776                 if (!priv->sh->mtrmng->policy_idx_tbl)
 777                         goto policy_add_err;
 778         }
 779         if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl,
 780                                 policy_id, &data))
 781                 goto policy_add_err;
 782         return 0;
 783 policy_add_err:
 784         if (mtr_policy) {
 785                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 786                         mtr_policy, error, false);
 787                 mlx5_free(mtr_policy);
 788                 if (ret)
 789                         return ret;
 790         }
 791         return -rte_mtr_error_set(error, ENOTSUP,
 792                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 793                                   NULL, "Failed to create devx policy.");
 794 }
 795
 796 /**
 797  * Callback to delete MTR policy.
 798  *
 799  * @param[in] dev
 800  *   Pointer to Ethernet device.
 801  * @param[in] policy_id
 802  *   Meter policy id.
 803  * @param[out] error
 804  *   Pointer to the error structure.
 805  *
 806  * @return
 807  *   0 on success, a negative errno value otherwise and rte_errno is set.
 808  */
 809 static int
 810 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 811                           uint32_t policy_id,
 812                           struct rte_mtr_error *error)
 813 {
 814         struct mlx5_priv *priv = dev->data->dev_private;
 815         struct mlx5_flow_meter_policy *mtr_policy;
 816         uint32_t policy_idx;
 817         int ret;
 818
 819         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 820                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 821                         return -rte_mtr_error_set(error, ENOTSUP,
 822                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 823                                 "Meter policy object is being used.");
 824                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 825                 return 0;
 826         }
 827         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 828         if (!mtr_policy)
 829                 return -rte_mtr_error_set(error, ENOTSUP,
 830                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 831                         "Meter policy id is invalid. ");
 832         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 833                                                 error, true);
 834         if (ret)
 835                 return ret;
 836         mlx5_free(mtr_policy);
 837         return 0;
 838 }
 839
 840 /**
 841  * Check meter validation.
 842  *
 843  * @param[in] priv
 844  *   Pointer to mlx5 private data structure.
 845  * @param[in] meter_id
 846  *   Meter id.
 847  * @param[in] params
 848  *   Pointer to rte meter parameters.
 849  * @param[out] error
 850  *   Pointer to rte meter error structure.
 851  *
 852  * @return
 853  *   0 on success, a negative errno value otherwise and rte_errno is set.
 854  */
 855 static int
 856 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 857                          struct rte_mtr_params *params,
 858                          struct rte_mtr_error *error)
 859 {
 860         /* Meter must use global drop action. */
 861         if (!priv->sh->dr_drop_action)
 862                 return -rte_mtr_error_set(error, ENOTSUP,
 863                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 864                                           NULL,
 865                                           "No drop action ready for meter.");
 866         /* Meter params must not be NULL. */
 867         if (params == NULL)
 868                 return -rte_mtr_error_set(error, EINVAL,
 869                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 870                                           NULL, "Meter object params null.");
 871         /* Previous meter color is not supported. */
 872         if (params->use_prev_mtr_color)
 873                 return -rte_mtr_error_set(error, ENOTSUP,
 874                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 875                                           NULL,
 876                                           "Previous meter color "
 877                                           "not supported.");
 878         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 879                 return -rte_mtr_error_set(error, ENOENT,
 880                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 881                                 NULL, "Meter policy id not valid.");
 882         /* Validate meter id. */
 883         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 884                 return -rte_mtr_error_set(error, EEXIST,
 885                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 886                         "Meter object already exists.");
 887         return 0;
 888 }
 889
 890 /**
 891  * Modify the flow meter action.
 892  *
 893  * @param[in] priv
 894  *   Pointer to mlx5 private data structure.
 895  * @param[in] fm
 896  *   Pointer to flow meter to be modified.
 897  * @param[in] srtcm
 898  *   Pointer to meter srtcm description parameter.
 899  * @param[in] modify_bits
 900  *   The bit in srtcm to be updated.
 901  * @param[in] active_state
 902  *   The state to be updated.
 903  * @return
 904  *   0 on success, o negative value otherwise.
 905  */
 906 static int
 907 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 908                 struct mlx5_flow_meter_info *fm,
 909                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 910                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 911 {
 912 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 913         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 914         uint32_t *attr;
 915         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 916         int ret;
 917         struct mlx5_aso_mtr *aso_mtr = NULL;
 918         uint32_t cbs_cir, ebs_eir, val;
 919
 920         if (priv->sh->meter_aso_en) {
 921                 fm->is_enable = !!is_enable;
 922                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 923                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 924                 if (ret)
 925                         return ret;
 926                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 927                 if (ret)
 928                         return ret;
 929         } else {
 930                 /* Fill command parameters. */
 931                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 932                 mod_attr.flow_meter_parameter = in;
 933                 mod_attr.flow_meter_parameter_sz =
 934                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 935                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 936                         mod_attr.active = !!active_state;
 937                 else
 938                         mod_attr.active = 0;
 939                 attr = in;
 940                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 941                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 942                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 943                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 944                                 ASO_DSEG_EXP_MASK;
 945                         MLX5_SET(flow_meter_parameters, attr,
 946                                 cbs_exponent, val);
 947                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 948                                 ASO_DSEG_MAN_MASK;
 949                         MLX5_SET(flow_meter_parameters, attr,
 950                                 cbs_mantissa, val);
 951                 }
 952                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
 953                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
 954                                 ASO_DSEG_EXP_MASK;
 955                         MLX5_SET(flow_meter_parameters, attr,
 956                                 cir_exponent, val);
 957                         val = cbs_cir & ASO_DSEG_MAN_MASK;
 958                         MLX5_SET(flow_meter_parameters, attr,
 959                                 cir_mantissa, val);
 960                 }
 961                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
 962                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
 963                                 ASO_DSEG_EXP_MASK;
 964                         MLX5_SET(flow_meter_parameters, attr,
 965                                 ebs_exponent, val);
 966                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
 967                                 ASO_DSEG_MAN_MASK;
 968                         MLX5_SET(flow_meter_parameters, attr,
 969                                 ebs_mantissa, val);
 970                 }
 971                 /* Apply modifications to meter only if it was created. */
 972                 if (fm->meter_action) {
 973                         ret = mlx5_glue->dv_modify_flow_action_meter
 974                                         (fm->meter_action, &mod_attr,
 975                                         rte_cpu_to_be_64(modify_bits));
 976                         if (ret)
 977                                 return ret;
 978                 }
 979                 /* Update succeedded modify meter parameters. */
 980                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 981                         fm->active_state = !!active_state;
 982         }
 983         return 0;
 984 #else
 985         (void)priv;
 986         (void)fm;
 987         (void)srtcm;
 988         (void)modify_bits;
 989         (void)active_state;
 990         (void)is_enable;
 991         return -ENOTSUP;
 992 #endif
 993 }
 994
 995 static int
 996 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
 997                                 struct mlx5_flow_meter_info *fm,
 998                                 uint64_t stats_mask)
 999 {
1000         fm->bytes_dropped =
1001                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1002         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1003         if (fm->bytes_dropped || fm->pkts_dropped) {
1004                 if (!fm->drop_cnt) {
1005                         /* Alloc policer counters. */
1006                         fm->drop_cnt = mlx5_counter_alloc(dev);
1007                         if (!fm->drop_cnt)
1008                                 return -1;
1009                 }
1010         } else {
1011                 if (fm->drop_cnt) {
1012                         mlx5_counter_free(dev, fm->drop_cnt);
1013                         fm->drop_cnt = 0;
1014                 }
1015         }
1016         return 0;
1017 }
1018
1019 /**
1020  * Create meter rules.
1021  *
1022  * @param[in] dev
1023  *   Pointer to Ethernet device.
1024  * @param[in] meter_id
1025  *   Meter id.
1026  * @param[in] params
1027  *   Pointer to rte meter parameters.
1028  * @param[in] shared
1029  *   Meter shared with other flow or not.
1030  * @param[out] error
1031  *   Pointer to rte meter error structure.
1032  *
1033  * @return
1034  *   0 on success, a negative errno value otherwise and rte_errno is set.
1035  */
1036 static int
1037 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1038                        struct rte_mtr_params *params, int shared,
1039                        struct rte_mtr_error *error)
1040 {
1041         struct mlx5_priv *priv = dev->data->dev_private;
1042         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1043         struct mlx5_flow_meter_profile *fmp;
1044         struct mlx5_flow_meter_info *fm;
1045         struct mlx5_legacy_flow_meter *legacy_fm;
1046         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1047         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1048                 .size = 0,
1049                 .trunk_size = 64,
1050                 .need_lock = 1,
1051                 .type = "mlx5_flow_mtr_flow_id_pool",
1052         };
1053         struct mlx5_aso_mtr *aso_mtr;
1054         uint32_t mtr_idx, policy_idx;
1055         union mlx5_l3t_data data;
1056         int ret;
1057         uint8_t domain_bitmap;
1058         uint8_t mtr_id_bits;
1059         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1060                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1061
1062         if (!priv->mtr_en)
1063                 return -rte_mtr_error_set(error, ENOTSUP,
1064                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1065                                         "Meter is not supported");
1066         /* Validate the parameters. */
1067         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1068         if (ret)
1069                 return ret;
1070         /* Meter profile must exist. */
1071         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1072         if (fmp == NULL)
1073                 return -rte_mtr_error_set(error, ENOENT,
1074                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1075                         NULL, "Meter profile id not valid.");
1076         /* Meter policy must exist. */
1077         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1078                 __atomic_add_fetch
1079                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1080                         1, __ATOMIC_RELAXED);
1081                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1082                 if (!priv->config.dv_esw_en)
1083                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1084         } else {
1085                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1086                                 params->meter_policy_id, &policy_idx);
1087                 if (!priv->sh->meter_aso_en)
1088                         return -rte_mtr_error_set(error, ENOTSUP,
1089                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1090                                 "Part of the policies cannot be "
1091                                 "supported without ASO ");
1092                 if (!mtr_policy)
1093                         return -rte_mtr_error_set(error, ENOENT,
1094                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1095                                 NULL, "Meter policy id not valid.");
1096                 domain_bitmap = (mtr_policy->ingress ?
1097                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1098                                 (mtr_policy->egress ?
1099                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1100                                 (mtr_policy->transfer ?
1101                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1102         }
1103         /* Allocate the flow meter memory. */
1104         if (priv->sh->meter_aso_en) {
1105                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1106                 if (!mtr_idx)
1107                         return -rte_mtr_error_set(error, ENOMEM,
1108                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1109                                 "Memory alloc failed for meter.");
1110                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1111                 fm = &aso_mtr->fm;
1112         } else {
1113                 legacy_fm = mlx5_ipool_zmalloc
1114                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1115                 if (legacy_fm == NULL)
1116                         return -rte_mtr_error_set(error, ENOMEM,
1117                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1118                                 "Memory alloc failed for meter.");
1119                 legacy_fm->idx = mtr_idx;
1120                 fm = &legacy_fm->fm;
1121         }
1122         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1123         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1124             mtr_reg_bits) {
1125                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1126                 goto error;
1127         }
1128         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1129                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1130         /* Fill the flow meter parameters. */
1131         fm->meter_id = meter_id;
1132         fm->policy_id = params->meter_policy_id;
1133         fm->profile = fmp;
1134         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1135                 goto error;
1136         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1137                 goto error;
1138         /* Add to the flow meter list. */
1139         if (!priv->sh->meter_aso_en)
1140                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1141         /* Add to the flow meter list. */
1142         fm->active_state = 1; /* Config meter starts as active. */
1143         fm->is_enable = 1;
1144         fm->shared = !!shared;
1145         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1146         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1147                 fm->def_policy = 1;
1148                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1149                 if (!fm->flow_ipool)
1150                         goto error;
1151         }
1152         rte_spinlock_init(&fm->sl);
1153         /* If ASO meter supported, update ASO flow meter by wqe. */
1154         if (priv->sh->meter_aso_en) {
1155                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1156                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1157                 if (ret)
1158                         goto error;
1159                 if (!priv->mtr_idx_tbl) {
1160                         priv->mtr_idx_tbl =
1161                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1162                         if (!priv->mtr_idx_tbl)
1163                                 goto error;
1164                 }
1165                 data.dword = mtr_idx;
1166                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1167                         goto error;
1168         }
1169         if (mtr_policy)
1170                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1171         return 0;
1172 error:
1173         mlx5_flow_destroy_mtr_tbls(dev, fm);
1174         /* Free policer counters. */
1175         if (fm->drop_cnt)
1176                 mlx5_counter_free(dev, fm->drop_cnt);
1177         if (priv->sh->meter_aso_en)
1178                 mlx5_flow_mtr_free(dev, mtr_idx);
1179         else
1180                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1181         return -rte_mtr_error_set(error, ENOTSUP,
1182                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1183                 NULL, "Failed to create devx meter.");
1184 }
1185
1186 static int
1187 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1188                         struct mlx5_flow_meter_info *fm,
1189                         uint32_t mtr_idx)
1190 {
1191         struct mlx5_priv *priv = dev->data->dev_private;
1192         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1193         struct mlx5_flow_meter_profile *fmp;
1194         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1195         struct mlx5_flow_meter_policy *mtr_policy;
1196
1197         /* Meter object must not have any owner. */
1198         MLX5_ASSERT(!fm->ref_cnt);
1199         /* Get meter profile. */
1200         fmp = fm->profile;
1201         if (fmp == NULL)
1202                 return -1;
1203         /* Update dependencies. */
1204         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1205         fm->profile = NULL;
1206         /* Remove from list. */
1207         if (!priv->sh->meter_aso_en) {
1208                 legacy_fm = container_of(fm,
1209                         struct mlx5_legacy_flow_meter, fm);
1210                 TAILQ_REMOVE(fms, legacy_fm, next);
1211         }
1212         /* Free drop counters. */
1213         if (fm->drop_cnt)
1214                 mlx5_counter_free(dev, fm->drop_cnt);
1215         /* Free meter flow table. */
1216         if (fm->flow_ipool) {
1217                 mlx5_ipool_destroy(fm->flow_ipool);
1218                 fm->flow_ipool = 0;
1219         }
1220         mlx5_flow_destroy_mtr_tbls(dev, fm);
1221         if (fm->def_policy)
1222                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1223                                 1, __ATOMIC_RELAXED);
1224         if (priv->sh->meter_aso_en) {
1225                 if (!fm->def_policy) {
1226                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1227                                                 fm->policy_id, NULL);
1228                         if (mtr_policy)
1229                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1230                                                 1, __ATOMIC_RELAXED);
1231                         fm->policy_id = 0;
1232                 }
1233                 fm->def_policy = 0;
1234                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1235                         return -1;
1236                 mlx5_flow_mtr_free(dev, mtr_idx);
1237         } else {
1238                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1239                                         legacy_fm->idx);
1240         }
1241         return 0;
1242 }
1243
1244 /**
1245  * Destroy meter rules.
1246  *
1247  * @param[in] dev
1248  *   Pointer to Ethernet device.
1249  * @param[in] meter_id
1250  *   Meter id.
1251  * @param[out] error
1252  *   Pointer to rte meter error structure.
1253  *
1254  * @return
1255  *   0 on success, a negative errno value otherwise and rte_errno is set.
1256  */
1257 static int
1258 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1259                         struct rte_mtr_error *error)
1260 {
1261         struct mlx5_priv *priv = dev->data->dev_private;
1262         struct mlx5_flow_meter_info *fm;
1263         uint32_t mtr_idx = 0;
1264
1265         if (!priv->mtr_en)
1266                 return -rte_mtr_error_set(error, ENOTSUP,
1267                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1268                                           NULL,
1269                                           "Meter is not supported");
1270         /* Meter object must exist. */
1271         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1272         if (fm == NULL)
1273                 return -rte_mtr_error_set(error, ENOENT,
1274                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1275                                           NULL,
1276                                           "Meter object id not valid.");
1277         /* Meter object must not have any owner. */
1278         if (fm->ref_cnt > 0)
1279                 return -rte_mtr_error_set(error, EBUSY,
1280                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1281                                           NULL,
1282                                           "Meter object is being used.");
1283         /* Destroy the meter profile. */
1284         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1285                 return -rte_mtr_error_set(error, EINVAL,
1286                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1287                                         NULL,
1288                                         "MTR object meter profile invalid.");
1289         return 0;
1290 }
1291
1292 /**
1293  * Modify meter state.
1294  *
1295  * @param[in] priv
1296  *   Pointer to mlx5 private data structure.
1297  * @param[in] fm
1298  *   Pointer to flow meter.
1299  * @param[in] new_state
1300  *   New state to update.
1301  * @param[out] error
1302  *   Pointer to rte meter error structure.
1303  *
1304  * @return
1305  *   0 on success, a negative errno value otherwise and rte_errno is set.
1306  */
1307 static int
1308 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1309                              struct mlx5_flow_meter_info *fm,
1310                              uint32_t new_state,
1311                              struct rte_mtr_error *error)
1312 {
1313         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1314                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1315                 .ebs_eir = 0,
1316         };
1317         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1318                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1319         int ret;
1320
1321         if (new_state == MLX5_FLOW_METER_DISABLE)
1322                 ret = mlx5_flow_meter_action_modify(priv, fm,
1323                                 &srtcm, modify_bits, 0, 0);
1324         else
1325                 ret = mlx5_flow_meter_action_modify(priv, fm,
1326                                                    &fm->profile->srtcm_prm,
1327                                                     modify_bits, 0, 1);
1328         if (ret)
1329                 return -rte_mtr_error_set(error, -ret,
1330                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1331                                           NULL,
1332                                           new_state ?
1333                                           "Failed to enable meter." :
1334                                           "Failed to disable meter.");
1335         return 0;
1336 }
1337
1338 /**
1339  * Callback to enable flow meter.
1340  *
1341  * @param[in] dev
1342  *   Pointer to Ethernet device.
1343  * @param[in] meter_id
1344  *   Meter id.
1345  * @param[out] error
1346  *   Pointer to rte meter error structure.
1347  *
1348  * @return
1349  *   0 on success, a negative errno value otherwise and rte_errno is set.
1350  */
1351 static int
1352 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1353                        uint32_t meter_id,
1354                        struct rte_mtr_error *error)
1355 {
1356         struct mlx5_priv *priv = dev->data->dev_private;
1357         struct mlx5_flow_meter_info *fm;
1358         int ret;
1359
1360         if (!priv->mtr_en)
1361                 return -rte_mtr_error_set(error, ENOTSUP,
1362                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1363                                           "Meter is not supported");
1364         /* Meter object must exist. */
1365         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1366         if (fm == NULL)
1367                 return -rte_mtr_error_set(error, ENOENT,
1368                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1369                                           NULL, "Meter not found.");
1370         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1371                 return 0;
1372         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1373                                            error);
1374         if (!ret)
1375                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1376         return ret;
1377 }
1378
1379 /**
1380  * Callback to disable flow meter.
1381  *
1382  * @param[in] dev
1383  *   Pointer to Ethernet device.
1384  * @param[in] meter_id
1385  *   Meter id.
1386  * @param[out] error
1387  *   Pointer to rte meter error structure.
1388  *
1389  * @return
1390  *   0 on success, a negative errno value otherwise and rte_errno is set.
1391  */
1392 static int
1393 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1394                         uint32_t meter_id,
1395                         struct rte_mtr_error *error)
1396 {
1397         struct mlx5_priv *priv = dev->data->dev_private;
1398         struct mlx5_flow_meter_info *fm;
1399         int ret;
1400
1401         if (!priv->mtr_en)
1402                 return -rte_mtr_error_set(error, ENOTSUP,
1403                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1404                                           "Meter is not supported");
1405         /* Meter object must exist. */
1406         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1407         if (fm == NULL)
1408                 return -rte_mtr_error_set(error, ENOENT,
1409                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1410                                           NULL, "Meter not found.");
1411         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1412                 return 0;
1413         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1414                                            error);
1415         if (!ret)
1416                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1417         return ret;
1418 }
1419
1420 /**
1421  * Callback to update meter profile.
1422  *
1423  * @param[in] dev
1424  *   Pointer to Ethernet device.
1425  * @param[in] meter_id
1426  *   Meter id.
1427  * @param[in] meter_profile_id
1428  *   To be updated meter profile id.
1429  * @param[out] error
1430  *   Pointer to rte meter error structure.
1431  *
1432  * @return
1433  *   0 on success, a negative errno value otherwise and rte_errno is set.
1434  */
1435 static int
1436 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1437                                uint32_t meter_id,
1438                                uint32_t meter_profile_id,
1439                                struct rte_mtr_error *error)
1440 {
1441         struct mlx5_priv *priv = dev->data->dev_private;
1442         struct mlx5_flow_meter_profile *fmp;
1443         struct mlx5_flow_meter_profile *old_fmp;
1444         struct mlx5_flow_meter_info *fm;
1445         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1446                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1447         int ret;
1448
1449         if (!priv->mtr_en)
1450                 return -rte_mtr_error_set(error, ENOTSUP,
1451                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1452                                           "Meter is not supported");
1453         /* Meter profile must exist. */
1454         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1455         if (fmp == NULL)
1456                 return -rte_mtr_error_set(error, ENOENT,
1457                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1458                                           NULL, "Meter profile not found.");
1459         /* Meter object must exist. */
1460         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1461         if (fm == NULL)
1462                 return -rte_mtr_error_set(error, ENOENT,
1463                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1464                                           NULL, "Meter not found.");
1465         /* MTR object already set to meter profile id. */
1466         old_fmp = fm->profile;
1467         if (fmp == old_fmp)
1468                 return 0;
1469         /* Update the profile. */
1470         fm->profile = fmp;
1471         /* Update meter params in HW (if not disabled). */
1472         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1473                 return 0;
1474         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1475                                               modify_bits, fm->active_state, 1);
1476         if (ret) {
1477                 fm->profile = old_fmp;
1478                 return -rte_mtr_error_set(error, -ret,
1479                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1480                                           NULL, "Failed to update meter"
1481                                           " parmeters in hardware.");
1482         }
1483         old_fmp->ref_cnt--;
1484         fmp->ref_cnt++;
1485         return 0;
1486 }
1487
1488 /**
1489  * Callback to update meter stats mask.
1490  *
1491  * @param[in] dev
1492  *   Pointer to Ethernet device.
1493  * @param[in] meter_id
1494  *   Meter id.
1495  * @param[in] stats_mask
1496  *   To be updated stats_mask.
1497  * @param[out] error
1498  *   Pointer to rte meter error structure.
1499  *
1500  * @return
1501  *   0 on success, a negative errno value otherwise and rte_errno is set.
1502  */
1503 static int
1504 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1505                              uint32_t meter_id,
1506                              uint64_t stats_mask,
1507                              struct rte_mtr_error *error)
1508 {
1509         struct mlx5_priv *priv = dev->data->dev_private;
1510         struct mlx5_flow_meter_info *fm;
1511
1512         if (!priv->mtr_en)
1513                 return -rte_mtr_error_set(error, ENOTSUP,
1514                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1515                                           "Meter is not supported");
1516         /* Meter object must exist. */
1517         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1518         if (fm == NULL)
1519                 return -rte_mtr_error_set(error, ENOENT,
1520                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1521                                           NULL, "Meter object id not valid.");
1522         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1523                 return -rte_mtr_error_set(error, ENOENT,
1524                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1525                                           NULL, "Fail to allocate "
1526                                           "counter for meter.");
1527         return 0;
1528 }
1529
1530 /**
1531  * Callback to read meter statistics.
1532  *
1533  * @param[in] dev
1534  *   Pointer to Ethernet device.
1535  * @param[in] meter_id
1536  *   Meter id.
1537  * @param[out] stats
1538  *   Pointer to store the statistics.
1539  * @param[out] stats_mask
1540  *   Pointer to store the stats_mask.
1541  * @param[in] clear
1542  *   Statistic to be cleared after read or not.
1543  * @param[out] error
1544  *   Pointer to rte meter error structure.
1545  *
1546  * @return
1547  *   0 on success, a negative errno value otherwise and rte_errno is set.
1548  */
1549 static int
1550 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1551                            uint32_t meter_id,
1552                            struct rte_mtr_stats *stats,
1553                            uint64_t *stats_mask,
1554                            int clear,
1555                            struct rte_mtr_error *error)
1556 {
1557         struct mlx5_priv *priv = dev->data->dev_private;
1558         struct mlx5_flow_meter_info *fm;
1559         uint64_t pkts;
1560         uint64_t bytes;
1561         int ret = 0;
1562
1563         if (!priv->mtr_en)
1564                 return -rte_mtr_error_set(error, ENOTSUP,
1565                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1566                                           "Meter is not supported");
1567         /* Meter object must exist. */
1568         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1569         if (fm == NULL)
1570                 return -rte_mtr_error_set(error, ENOENT,
1571                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1572                                           NULL, "Meter object id not valid.");
1573         *stats_mask = 0;
1574         if (fm->bytes_dropped)
1575                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1576         if (fm->pkts_dropped)
1577                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1578         memset(stats, 0, sizeof(*stats));
1579         if (fm->drop_cnt) {
1580                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1581                                                  &bytes);
1582                 if (ret)
1583                         goto error;
1584                 /* If need to read the packets, set it. */
1585                 if (fm->pkts_dropped)
1586                         stats->n_pkts_dropped = pkts;
1587                 /* If need to read the bytes, set it. */
1588                 if (fm->bytes_dropped)
1589                         stats->n_bytes_dropped = bytes;
1590         }
1591         return 0;
1592 error:
1593         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1594                                  "Failed to read meter drop counters.");
1595 }
1596
1597 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1598         .capabilities_get = mlx5_flow_mtr_cap_get,
1599         .meter_profile_add = mlx5_flow_meter_profile_add,
1600         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1601         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1602         .meter_policy_add = mlx5_flow_meter_policy_add,
1603         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1604         .create = mlx5_flow_meter_create,
1605         .destroy = mlx5_flow_meter_destroy,
1606         .meter_enable = mlx5_flow_meter_enable,
1607         .meter_disable = mlx5_flow_meter_disable,
1608         .meter_profile_update = mlx5_flow_meter_profile_update,
1609         .meter_dscp_table_update = NULL,
1610         .stats_update = mlx5_flow_meter_stats_update,
1611         .stats_read = mlx5_flow_meter_stats_read,
1612 };
1613
1614 /**
1615  * Get meter operations.
1616  *
1617  * @param dev
1618  *   Pointer to Ethernet device structure.
1619  * @param arg
1620  *   Pointer to set the mtr operations.
1621  *
1622  * @return
1623  *   Always 0.
1624  */
1625 int
1626 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1627 {
1628         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1629         return 0;
1630 }
1631
1632 /**
1633  * Find meter by id.
1634  *
1635  * @param priv
1636  *   Pointer to mlx5_priv.
1637  * @param meter_id
1638  *   Meter id.
1639  * @param mtr_idx
1640  *   Pointer to Meter index.
1641  *
1642  * @return
1643  *   Pointer to the meter info found on success, NULL otherwise.
1644  */
1645 struct mlx5_flow_meter_info *
1646 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1647                 uint32_t *mtr_idx)
1648 {
1649         struct mlx5_legacy_flow_meter *legacy_fm;
1650         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1651         struct mlx5_aso_mtr *aso_mtr;
1652         struct mlx5_aso_mtr_pools_mng *pools_mng =
1653                                 &priv->sh->mtrmng->pools_mng;
1654         union mlx5_l3t_data data;
1655
1656         if (priv->sh->meter_aso_en) {
1657                 rte_spinlock_lock(&pools_mng->mtrsl);
1658                 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1659                         rte_spinlock_unlock(&pools_mng->mtrsl);
1660                         return NULL;
1661                 }
1662                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1663                         !data.dword) {
1664                         rte_spinlock_unlock(&pools_mng->mtrsl);
1665                         return NULL;
1666                 }
1667                 if (mtr_idx)
1668                         *mtr_idx = data.dword;
1669                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1670                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1671                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1672                 rte_spinlock_unlock(&pools_mng->mtrsl);
1673                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1674                         return NULL;
1675                 return &aso_mtr->fm;
1676         }
1677         TAILQ_FOREACH(legacy_fm, fms, next)
1678                 if (meter_id == legacy_fm->fm.meter_id) {
1679                         if (mtr_idx)
1680                                 *mtr_idx = legacy_fm->idx;
1681                         return &legacy_fm->fm;
1682                 }
1683         return NULL;
1684 }
1685
1686 /**
1687  * Find meter by index.
1688  *
1689  * @param priv
1690  *   Pointer to mlx5_priv.
1691  * @param idx
1692  *   Meter index.
1693  *
1694  * @return
1695  *   Pointer to the meter info found on success, NULL otherwise.
1696  */
1697 struct mlx5_flow_meter_info *
1698 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1699 {
1700         struct mlx5_aso_mtr *aso_mtr;
1701
1702         if (priv->sh->meter_aso_en) {
1703                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1704                 if (!aso_mtr)
1705                         return NULL;
1706                 return &aso_mtr->fm;
1707         } else {
1708                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1709         }
1710 }
1711
1712 /**
1713  * Attach meter to flow.
1714  * Unidirectional Meter creation can only be done
1715  * when flow direction is known, i.e. when calling meter_attach.
1716  *
1717  * @param [in] priv
1718  *  Pointer to mlx5 private data.
1719  * @param[in] fm
1720  *   Pointer to flow meter.
1721  * @param [in] attr
1722  *  Pointer to flow attributes.
1723  * @param [out] error
1724  *  Pointer to error structure.
1725  *
1726  * @return
1727  *   0 on success, a negative errno value otherwise and rte_errno is set.
1728  */
1729 int
1730 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1731                        struct mlx5_flow_meter_info *fm,
1732                        const struct rte_flow_attr *attr,
1733                        struct rte_flow_error *error)
1734 {
1735         int ret = 0;
1736
1737         if (priv->sh->meter_aso_en) {
1738                 struct mlx5_aso_mtr *aso_mtr;
1739
1740                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1741                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1742                         return rte_flow_error_set(error, ENOENT,
1743                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1744                                         NULL,
1745                                         "Timeout in meter configuration");
1746                 }
1747                 rte_spinlock_lock(&fm->sl);
1748                 if (fm->shared || !fm->ref_cnt) {
1749                         fm->ref_cnt++;
1750                 } else {
1751                         rte_flow_error_set(error, EINVAL,
1752                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1753                                    "Meter cannot be shared");
1754                         ret = -1;
1755                 }
1756                 rte_spinlock_unlock(&fm->sl);
1757         } else {
1758                 rte_spinlock_lock(&fm->sl);
1759                 if (fm->meter_action) {
1760                         if (fm->shared &&
1761                             attr->transfer == fm->transfer &&
1762                             attr->ingress == fm->ingress &&
1763                             attr->egress == fm->egress) {
1764                                 fm->ref_cnt++;
1765                         } else {
1766                                 rte_flow_error_set(error, EINVAL,
1767                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1768                                         fm->shared ?
1769                                         "Meter attr not match." :
1770                                         "Meter cannot be shared.");
1771                                 ret = -1;
1772                         }
1773                 } else {
1774                         fm->ingress = attr->ingress;
1775                         fm->egress = attr->egress;
1776                         fm->transfer = attr->transfer;
1777                         fm->ref_cnt = 1;
1778                         /* This also creates the meter object. */
1779                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1780                                                                          fm);
1781                         if (!fm->meter_action) {
1782                                 fm->ref_cnt = 0;
1783                                 fm->ingress = 0;
1784                                 fm->egress = 0;
1785                                 fm->transfer = 0;
1786                                 rte_flow_error_set(error, EINVAL,
1787                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1788                                         "Meter action create failed.");
1789                                 ret = -1;
1790                         }
1791                 }
1792                 rte_spinlock_unlock(&fm->sl);
1793         }
1794         return ret ? -rte_errno : 0;
1795 }
1796
1797 /**
1798  * Detach meter from flow.
1799  *
1800  * @param [in] priv
1801  *  Pointer to mlx5 private data.
1802  * @param [in] fm
1803  *  Pointer to flow meter.
1804  */
1805 void
1806 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1807                        struct mlx5_flow_meter_info *fm)
1808 {
1809 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1810         rte_spinlock_lock(&fm->sl);
1811         MLX5_ASSERT(fm->ref_cnt);
1812         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1813                 mlx5_glue->destroy_flow_action(fm->meter_action);
1814                 fm->meter_action = NULL;
1815                 fm->ingress = 0;
1816                 fm->egress = 0;
1817                 fm->transfer = 0;
1818         }
1819         rte_spinlock_unlock(&fm->sl);
1820 #else
1821         (void)priv;
1822         (void)fm;
1823 #endif
1824 }
1825
1826 /**
1827  * Flush meter with Rx queue configuration.
1828  *
1829  * @param[in] dev
1830  *   Pointer to Ethernet device.
1831  */
1832 void
1833 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1834 {
1835         struct mlx5_priv *priv = dev->data->dev_private;
1836         struct mlx5_flow_meter_sub_policy *sub_policy;
1837         struct mlx5_flow_meter_policy *mtr_policy;
1838         void *entry;
1839         uint32_t i, policy_idx;
1840
1841         if (!priv->mtr_en)
1842                 return;
1843         if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1844                 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1845                                         i, entry) {
1846                         policy_idx = *(uint32_t *)entry;
1847                         sub_policy = mlx5_ipool_get
1848                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1849                                 policy_idx);
1850                         if (!sub_policy || !sub_policy->main_policy)
1851                                 continue;
1852                         mtr_policy = sub_policy->main_policy;
1853                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1854                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1855                                         mtr_policy);
1856                 }
1857         }
1858 }
1859
1860 /**
1861  * Flush meter configuration.
1862  *
1863  * @param[in] dev
1864  *   Pointer to Ethernet device.
1865  * @param[out] error
1866  *   Pointer to rte meter error structure.
1867  *
1868  * @return
1869  *   0 on success, a negative errno value otherwise and rte_errno is set.
1870  */
1871 int
1872 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1873 {
1874         struct mlx5_priv *priv = dev->data->dev_private;
1875         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1876         struct mlx5_flow_meter_profile *fmp;
1877         struct mlx5_legacy_flow_meter *legacy_fm;
1878         struct mlx5_flow_meter_info *fm;
1879         struct mlx5_flow_meter_sub_policy *sub_policy;
1880         void *tmp;
1881         uint32_t i, mtr_idx, policy_idx;
1882         void *entry;
1883         struct mlx5_aso_mtr *aso_mtr;
1884
1885         if (!priv->mtr_en)
1886                 return 0;
1887         if (priv->sh->meter_aso_en) {
1888                 if (priv->mtr_idx_tbl) {
1889                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
1890                                 mtr_idx = *(uint32_t *)entry;
1891                                 if (mtr_idx) {
1892                                         aso_mtr =
1893                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
1894                                         fm = &aso_mtr->fm;
1895                                         (void)mlx5_flow_meter_params_flush(dev,
1896                                                 fm, mtr_idx);
1897                                 }
1898                         }
1899                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
1900                         priv->mtr_idx_tbl = NULL;
1901                 }
1902         } else {
1903                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1904                         fm = &legacy_fm->fm;
1905                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
1906                                 return -rte_mtr_error_set(error, EINVAL,
1907                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1908                                 NULL, "MTR object meter profile invalid.");
1909                 }
1910         }
1911         if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1912                 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1913                                         i, entry) {
1914                         policy_idx = *(uint32_t *)entry;
1915                         sub_policy = mlx5_ipool_get
1916                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1917                                 policy_idx);
1918                         if (!sub_policy)
1919                                 return -rte_mtr_error_set(error,
1920                                                 EINVAL,
1921                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1922                                                 NULL, "MTR object "
1923                                                 "meter policy invalid.");
1924                         if (__mlx5_flow_meter_policy_delete(dev, i,
1925                                                 sub_policy->main_policy,
1926                                                 error, true))
1927                                 return -rte_mtr_error_set(error,
1928                                                 EINVAL,
1929                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1930                                                 NULL, "MTR object "
1931                                                 "meter policy invalid.");
1932                         mlx5_free(sub_policy->main_policy);
1933                 }
1934                 mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl);
1935                 priv->sh->mtrmng->policy_idx_tbl = NULL;
1936         }
1937         if (priv->mtr_profile_tbl) {
1938                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
1939                         fmp = entry;
1940                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
1941                                                            error))
1942                                 return -rte_mtr_error_set(error, EINVAL,
1943                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1944                                                 NULL, "Fail to destroy "
1945                                                 "meter profile.");
1946                 }
1947                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
1948                 priv->mtr_profile_tbl = NULL;
1949         }
1950         /* Delete default policy table. */
1951         mlx5_flow_destroy_def_policy(dev);
1952         if (priv->sh->refcnt == 1)
1953                 mlx5_flow_destroy_mtr_drop_tbls(dev);
1954         return 0;
1955 }