drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133
 134         /* Profile must not be NULL. */
 135         if (profile == NULL)
 136                 return -rte_mtr_error_set(error, EINVAL,
 137                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 138                                           NULL, "Meter profile is null.");
 139         /* Meter profile ID must be valid. */
 140         if (meter_profile_id == UINT32_MAX)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 143                                           NULL, "Meter profile id not valid.");
 144         /* Meter profile must not exist. */
 145         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 146         if (fmp)
 147                 return -rte_mtr_error_set(error, EEXIST,
 148                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 149                                           NULL,
 150                                           "Meter profile already exists.");
 151         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 152                 if (priv->config.hca_attr.qos.flow_meter_old) {
 153                         /* Verify support for flow meter parameters. */
 154                         if (priv->sh->meter_aso_en && profile->packet_mode) {
 155                                 if (profile->srtcm_rfc2697.cir > 0 &&
 156                                         (profile->srtcm_rfc2697.cir <<
 157                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 158                                         <= MLX5_SRTCM_CIR_MAX &&
 159                                         profile->srtcm_rfc2697.cbs > 0 &&
 160                                         (profile->srtcm_rfc2697.cbs <<
 161                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 162                                         <= MLX5_SRTCM_CBS_MAX &&
 163                                         (profile->srtcm_rfc2697.ebs <<
 164                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 165                                         <= MLX5_SRTCM_EBS_MAX)
 166                                         return 0;
 167                                 return -rte_mtr_error_set
 168                                              (error, ENOTSUP,
 169                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 170                                               NULL,
 171                                               profile->srtcm_rfc2697.ebs ?
 172                                               "Metering value ebs must be 0." :
 173                                               "Invalid metering parameters.");
 174                         }
 175                         if (profile->srtcm_rfc2697.cir > 0 &&
 176                                 profile->srtcm_rfc2697.cir <=
 177                                                 MLX5_SRTCM_CIR_MAX &&
 178                                 profile->srtcm_rfc2697.cbs > 0 &&
 179                                 profile->srtcm_rfc2697.cbs <=
 180                                                 MLX5_SRTCM_CBS_MAX &&
 181                                 profile->srtcm_rfc2697.ebs <=
 182                                                 MLX5_SRTCM_EBS_MAX)
 183                                 return 0;
 184                         return -rte_mtr_error_set(error, ENOTSUP,
 185                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 186                                         NULL,
 187                                         profile->srtcm_rfc2697.ebs ?
 188                                         "Metering value ebs must be 0." :
 189                                         "Invalid metering parameters.");
 190                 }
 191         }
 192         return -rte_mtr_error_set(error, ENOTSUP,
 193                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 194                                   NULL, "Metering algorithm not supported.");
 195 }
 196
 197 /**
 198  * Calculate mantissa and exponent for cir.
 199  *
 200  * @param[in] cir
 201  *   Value to be calculated.
 202  * @param[out] man
 203  *   Pointer to the mantissa.
 204  * @param[out] exp
 205  *   Pointer to the exp.
 206  */
 207 static void
 208 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 209 {
 210         int64_t _cir;
 211         int64_t delta = INT64_MAX;
 212         uint8_t _man = 0;
 213         uint8_t _exp = 0;
 214         uint64_t m, e;
 215
 216         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 217                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 218                         _cir = (1000000000ULL * m) >> e;
 219                         if (llabs(cir - _cir) <= delta) {
 220                                 delta = llabs(cir - _cir);
 221                                 _man = m;
 222                                 _exp = e;
 223                         }
 224                 }
 225         }
 226         *man = _man;
 227         *exp = _exp;
 228 }
 229
 230 /**
 231  * Calculate mantissa and exponent for xbs.
 232  *
 233  * @param[in] xbs
 234  *   Value to be calculated.
 235  * @param[out] man
 236  *   Pointer to the mantissa.
 237  * @param[out] exp
 238  *   Pointer to the exp.
 239  */
 240 static void
 241 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 242 {
 243         int _exp;
 244         double _man;
 245
 246         /* Special case xbs == 0 ? both exp and matissa are 0. */
 247         if (xbs == 0) {
 248                 *man = 0;
 249                 *exp = 0;
 250                 return;
 251         }
 252         /* xbs = xbs_mantissa * 2^xbs_exponent */
 253         _man = frexp(xbs, &_exp);
 254         _man = _man * pow(2, MLX5_MAN_WIDTH);
 255         _exp = _exp - MLX5_MAN_WIDTH;
 256         *man = (uint8_t)ceil(_man);
 257         *exp = _exp;
 258 }
 259
 260 /**
 261  * Fill the prm meter parameter.
 262  *
 263  * @param[in,out] fmp
 264  *   Pointer to meter profie to be converted.
 265  * @param[out] error
 266  *   Pointer to the error structure.
 267  *
 268  * @return
 269  *   0 on success, a negative errno value otherwise and rte_errno is set.
 270  */
 271 static int
 272 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 273                         struct mlx5_priv *priv, struct rte_mtr_error *error)
 274 {
 275         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 276         uint8_t man, exp;
 277         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 278         uint32_t ebs_exp, ebs_man;
 279         uint64_t cir, cbs, ebs;
 280
 281         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 282                 return -rte_mtr_error_set(error, ENOTSUP,
 283                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 284                                 NULL, "Metering algorithm not supported.");
 285         if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
 286                 return -rte_mtr_error_set(error, ENOTSUP,
 287                         RTE_MTR_ERROR_TYPE_METER_PROFILE,
 288                         NULL, "Metering algorithm packet mode not supported.");
 289         if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
 290                 cir = fmp->profile.srtcm_rfc2697.cir <<
 291                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 292                 cbs = fmp->profile.srtcm_rfc2697.cbs <<
 293                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 294                 ebs = fmp->profile.srtcm_rfc2697.ebs <<
 295                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 296         } else {
 297                 cir = fmp->profile.srtcm_rfc2697.cir;
 298                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 299                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 300         }
 301         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 302         mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
 303         /* Check if cir mantissa is too large. */
 304         if (exp > ASO_DSEG_CIR_EXP_MASK)
 305                 return -rte_mtr_error_set(error, ENOTSUP,
 306                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 307                                           "meter profile parameter cir is"
 308                                           " not supported.");
 309         cir_man = man;
 310         cir_exp = exp;
 311          /* cbs = cbs_mantissa * 2^cbs_exponent */
 312         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 313         /* Check if cbs mantissa is too large. */
 314         if (exp > ASO_DSEG_EXP_MASK)
 315                 return -rte_mtr_error_set(error, ENOTSUP,
 316                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 317                                           "meter profile parameter cbs is"
 318                                           " not supported.");
 319         cbs_man = man;
 320         cbs_exp = exp;
 321         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 322                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 323                                           cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 324                                           cir_man);
 325         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 326         /* Check if ebs mantissa is too large. */
 327         if (exp > ASO_DSEG_EXP_MASK)
 328                 return -rte_mtr_error_set(error, ENOTSUP,
 329                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 330                                           "meter profile parameter ebs is"
 331                                           " not supported.");
 332         ebs_man = man;
 333         ebs_exp = exp;
 334         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 335                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 336         return 0;
 337 }
 338
 339 /**
 340  * Callback to get MTR capabilities.
 341  *
 342  * @param[in] dev
 343  *   Pointer to Ethernet device.
 344  * @param[out] cap
 345  *   Pointer to save MTR capabilities.
 346  * @param[out] error
 347  *   Pointer to the error structure.
 348  *
 349  * @return
 350  *   0 on success, a negative errno value otherwise and rte_errno is set.
 351  */
 352 static int
 353 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 354                  struct rte_mtr_capabilities *cap,
 355                  struct rte_mtr_error *error __rte_unused)
 356 {
 357         struct mlx5_priv *priv = dev->data->dev_private;
 358         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 359
 360         if (!priv->mtr_en)
 361                 return -rte_mtr_error_set(error, ENOTSUP,
 362                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 363                                           "Meter is not supported");
 364         memset(cap, 0, sizeof(*cap));
 365         if (priv->sh->meter_aso_en) {
 366                 /* 2 meters per one ASO cache line. */
 367                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 368                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 369         } else {
 370                 cap->n_max = 1 << qattr->log_max_flow_meter;
 371                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 372         }
 373         cap->srtcm_rfc2697_byte_mode_supported = 1;
 374         cap->n_shared_max = cap->n_max;
 375         cap->identical = 1;
 376         cap->shared_identical = 1;
 377         cap->shared_n_flows_per_mtr_max = 4 << 20;
 378         /* 2M flows can share the same meter. */
 379         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 380         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 381         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 382         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 383                           RTE_MTR_STATS_N_PKTS_DROPPED;
 384         return 0;
 385 }
 386
 387 /**
 388  * Callback to add MTR profile.
 389  *
 390  * @param[in] dev
 391  *   Pointer to Ethernet device.
 392  * @param[in] meter_profile_id
 393  *   Meter profile id.
 394  * @param[in] profile
 395  *   Pointer to meter profile detail.
 396  * @param[out] error
 397  *   Pointer to the error structure.
 398  *
 399  * @return
 400  *   0 on success, a negative errno value otherwise and rte_errno is set.
 401  */
 402 static int
 403 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 404                        uint32_t meter_profile_id,
 405                        struct rte_mtr_meter_profile *profile,
 406                        struct rte_mtr_error *error)
 407 {
 408         struct mlx5_priv *priv = dev->data->dev_private;
 409         struct mlx5_flow_meter_profile *fmp;
 410         union mlx5_l3t_data data;
 411         int ret;
 412
 413         if (!priv->mtr_en)
 414                 return -rte_mtr_error_set(error, ENOTSUP,
 415                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 416                                           "Meter is not supported");
 417         /* Check input params. */
 418         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 419                                                profile, error);
 420         if (ret)
 421                 return ret;
 422         /* Meter profile memory allocation. */
 423         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 424                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 425         if (fmp == NULL)
 426                 return -rte_mtr_error_set(error, ENOMEM,
 427                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 428                                           NULL, "Meter profile memory "
 429                                           "alloc failed.");
 430         /* Fill profile info. */
 431         fmp->id = meter_profile_id;
 432         fmp->profile = *profile;
 433         /* Fill the flow meter parameters for the PRM. */
 434         ret = mlx5_flow_meter_param_fill(fmp, priv, error);
 435         if (ret)
 436                 goto error;
 437         data.ptr = fmp;
 438         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 439                                  meter_profile_id, &data);
 440         if (ret)
 441                 return -rte_mtr_error_set(error, ENOTSUP,
 442                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 443                                           NULL, "Meter profile insert fail.");
 444         return 0;
 445 error:
 446         mlx5_free(fmp);
 447         return ret;
 448 }
 449
 450 /**
 451  * Callback to delete MTR profile.
 452  *
 453  * @param[in] dev
 454  *   Pointer to Ethernet device.
 455  * @param[in] meter_profile_id
 456  *   Meter profile id.
 457  * @param[out] error
 458  *   Pointer to the error structure.
 459  *
 460  * @return
 461  *   0 on success, a negative errno value otherwise and rte_errno is set.
 462  */
 463 static int
 464 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 465                           uint32_t meter_profile_id,
 466                           struct rte_mtr_error *error)
 467 {
 468         struct mlx5_priv *priv = dev->data->dev_private;
 469         struct mlx5_flow_meter_profile *fmp;
 470
 471         if (!priv->mtr_en)
 472                 return -rte_mtr_error_set(error, ENOTSUP,
 473                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 474                                           "Meter is not supported");
 475         /* Meter profile must exist. */
 476         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 477         if (fmp == NULL)
 478                 return -rte_mtr_error_set(error, ENOENT,
 479                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 480                                           &meter_profile_id,
 481                                           "Meter profile id is invalid.");
 482         /* Check profile is unused. */
 483         if (fmp->ref_cnt)
 484                 return -rte_mtr_error_set(error, EBUSY,
 485                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 486                                           NULL, "Meter profile is in use.");
 487         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 488                 return -rte_mtr_error_set(error, EBUSY,
 489                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 490                                           NULL, "Meter profile remove fail.");
 491         mlx5_free(fmp);
 492         return 0;
 493 }
 494
 495 /**
 496  * Find policy by id.
 497  *
 498  * @param[in] dev
 499  *   Pointer to Ethernet device.
 500  * @param policy_id
 501  *   Policy id.
 502  *
 503  * @return
 504  *   Pointer to the policy found on success, NULL otherwise.
 505  */
 506 struct mlx5_flow_meter_policy *
 507 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 508                             uint32_t policy_id,
 509                             uint32_t *policy_idx)
 510 {
 511         struct mlx5_priv *priv = dev->data->dev_private;
 512         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 513         union mlx5_l3t_data data;
 514
 515         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 516                 return NULL;
 517         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 518                                 !data.dword)
 519                 return NULL;
 520         if (policy_idx)
 521                 *policy_idx = data.dword;
 522         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 523                                         data.dword);
 524         /* Remove reference taken by the mlx5_l3t_get_entry. */
 525         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 526         if (sub_policy)
 527                 if (sub_policy->main_policy_id)
 528                         return sub_policy->main_policy;
 529         return NULL;
 530 }
 531
 532 /**
 533  * Get the last meter's policy from one meter's policy in hierarchy.
 534  *
 535  * @param[in] dev
 536  *   Pointer to Ethernet device.
 537  * @param[in] policy
 538  *   Pointer to flow meter policy.
 539  *
 540  * @return
 541  *   Pointer to the final meter's policy, or NULL when fail.
 542  */
 543 struct mlx5_flow_meter_policy *
 544 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 545                                         struct mlx5_flow_meter_policy *policy)
 546 {
 547         struct mlx5_priv *priv = dev->data->dev_private;
 548         struct mlx5_flow_meter_info *next_fm;
 549         struct mlx5_flow_meter_policy *next_policy = policy;
 550
 551         while (next_policy->is_hierarchy) {
 552                 next_fm = mlx5_flow_meter_find(priv,
 553                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 554                 if (!next_fm || next_fm->def_policy)
 555                         return NULL;
 556                 next_policy = mlx5_flow_meter_policy_find(dev,
 557                                                 next_fm->policy_id, NULL);
 558                 MLX5_ASSERT(next_policy);
 559         }
 560         return next_policy;
 561 }
 562
 563 /**
 564  * Callback to check MTR policy action validate
 565  *
 566  * @param[in] dev
 567  *   Pointer to Ethernet device.
 568  * @param[in] actions
 569  *   Pointer to meter policy action detail.
 570  * @param[out] error
 571  *   Pointer to the error structure.
 572  *
 573  * @return
 574  *   0 on success, a negative errno value otherwise and rte_errno is set.
 575  */
 576 static int
 577 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 578         struct rte_mtr_meter_policy_params *policy,
 579         struct rte_mtr_error *error)
 580 {
 581         struct mlx5_priv *priv = dev->data->dev_private;
 582         struct rte_flow_attr attr = { .transfer =
 583                         priv->config.dv_esw_en ? 1 : 0};
 584         bool is_rss = false;
 585         uint8_t policy_mode;
 586         uint8_t domain_bitmap;
 587         int ret;
 588
 589         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 590                 return -rte_mtr_error_set(error, ENOTSUP,
 591                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 592                                 NULL, "meter policy unsupported.");
 593         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 594                         &is_rss, &domain_bitmap, &policy_mode, error);
 595         if (ret)
 596                 return ret;
 597         return 0;
 598 }
 599
 600 static int
 601 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 602                         uint32_t policy_id,
 603                         struct mlx5_flow_meter_policy *mtr_policy,
 604                         struct rte_mtr_error *error,
 605                         bool clear_l3t)
 606 {
 607         struct mlx5_priv *priv = dev->data->dev_private;
 608         struct mlx5_flow_meter_sub_policy *sub_policy;
 609         uint32_t i, j;
 610         uint16_t sub_policy_num;
 611
 612         rte_spinlock_lock(&mtr_policy->sl);
 613         if (mtr_policy->ref_cnt) {
 614                 rte_spinlock_unlock(&mtr_policy->sl);
 615                 return -rte_mtr_error_set(error, EBUSY,
 616                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 617                                  NULL,
 618                                 "Meter policy object is being used.");
 619         }
 620         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 621         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 622         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 623                 sub_policy_num = (mtr_policy->sub_policy_num >>
 624                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 625                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 626                 if (sub_policy_num) {
 627                         for (j = 0; j < sub_policy_num; j++) {
 628                                 sub_policy = mtr_policy->sub_policys[i][j];
 629                                 if (sub_policy)
 630                                         mlx5_ipool_free
 631                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 632                                         sub_policy->idx);
 633                         }
 634                 }
 635         }
 636         if (priv->policy_idx_tbl && clear_l3t) {
 637                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 638                         rte_spinlock_unlock(&mtr_policy->sl);
 639                         return -rte_mtr_error_set(error, ENOTSUP,
 640                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 641                                 "Fail to delete policy in index table.");
 642                 }
 643         }
 644         rte_spinlock_unlock(&mtr_policy->sl);
 645         return 0;
 646 }
 647
 648 /**
 649  * Callback to add MTR policy.
 650  *
 651  * @param[in] dev
 652  *   Pointer to Ethernet device.
 653  * @param[out] policy_id
 654  *   Pointer to policy id
 655  * @param[in] actions
 656  *   Pointer to meter policy action detail.
 657  * @param[out] error
 658  *   Pointer to the error structure.
 659  *
 660  * @return
 661  *   0 on success, a negative errno value otherwise and rte_errno is set.
 662  */
 663 static int
 664 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 665                         uint32_t policy_id,
 666                         struct rte_mtr_meter_policy_params *policy,
 667                         struct rte_mtr_error *error)
 668 {
 669         struct mlx5_priv *priv = dev->data->dev_private;
 670         struct rte_flow_attr attr = { .transfer =
 671                         priv->config.dv_esw_en ? 1 : 0};
 672         uint32_t sub_policy_idx = 0;
 673         uint32_t policy_idx = 0;
 674         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 675         struct mlx5_flow_meter_sub_policy *sub_policy;
 676         bool is_rss = false;
 677         uint8_t policy_mode;
 678         uint32_t i;
 679         int ret;
 680         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 681         uint16_t sub_policy_num;
 682         uint8_t domain_bitmap = 0;
 683         union mlx5_l3t_data data;
 684         bool skip_rule = false;
 685
 686         if (!priv->mtr_en)
 687                 return -rte_mtr_error_set(error, ENOTSUP,
 688                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 689                                           NULL, "meter policy unsupported. ");
 690         if (policy_id == MLX5_INVALID_POLICY_ID)
 691                 return -rte_mtr_error_set(error, ENOTSUP,
 692                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 693                                           NULL, "policy ID is invalid. ");
 694         if (policy_id == priv->sh->mtrmng->def_policy_id)
 695                 return -rte_mtr_error_set(error, EEXIST,
 696                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 697                                           NULL, "default policy ID exists. ");
 698         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 699         if (mtr_policy)
 700                 return -rte_mtr_error_set(error, EEXIST,
 701                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 702                                           NULL, "policy ID exists. ");
 703         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 704                                           &is_rss, &domain_bitmap,
 705                                           &policy_mode, error);
 706         if (ret)
 707                 return ret;
 708         if (!domain_bitmap)
 709                 return -rte_mtr_error_set(error, ENOTSUP,
 710                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 711                                           NULL, "fail to find policy domain.");
 712         if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
 713                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 714                         return -rte_mtr_error_set(error, EEXIST,
 715                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 716                                 NULL, "a policy with similar actions "
 717                                 "is already configured");
 718                 if (mlx5_flow_create_def_policy(dev))
 719                         return -rte_mtr_error_set(error, ENOTSUP,
 720                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 721                                 NULL,
 722                                 "fail to create non-terminated policy.");
 723                 priv->sh->mtrmng->def_policy_id = policy_id;
 724                 return 0;
 725         }
 726         if (!priv->sh->meter_aso_en)
 727                 return -rte_mtr_error_set(error, ENOTSUP,
 728                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 729                         "no ASO capability to support the policy ");
 730         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 731                 if (!(domain_bitmap & (1 << i)))
 732                         continue;
 733                 /*
 734                  * If RSS is found, it means that only the ingress domain can
 735                  * be supported. It is invalid to support RSS for one color
 736                  * and egress / transfer domain actions for another. Drop and
 737                  * jump action should have no impact.
 738                  */
 739                 if (is_rss) {
 740                         policy_size +=
 741                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 742                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 743                         break;
 744                 }
 745                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 746         }
 747         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 748                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 749         if (!mtr_policy)
 750                 return -rte_mtr_error_set(error, ENOMEM,
 751                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 752                                 "Memory alloc failed for meter policy.");
 753         policy_size = sizeof(struct mlx5_flow_meter_policy);
 754         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 755                 if (!(domain_bitmap & (1 << i)))
 756                         continue;
 757                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 758                         mtr_policy->ingress = 1;
 759                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 760                         mtr_policy->egress = 1;
 761                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 762                         mtr_policy->transfer = 1;
 763                 sub_policy = mlx5_ipool_zmalloc
 764                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 765                                  &sub_policy_idx);
 766                 if (!sub_policy ||
 767                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 768                         goto policy_add_err;
 769                 sub_policy->idx = sub_policy_idx;
 770                 sub_policy->main_policy = mtr_policy;
 771                 if (!policy_idx) {
 772                         policy_idx = sub_policy_idx;
 773                         sub_policy->main_policy_id = 1;
 774                 }
 775                 mtr_policy->sub_policys[i] =
 776                         (struct mlx5_flow_meter_sub_policy **)
 777                         ((uint8_t *)mtr_policy + policy_size);
 778                 mtr_policy->sub_policys[i][0] = sub_policy;
 779                 sub_policy_num = (mtr_policy->sub_policy_num >>
 780                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 781                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 782                 sub_policy_num++;
 783                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 784                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 785                 mtr_policy->sub_policy_num |=
 786                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 787                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 788                 /*
 789                  * If RSS is found, it means that only the ingress domain can
 790                  * be supported. It is invalid to support RSS for one color
 791                  * and egress / transfer domain actions for another. Drop and
 792                  * jump action should have no impact.
 793                  */
 794                 if (is_rss) {
 795                         mtr_policy->is_rss = 1;
 796                         break;
 797                 }
 798                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 799         }
 800         rte_spinlock_init(&mtr_policy->sl);
 801         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 802                                         policy->actions, error);
 803         if (ret)
 804                 goto policy_add_err;
 805         if (mtr_policy->is_hierarchy) {
 806                 struct mlx5_flow_meter_policy *final_policy;
 807
 808                 final_policy =
 809                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 810                 if (!final_policy)
 811                         goto policy_add_err;
 812                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 813         }
 814         /*
 815          * If either Green or Yellow has queue / RSS action, all the policy
 816          * rules will be created later in the flow splitting stage.
 817          */
 818         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 819                 /* Create policy rules in HW. */
 820                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 821                 if (ret)
 822                         goto policy_add_err;
 823         }
 824         data.dword = policy_idx;
 825         if (!priv->policy_idx_tbl) {
 826                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 827                 if (!priv->policy_idx_tbl)
 828                         goto policy_add_err;
 829         }
 830         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 831                 goto policy_add_err;
 832         return 0;
 833 policy_add_err:
 834         if (mtr_policy) {
 835                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 836                         mtr_policy, error, false);
 837                 mlx5_free(mtr_policy);
 838                 if (ret)
 839                         return ret;
 840         }
 841         return -rte_mtr_error_set(error, ENOTSUP,
 842                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 843                                   NULL, "Failed to create devx policy.");
 844 }
 845
 846 /**
 847  * Callback to delete MTR policy.
 848  *
 849  * @param[in] dev
 850  *   Pointer to Ethernet device.
 851  * @param[in] policy_id
 852  *   Meter policy id.
 853  * @param[out] error
 854  *   Pointer to the error structure.
 855  *
 856  * @return
 857  *   0 on success, a negative errno value otherwise and rte_errno is set.
 858  */
 859 static int
 860 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 861                           uint32_t policy_id,
 862                           struct rte_mtr_error *error)
 863 {
 864         struct mlx5_priv *priv = dev->data->dev_private;
 865         struct mlx5_flow_meter_policy *mtr_policy;
 866         uint32_t policy_idx;
 867         int ret;
 868
 869         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 870                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 871                         return -rte_mtr_error_set(error, ENOTSUP,
 872                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 873                                 "Meter policy object is being used.");
 874                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 875                 return 0;
 876         }
 877         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 878         if (!mtr_policy)
 879                 return -rte_mtr_error_set(error, ENOTSUP,
 880                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 881                         "Meter policy id is invalid. ");
 882         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 883                                                 error, true);
 884         if (ret)
 885                 return ret;
 886         mlx5_free(mtr_policy);
 887         return 0;
 888 }
 889
 890 /**
 891  * Check meter validation.
 892  *
 893  * @param[in] priv
 894  *   Pointer to mlx5 private data structure.
 895  * @param[in] meter_id
 896  *   Meter id.
 897  * @param[in] params
 898  *   Pointer to rte meter parameters.
 899  * @param[out] error
 900  *   Pointer to rte meter error structure.
 901  *
 902  * @return
 903  *   0 on success, a negative errno value otherwise and rte_errno is set.
 904  */
 905 static int
 906 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 907                          struct rte_mtr_params *params,
 908                          struct rte_mtr_error *error)
 909 {
 910         /* Meter must use global drop action. */
 911         if (!priv->sh->dr_drop_action)
 912                 return -rte_mtr_error_set(error, ENOTSUP,
 913                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 914                                           NULL,
 915                                           "No drop action ready for meter.");
 916         /* Meter params must not be NULL. */
 917         if (params == NULL)
 918                 return -rte_mtr_error_set(error, EINVAL,
 919                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 920                                           NULL, "Meter object params null.");
 921         /* Previous meter color is not supported. */
 922         if (params->use_prev_mtr_color)
 923                 return -rte_mtr_error_set(error, ENOTSUP,
 924                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 925                                           NULL,
 926                                           "Previous meter color "
 927                                           "not supported.");
 928         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 929                 return -rte_mtr_error_set(error, ENOENT,
 930                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 931                                 NULL, "Meter policy id not valid.");
 932         /* Validate meter id. */
 933         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 934                 return -rte_mtr_error_set(error, EEXIST,
 935                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 936                         "Meter object already exists.");
 937         return 0;
 938 }
 939
 940 /**
 941  * Modify the flow meter action.
 942  *
 943  * @param[in] priv
 944  *   Pointer to mlx5 private data structure.
 945  * @param[in] fm
 946  *   Pointer to flow meter to be modified.
 947  * @param[in] srtcm
 948  *   Pointer to meter srtcm description parameter.
 949  * @param[in] modify_bits
 950  *   The bit in srtcm to be updated.
 951  * @param[in] active_state
 952  *   The state to be updated.
 953  * @return
 954  *   0 on success, o negative value otherwise.
 955  */
 956 static int
 957 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 958                 struct mlx5_flow_meter_info *fm,
 959                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 960                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 961 {
 962 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 963         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 964         uint32_t *attr;
 965         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 966         int ret;
 967         struct mlx5_aso_mtr *aso_mtr = NULL;
 968         uint32_t cbs_cir, ebs_eir, val;
 969
 970         if (priv->sh->meter_aso_en) {
 971                 fm->is_enable = !!is_enable;
 972                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 973                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 974                 if (ret)
 975                         return ret;
 976                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 977                 if (ret)
 978                         return ret;
 979         } else {
 980                 /* Fill command parameters. */
 981                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 982                 mod_attr.flow_meter_parameter = in;
 983                 mod_attr.flow_meter_parameter_sz =
 984                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 985                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 986                         mod_attr.active = !!active_state;
 987                 else
 988                         mod_attr.active = 0;
 989                 attr = in;
 990                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 991                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 992                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 993                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 994                                 ASO_DSEG_EXP_MASK;
 995                         MLX5_SET(flow_meter_parameters, attr,
 996                                 cbs_exponent, val);
 997                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 998                                 ASO_DSEG_MAN_MASK;
 999                         MLX5_SET(flow_meter_parameters, attr,
1000                                 cbs_mantissa, val);
1001                 }
1002                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1003                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
1004                                 ASO_DSEG_EXP_MASK;
1005                         MLX5_SET(flow_meter_parameters, attr,
1006                                 cir_exponent, val);
1007                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1008                         MLX5_SET(flow_meter_parameters, attr,
1009                                 cir_mantissa, val);
1010                 }
1011                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1012                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1013                                 ASO_DSEG_EXP_MASK;
1014                         MLX5_SET(flow_meter_parameters, attr,
1015                                 ebs_exponent, val);
1016                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1017                                 ASO_DSEG_MAN_MASK;
1018                         MLX5_SET(flow_meter_parameters, attr,
1019                                 ebs_mantissa, val);
1020                 }
1021                 /* Apply modifications to meter only if it was created. */
1022                 if (fm->meter_action) {
1023                         ret = mlx5_glue->dv_modify_flow_action_meter
1024                                         (fm->meter_action, &mod_attr,
1025                                         rte_cpu_to_be_64(modify_bits));
1026                         if (ret)
1027                                 return ret;
1028                 }
1029                 /* Update succeedded modify meter parameters. */
1030                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1031                         fm->active_state = !!active_state;
1032         }
1033         return 0;
1034 #else
1035         (void)priv;
1036         (void)fm;
1037         (void)srtcm;
1038         (void)modify_bits;
1039         (void)active_state;
1040         (void)is_enable;
1041         return -ENOTSUP;
1042 #endif
1043 }
1044
1045 static int
1046 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1047                                 struct mlx5_flow_meter_info *fm,
1048                                 uint64_t stats_mask)
1049 {
1050         fm->bytes_dropped =
1051                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1052         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1053         if (fm->bytes_dropped || fm->pkts_dropped) {
1054                 if (!fm->drop_cnt) {
1055                         /* Alloc policer counters. */
1056                         fm->drop_cnt = mlx5_counter_alloc(dev);
1057                         if (!fm->drop_cnt)
1058                                 return -1;
1059                 }
1060         } else {
1061                 if (fm->drop_cnt) {
1062                         mlx5_counter_free(dev, fm->drop_cnt);
1063                         fm->drop_cnt = 0;
1064                 }
1065         }
1066         return 0;
1067 }
1068
1069 /**
1070  * Create meter rules.
1071  *
1072  * @param[in] dev
1073  *   Pointer to Ethernet device.
1074  * @param[in] meter_id
1075  *   Meter id.
1076  * @param[in] params
1077  *   Pointer to rte meter parameters.
1078  * @param[in] shared
1079  *   Meter shared with other flow or not.
1080  * @param[out] error
1081  *   Pointer to rte meter error structure.
1082  *
1083  * @return
1084  *   0 on success, a negative errno value otherwise and rte_errno is set.
1085  */
1086 static int
1087 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1088                        struct rte_mtr_params *params, int shared,
1089                        struct rte_mtr_error *error)
1090 {
1091         struct mlx5_priv *priv = dev->data->dev_private;
1092         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1093         struct mlx5_flow_meter_profile *fmp;
1094         struct mlx5_flow_meter_info *fm;
1095         struct mlx5_legacy_flow_meter *legacy_fm;
1096         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1097         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1098                 .size = 0,
1099                 .trunk_size = 64,
1100                 .need_lock = 1,
1101                 .type = "mlx5_flow_mtr_flow_id_pool",
1102         };
1103         struct mlx5_aso_mtr *aso_mtr;
1104         uint32_t mtr_idx, policy_idx;
1105         union mlx5_l3t_data data;
1106         int ret;
1107         uint8_t domain_bitmap;
1108         uint8_t mtr_id_bits;
1109         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1110                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1111
1112         if (!priv->mtr_en)
1113                 return -rte_mtr_error_set(error, ENOTSUP,
1114                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1115                                         "Meter is not supported");
1116         /* Validate the parameters. */
1117         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1118         if (ret)
1119                 return ret;
1120         /* Meter profile must exist. */
1121         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1122         if (fmp == NULL)
1123                 return -rte_mtr_error_set(error, ENOENT,
1124                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1125                         NULL, "Meter profile id not valid.");
1126         /* Meter policy must exist. */
1127         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1128                 __atomic_add_fetch
1129                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1130                         1, __ATOMIC_RELAXED);
1131                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1132                 if (!priv->config.dv_esw_en)
1133                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1134         } else {
1135                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1136                                 params->meter_policy_id, &policy_idx);
1137                 if (!priv->sh->meter_aso_en)
1138                         return -rte_mtr_error_set(error, ENOTSUP,
1139                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1140                                 "Part of the policies cannot be "
1141                                 "supported without ASO ");
1142                 if (!mtr_policy)
1143                         return -rte_mtr_error_set(error, ENOENT,
1144                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1145                                 NULL, "Meter policy id not valid.");
1146                 domain_bitmap = (mtr_policy->ingress ?
1147                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1148                                 (mtr_policy->egress ?
1149                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1150                                 (mtr_policy->transfer ?
1151                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1152         }
1153         /* Allocate the flow meter memory. */
1154         if (priv->sh->meter_aso_en) {
1155                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1156                 if (!mtr_idx)
1157                         return -rte_mtr_error_set(error, ENOMEM,
1158                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1159                                 "Memory alloc failed for meter.");
1160                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1161                 fm = &aso_mtr->fm;
1162         } else {
1163                 legacy_fm = mlx5_ipool_zmalloc
1164                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1165                 if (legacy_fm == NULL)
1166                         return -rte_mtr_error_set(error, ENOMEM,
1167                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1168                                 "Memory alloc failed for meter.");
1169                 legacy_fm->idx = mtr_idx;
1170                 fm = &legacy_fm->fm;
1171         }
1172         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1173         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1174             mtr_reg_bits) {
1175                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1176                 goto error;
1177         }
1178         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1179                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1180         /* Fill the flow meter parameters. */
1181         fm->meter_id = meter_id;
1182         fm->policy_id = params->meter_policy_id;
1183         fm->profile = fmp;
1184         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1185                 goto error;
1186         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1187                 goto error;
1188         /* Add to the flow meter list. */
1189         if (!priv->sh->meter_aso_en)
1190                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1191         /* Add to the flow meter list. */
1192         fm->active_state = 1; /* Config meter starts as active. */
1193         fm->is_enable = 1;
1194         fm->shared = !!shared;
1195         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1196         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1197                 fm->def_policy = 1;
1198                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1199                 if (!fm->flow_ipool)
1200                         goto error;
1201         }
1202         rte_spinlock_init(&fm->sl);
1203         /* If ASO meter supported, update ASO flow meter by wqe. */
1204         if (priv->sh->meter_aso_en) {
1205                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1206                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1207                 if (ret)
1208                         goto error;
1209                 if (!priv->mtr_idx_tbl) {
1210                         priv->mtr_idx_tbl =
1211                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1212                         if (!priv->mtr_idx_tbl)
1213                                 goto error;
1214                 }
1215                 data.dword = mtr_idx;
1216                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1217                         goto error;
1218         }
1219         if (mtr_policy)
1220                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1221         return 0;
1222 error:
1223         mlx5_flow_destroy_mtr_tbls(dev, fm);
1224         /* Free policer counters. */
1225         if (fm->drop_cnt)
1226                 mlx5_counter_free(dev, fm->drop_cnt);
1227         if (priv->sh->meter_aso_en)
1228                 mlx5_flow_mtr_free(dev, mtr_idx);
1229         else
1230                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1231         return -rte_mtr_error_set(error, ENOTSUP,
1232                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1233                 NULL, "Failed to create devx meter.");
1234 }
1235
1236 static int
1237 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1238                         struct mlx5_flow_meter_info *fm,
1239                         uint32_t mtr_idx)
1240 {
1241         struct mlx5_priv *priv = dev->data->dev_private;
1242         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1243         struct mlx5_flow_meter_profile *fmp;
1244         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1245         struct mlx5_flow_meter_policy *mtr_policy;
1246
1247         /* Meter object must not have any owner. */
1248         MLX5_ASSERT(!fm->ref_cnt);
1249         /* Get meter profile. */
1250         fmp = fm->profile;
1251         if (fmp == NULL)
1252                 return -1;
1253         /* Update dependencies. */
1254         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1255         fm->profile = NULL;
1256         /* Remove from list. */
1257         if (!priv->sh->meter_aso_en) {
1258                 legacy_fm = container_of(fm,
1259                         struct mlx5_legacy_flow_meter, fm);
1260                 TAILQ_REMOVE(fms, legacy_fm, next);
1261         }
1262         /* Free drop counters. */
1263         if (fm->drop_cnt)
1264                 mlx5_counter_free(dev, fm->drop_cnt);
1265         /* Free meter flow table. */
1266         if (fm->flow_ipool) {
1267                 mlx5_ipool_destroy(fm->flow_ipool);
1268                 fm->flow_ipool = 0;
1269         }
1270         mlx5_flow_destroy_mtr_tbls(dev, fm);
1271         if (fm->def_policy)
1272                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1273                                 1, __ATOMIC_RELAXED);
1274         if (priv->sh->meter_aso_en) {
1275                 if (!fm->def_policy) {
1276                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1277                                                 fm->policy_id, NULL);
1278                         if (mtr_policy)
1279                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1280                                                 1, __ATOMIC_RELAXED);
1281                         fm->policy_id = 0;
1282                 }
1283                 fm->def_policy = 0;
1284                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1285                         return -1;
1286                 mlx5_flow_mtr_free(dev, mtr_idx);
1287         } else {
1288                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1289                                         legacy_fm->idx);
1290         }
1291         return 0;
1292 }
1293
1294 /**
1295  * Destroy meter rules.
1296  *
1297  * @param[in] dev
1298  *   Pointer to Ethernet device.
1299  * @param[in] meter_id
1300  *   Meter id.
1301  * @param[out] error
1302  *   Pointer to rte meter error structure.
1303  *
1304  * @return
1305  *   0 on success, a negative errno value otherwise and rte_errno is set.
1306  */
1307 static int
1308 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1309                         struct rte_mtr_error *error)
1310 {
1311         struct mlx5_priv *priv = dev->data->dev_private;
1312         struct mlx5_flow_meter_info *fm;
1313         uint32_t mtr_idx = 0;
1314
1315         if (!priv->mtr_en)
1316                 return -rte_mtr_error_set(error, ENOTSUP,
1317                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1318                                           NULL,
1319                                           "Meter is not supported");
1320         /* Meter object must exist. */
1321         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1322         if (fm == NULL)
1323                 return -rte_mtr_error_set(error, ENOENT,
1324                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1325                                           NULL,
1326                                           "Meter object id not valid.");
1327         /* Meter object must not have any owner. */
1328         if (fm->ref_cnt > 0)
1329                 return -rte_mtr_error_set(error, EBUSY,
1330                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1331                                           NULL,
1332                                           "Meter object is being used.");
1333         /* Destroy the meter profile. */
1334         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1335                 return -rte_mtr_error_set(error, EINVAL,
1336                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1337                                         NULL,
1338                                         "MTR object meter profile invalid.");
1339         return 0;
1340 }
1341
1342 /**
1343  * Modify meter state.
1344  *
1345  * @param[in] priv
1346  *   Pointer to mlx5 private data structure.
1347  * @param[in] fm
1348  *   Pointer to flow meter.
1349  * @param[in] new_state
1350  *   New state to update.
1351  * @param[out] error
1352  *   Pointer to rte meter error structure.
1353  *
1354  * @return
1355  *   0 on success, a negative errno value otherwise and rte_errno is set.
1356  */
1357 static int
1358 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1359                              struct mlx5_flow_meter_info *fm,
1360                              uint32_t new_state,
1361                              struct rte_mtr_error *error)
1362 {
1363         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1364                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1365                 .ebs_eir = 0,
1366         };
1367         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1368                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1369         int ret;
1370
1371         if (new_state == MLX5_FLOW_METER_DISABLE)
1372                 ret = mlx5_flow_meter_action_modify(priv, fm,
1373                                 &srtcm, modify_bits, 0, 0);
1374         else
1375                 ret = mlx5_flow_meter_action_modify(priv, fm,
1376                                                    &fm->profile->srtcm_prm,
1377                                                     modify_bits, 0, 1);
1378         if (ret)
1379                 return -rte_mtr_error_set(error, -ret,
1380                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1381                                           NULL,
1382                                           new_state ?
1383                                           "Failed to enable meter." :
1384                                           "Failed to disable meter.");
1385         return 0;
1386 }
1387
1388 /**
1389  * Callback to enable flow meter.
1390  *
1391  * @param[in] dev
1392  *   Pointer to Ethernet device.
1393  * @param[in] meter_id
1394  *   Meter id.
1395  * @param[out] error
1396  *   Pointer to rte meter error structure.
1397  *
1398  * @return
1399  *   0 on success, a negative errno value otherwise and rte_errno is set.
1400  */
1401 static int
1402 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1403                        uint32_t meter_id,
1404                        struct rte_mtr_error *error)
1405 {
1406         struct mlx5_priv *priv = dev->data->dev_private;
1407         struct mlx5_flow_meter_info *fm;
1408         int ret;
1409
1410         if (!priv->mtr_en)
1411                 return -rte_mtr_error_set(error, ENOTSUP,
1412                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1413                                           "Meter is not supported");
1414         /* Meter object must exist. */
1415         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1416         if (fm == NULL)
1417                 return -rte_mtr_error_set(error, ENOENT,
1418                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1419                                           NULL, "Meter not found.");
1420         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1421                 return 0;
1422         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1423                                            error);
1424         if (!ret)
1425                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1426         return ret;
1427 }
1428
1429 /**
1430  * Callback to disable flow meter.
1431  *
1432  * @param[in] dev
1433  *   Pointer to Ethernet device.
1434  * @param[in] meter_id
1435  *   Meter id.
1436  * @param[out] error
1437  *   Pointer to rte meter error structure.
1438  *
1439  * @return
1440  *   0 on success, a negative errno value otherwise and rte_errno is set.
1441  */
1442 static int
1443 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1444                         uint32_t meter_id,
1445                         struct rte_mtr_error *error)
1446 {
1447         struct mlx5_priv *priv = dev->data->dev_private;
1448         struct mlx5_flow_meter_info *fm;
1449         int ret;
1450
1451         if (!priv->mtr_en)
1452                 return -rte_mtr_error_set(error, ENOTSUP,
1453                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1454                                           "Meter is not supported");
1455         /* Meter object must exist. */
1456         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1457         if (fm == NULL)
1458                 return -rte_mtr_error_set(error, ENOENT,
1459                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1460                                           NULL, "Meter not found.");
1461         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1462                 return 0;
1463         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1464                                            error);
1465         if (!ret)
1466                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1467         return ret;
1468 }
1469
1470 /**
1471  * Callback to update meter profile.
1472  *
1473  * @param[in] dev
1474  *   Pointer to Ethernet device.
1475  * @param[in] meter_id
1476  *   Meter id.
1477  * @param[in] meter_profile_id
1478  *   To be updated meter profile id.
1479  * @param[out] error
1480  *   Pointer to rte meter error structure.
1481  *
1482  * @return
1483  *   0 on success, a negative errno value otherwise and rte_errno is set.
1484  */
1485 static int
1486 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1487                                uint32_t meter_id,
1488                                uint32_t meter_profile_id,
1489                                struct rte_mtr_error *error)
1490 {
1491         struct mlx5_priv *priv = dev->data->dev_private;
1492         struct mlx5_flow_meter_profile *fmp;
1493         struct mlx5_flow_meter_profile *old_fmp;
1494         struct mlx5_flow_meter_info *fm;
1495         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1496                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1497         int ret;
1498
1499         if (!priv->mtr_en)
1500                 return -rte_mtr_error_set(error, ENOTSUP,
1501                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1502                                           "Meter is not supported");
1503         /* Meter profile must exist. */
1504         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1505         if (fmp == NULL)
1506                 return -rte_mtr_error_set(error, ENOENT,
1507                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1508                                           NULL, "Meter profile not found.");
1509         /* Meter object must exist. */
1510         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1511         if (fm == NULL)
1512                 return -rte_mtr_error_set(error, ENOENT,
1513                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1514                                           NULL, "Meter not found.");
1515         /* MTR object already set to meter profile id. */
1516         old_fmp = fm->profile;
1517         if (fmp == old_fmp)
1518                 return 0;
1519         /* Update the profile. */
1520         fm->profile = fmp;
1521         /* Update meter params in HW (if not disabled). */
1522         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1523                 return 0;
1524         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1525                                               modify_bits, fm->active_state, 1);
1526         if (ret) {
1527                 fm->profile = old_fmp;
1528                 return -rte_mtr_error_set(error, -ret,
1529                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1530                                           NULL, "Failed to update meter"
1531                                           " parmeters in hardware.");
1532         }
1533         old_fmp->ref_cnt--;
1534         fmp->ref_cnt++;
1535         return 0;
1536 }
1537
1538 /**
1539  * Callback to update meter stats mask.
1540  *
1541  * @param[in] dev
1542  *   Pointer to Ethernet device.
1543  * @param[in] meter_id
1544  *   Meter id.
1545  * @param[in] stats_mask
1546  *   To be updated stats_mask.
1547  * @param[out] error
1548  *   Pointer to rte meter error structure.
1549  *
1550  * @return
1551  *   0 on success, a negative errno value otherwise and rte_errno is set.
1552  */
1553 static int
1554 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1555                              uint32_t meter_id,
1556                              uint64_t stats_mask,
1557                              struct rte_mtr_error *error)
1558 {
1559         struct mlx5_priv *priv = dev->data->dev_private;
1560         struct mlx5_flow_meter_info *fm;
1561
1562         if (!priv->mtr_en)
1563                 return -rte_mtr_error_set(error, ENOTSUP,
1564                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1565                                           "Meter is not supported");
1566         /* Meter object must exist. */
1567         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1568         if (fm == NULL)
1569                 return -rte_mtr_error_set(error, ENOENT,
1570                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1571                                           NULL, "Meter object id not valid.");
1572         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1573                 return -rte_mtr_error_set(error, ENOENT,
1574                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1575                                           NULL, "Fail to allocate "
1576                                           "counter for meter.");
1577         return 0;
1578 }
1579
1580 /**
1581  * Callback to read meter statistics.
1582  *
1583  * @param[in] dev
1584  *   Pointer to Ethernet device.
1585  * @param[in] meter_id
1586  *   Meter id.
1587  * @param[out] stats
1588  *   Pointer to store the statistics.
1589  * @param[out] stats_mask
1590  *   Pointer to store the stats_mask.
1591  * @param[in] clear
1592  *   Statistic to be cleared after read or not.
1593  * @param[out] error
1594  *   Pointer to rte meter error structure.
1595  *
1596  * @return
1597  *   0 on success, a negative errno value otherwise and rte_errno is set.
1598  */
1599 static int
1600 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1601                            uint32_t meter_id,
1602                            struct rte_mtr_stats *stats,
1603                            uint64_t *stats_mask,
1604                            int clear,
1605                            struct rte_mtr_error *error)
1606 {
1607         struct mlx5_priv *priv = dev->data->dev_private;
1608         struct mlx5_flow_meter_info *fm;
1609         uint64_t pkts;
1610         uint64_t bytes;
1611         int ret = 0;
1612
1613         if (!priv->mtr_en)
1614                 return -rte_mtr_error_set(error, ENOTSUP,
1615                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1616                                           "Meter is not supported");
1617         /* Meter object must exist. */
1618         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1619         if (fm == NULL)
1620                 return -rte_mtr_error_set(error, ENOENT,
1621                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1622                                           NULL, "Meter object id not valid.");
1623         *stats_mask = 0;
1624         if (fm->bytes_dropped)
1625                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1626         if (fm->pkts_dropped)
1627                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1628         memset(stats, 0, sizeof(*stats));
1629         if (fm->drop_cnt) {
1630                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1631                                                  &bytes);
1632                 if (ret)
1633                         goto error;
1634                 /* If need to read the packets, set it. */
1635                 if (fm->pkts_dropped)
1636                         stats->n_pkts_dropped = pkts;
1637                 /* If need to read the bytes, set it. */
1638                 if (fm->bytes_dropped)
1639                         stats->n_bytes_dropped = bytes;
1640         }
1641         return 0;
1642 error:
1643         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1644                                  "Failed to read meter drop counters.");
1645 }
1646
1647 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1648         .capabilities_get = mlx5_flow_mtr_cap_get,
1649         .meter_profile_add = mlx5_flow_meter_profile_add,
1650         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1651         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1652         .meter_policy_add = mlx5_flow_meter_policy_add,
1653         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1654         .create = mlx5_flow_meter_create,
1655         .destroy = mlx5_flow_meter_destroy,
1656         .meter_enable = mlx5_flow_meter_enable,
1657         .meter_disable = mlx5_flow_meter_disable,
1658         .meter_profile_update = mlx5_flow_meter_profile_update,
1659         .meter_dscp_table_update = NULL,
1660         .stats_update = mlx5_flow_meter_stats_update,
1661         .stats_read = mlx5_flow_meter_stats_read,
1662 };
1663
1664 /**
1665  * Get meter operations.
1666  *
1667  * @param dev
1668  *   Pointer to Ethernet device structure.
1669  * @param arg
1670  *   Pointer to set the mtr operations.
1671  *
1672  * @return
1673  *   Always 0.
1674  */
1675 int
1676 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1677 {
1678         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1679         return 0;
1680 }
1681
1682 /**
1683  * Find meter by id.
1684  *
1685  * @param priv
1686  *   Pointer to mlx5_priv.
1687  * @param meter_id
1688  *   Meter id.
1689  * @param mtr_idx
1690  *   Pointer to Meter index.
1691  *
1692  * @return
1693  *   Pointer to the meter info found on success, NULL otherwise.
1694  */
1695 struct mlx5_flow_meter_info *
1696 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1697                 uint32_t *mtr_idx)
1698 {
1699         struct mlx5_legacy_flow_meter *legacy_fm;
1700         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1701         struct mlx5_aso_mtr *aso_mtr;
1702         struct mlx5_aso_mtr_pools_mng *pools_mng =
1703                                 &priv->sh->mtrmng->pools_mng;
1704         union mlx5_l3t_data data;
1705
1706         if (priv->sh->meter_aso_en) {
1707                 rte_spinlock_lock(&pools_mng->mtrsl);
1708                 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1709                         rte_spinlock_unlock(&pools_mng->mtrsl);
1710                         return NULL;
1711                 }
1712                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1713                         !data.dword) {
1714                         rte_spinlock_unlock(&pools_mng->mtrsl);
1715                         return NULL;
1716                 }
1717                 if (mtr_idx)
1718                         *mtr_idx = data.dword;
1719                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1720                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1721                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1722                 rte_spinlock_unlock(&pools_mng->mtrsl);
1723                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1724                         return NULL;
1725                 return &aso_mtr->fm;
1726         }
1727         TAILQ_FOREACH(legacy_fm, fms, next)
1728                 if (meter_id == legacy_fm->fm.meter_id) {
1729                         if (mtr_idx)
1730                                 *mtr_idx = legacy_fm->idx;
1731                         return &legacy_fm->fm;
1732                 }
1733         return NULL;
1734 }
1735
1736 /**
1737  * Find meter by index.
1738  *
1739  * @param priv
1740  *   Pointer to mlx5_priv.
1741  * @param idx
1742  *   Meter index.
1743  *
1744  * @return
1745  *   Pointer to the meter info found on success, NULL otherwise.
1746  */
1747 struct mlx5_flow_meter_info *
1748 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1749 {
1750         struct mlx5_aso_mtr *aso_mtr;
1751
1752         if (priv->sh->meter_aso_en) {
1753                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1754                 if (!aso_mtr)
1755                         return NULL;
1756                 return &aso_mtr->fm;
1757         } else {
1758                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1759         }
1760 }
1761
1762 /**
1763  * Attach meter to flow.
1764  * Unidirectional Meter creation can only be done
1765  * when flow direction is known, i.e. when calling meter_attach.
1766  *
1767  * @param [in] priv
1768  *  Pointer to mlx5 private data.
1769  * @param[in] fm
1770  *   Pointer to flow meter.
1771  * @param [in] attr
1772  *  Pointer to flow attributes.
1773  * @param [out] error
1774  *  Pointer to error structure.
1775  *
1776  * @return
1777  *   0 on success, a negative errno value otherwise and rte_errno is set.
1778  */
1779 int
1780 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1781                        struct mlx5_flow_meter_info *fm,
1782                        const struct rte_flow_attr *attr,
1783                        struct rte_flow_error *error)
1784 {
1785         int ret = 0;
1786
1787         if (priv->sh->meter_aso_en) {
1788                 struct mlx5_aso_mtr *aso_mtr;
1789
1790                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1791                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1792                         return rte_flow_error_set(error, ENOENT,
1793                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1794                                         NULL,
1795                                         "Timeout in meter configuration");
1796                 }
1797                 rte_spinlock_lock(&fm->sl);
1798                 if (fm->shared || !fm->ref_cnt) {
1799                         fm->ref_cnt++;
1800                 } else {
1801                         rte_flow_error_set(error, EINVAL,
1802                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1803                                    "Meter cannot be shared");
1804                         ret = -1;
1805                 }
1806                 rte_spinlock_unlock(&fm->sl);
1807         } else {
1808                 rte_spinlock_lock(&fm->sl);
1809                 if (fm->meter_action) {
1810                         if (fm->shared &&
1811                             attr->transfer == fm->transfer &&
1812                             attr->ingress == fm->ingress &&
1813                             attr->egress == fm->egress) {
1814                                 fm->ref_cnt++;
1815                         } else {
1816                                 rte_flow_error_set(error, EINVAL,
1817                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1818                                         fm->shared ?
1819                                         "Meter attr not match." :
1820                                         "Meter cannot be shared.");
1821                                 ret = -1;
1822                         }
1823                 } else {
1824                         fm->ingress = attr->ingress;
1825                         fm->egress = attr->egress;
1826                         fm->transfer = attr->transfer;
1827                         fm->ref_cnt = 1;
1828                         /* This also creates the meter object. */
1829                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1830                                                                          fm);
1831                         if (!fm->meter_action) {
1832                                 fm->ref_cnt = 0;
1833                                 fm->ingress = 0;
1834                                 fm->egress = 0;
1835                                 fm->transfer = 0;
1836                                 rte_flow_error_set(error, EINVAL,
1837                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1838                                         "Meter action create failed.");
1839                                 ret = -1;
1840                         }
1841                 }
1842                 rte_spinlock_unlock(&fm->sl);
1843         }
1844         return ret ? -rte_errno : 0;
1845 }
1846
1847 /**
1848  * Detach meter from flow.
1849  *
1850  * @param [in] priv
1851  *  Pointer to mlx5 private data.
1852  * @param [in] fm
1853  *  Pointer to flow meter.
1854  */
1855 void
1856 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1857                        struct mlx5_flow_meter_info *fm)
1858 {
1859 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1860         rte_spinlock_lock(&fm->sl);
1861         MLX5_ASSERT(fm->ref_cnt);
1862         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1863                 mlx5_glue->destroy_flow_action(fm->meter_action);
1864                 fm->meter_action = NULL;
1865                 fm->ingress = 0;
1866                 fm->egress = 0;
1867                 fm->transfer = 0;
1868         }
1869         rte_spinlock_unlock(&fm->sl);
1870 #else
1871         (void)priv;
1872         (void)fm;
1873 #endif
1874 }
1875
1876 /**
1877  * Flush meter with Rx queue configuration.
1878  *
1879  * @param[in] dev
1880  *   Pointer to Ethernet device.
1881  */
1882 void
1883 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1884 {
1885         struct mlx5_priv *priv = dev->data->dev_private;
1886         struct mlx5_flow_meter_sub_policy *sub_policy;
1887         struct mlx5_flow_meter_policy *mtr_policy;
1888         void *entry;
1889         uint32_t i, policy_idx;
1890
1891         if (!priv->mtr_en)
1892                 return;
1893         if (priv->policy_idx_tbl) {
1894                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1895                         policy_idx = *(uint32_t *)entry;
1896                         sub_policy = mlx5_ipool_get
1897                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1898                                 policy_idx);
1899                         if (!sub_policy || !sub_policy->main_policy)
1900                                 continue;
1901                         mtr_policy = sub_policy->main_policy;
1902                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1903                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1904                                         mtr_policy);
1905                 }
1906         }
1907 }
1908
1909 /**
1910  * Iterate a meter hierarchy and flush all meters and policies if possible.
1911  *
1912  * @param[in] dev
1913  *   Pointer to Ethernet device.
1914  * @param[in] fm
1915  *   Pointer to flow meter.
1916  * @param[in] mtr_idx
1917  *   .Meter's index
1918  * @param[out] error
1919  *   Pointer to rte meter error structure.
1920  *
1921  * @return
1922  *   0 on success, a negative errno value otherwise and rte_errno is set.
1923  */
1924 static int
1925 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
1926                                 struct mlx5_flow_meter_info *fm,
1927                                 uint32_t mtr_idx,
1928                                 struct rte_mtr_error *error)
1929 {
1930         struct mlx5_priv *priv = dev->data->dev_private;
1931         struct mlx5_flow_meter_policy *policy;
1932         uint32_t policy_id;
1933         struct mlx5_flow_meter_info *next_fm;
1934         uint32_t next_mtr_idx;
1935         struct mlx5_flow_meter_policy *next_policy = NULL;
1936
1937         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
1938         MLX5_ASSERT(policy);
1939         while (!fm->ref_cnt && policy->is_hierarchy) {
1940                 policy_id = fm->policy_id;
1941                 next_fm = mlx5_flow_meter_find(priv,
1942                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
1943                                 &next_mtr_idx);
1944                 if (next_fm) {
1945                         next_policy = mlx5_flow_meter_policy_find(dev,
1946                                                         next_fm->policy_id,
1947                                                         NULL);
1948                         MLX5_ASSERT(next_policy);
1949                 }
1950                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1951                         return -rte_mtr_error_set(error, ENOTSUP,
1952                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
1953                                                 NULL,
1954                                                 "Failed to flush meter.");
1955                 if (policy->ref_cnt)
1956                         break;
1957                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
1958                                                 policy, error, true))
1959                         return -rte_errno;
1960                 mlx5_free(policy);
1961                 if (!next_fm || !next_policy)
1962                         break;
1963                 fm = next_fm;
1964                 mtr_idx = next_mtr_idx;
1965                 policy = next_policy;
1966         }
1967         return 0;
1968 }
1969
1970 /**
1971  * Flush all the hierarchy meters and their policies.
1972  *
1973  * @param[in] dev
1974  *   Pointer to Ethernet device.
1975  * @param[out] error
1976  *   Pointer to rte meter error structure.
1977  *
1978  * @return
1979  *   0 on success, a negative errno value otherwise and rte_errno is set.
1980  */
1981 static int
1982 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
1983                                       struct rte_mtr_error *error)
1984 {
1985         struct mlx5_priv *priv = dev->data->dev_private;
1986         struct mlx5_flow_meter_info *fm;
1987         struct mlx5_flow_meter_policy *policy;
1988         struct mlx5_flow_meter_sub_policy *sub_policy;
1989         struct mlx5_flow_meter_info *next_fm;
1990         struct mlx5_aso_mtr *aso_mtr;
1991         uint32_t mtr_idx = 0;
1992         uint32_t i, policy_idx;
1993         void *entry;
1994
1995         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
1996                 return 0;
1997         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
1998                 mtr_idx = *(uint32_t *)entry;
1999                 if (!mtr_idx)
2000                         continue;
2001                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2002                 fm = &aso_mtr->fm;
2003                 if (fm->ref_cnt || fm->def_policy)
2004                         continue;
2005                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2006                         return -rte_errno;
2007         }
2008         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2009                 policy_idx = *(uint32_t *)entry;
2010                 sub_policy = mlx5_ipool_get
2011                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2012                                 policy_idx);
2013                 if (!sub_policy)
2014                         return -rte_mtr_error_set(error,
2015                                         EINVAL,
2016                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2017                                         NULL, "Meter policy invalid.");
2018                 policy = sub_policy->main_policy;
2019                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2020                         continue;
2021                 next_fm = mlx5_flow_meter_find(priv,
2022                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2023                                 &mtr_idx);
2024                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2025                                                     error, true))
2026                         return -rte_mtr_error_set(error,
2027                                         EINVAL,
2028                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2029                                         NULL, "Meter policy invalid.");
2030                 mlx5_free(policy);
2031                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2032                         continue;
2033                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2034                                                     mtr_idx, error))
2035                         return -rte_errno;
2036         }
2037         return 0;
2038 }
2039 /**
2040  * Flush meter configuration.
2041  *
2042  * @param[in] dev
2043  *   Pointer to Ethernet device.
2044  * @param[out] error
2045  *   Pointer to rte meter error structure.
2046  *
2047  * @return
2048  *   0 on success, a negative errno value otherwise and rte_errno is set.
2049  */
2050 int
2051 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2052 {
2053         struct mlx5_priv *priv = dev->data->dev_private;
2054         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2055         struct mlx5_flow_meter_profile *fmp;
2056         struct mlx5_legacy_flow_meter *legacy_fm;
2057         struct mlx5_flow_meter_info *fm;
2058         struct mlx5_flow_meter_sub_policy *sub_policy;
2059         void *tmp;
2060         uint32_t i, mtr_idx, policy_idx;
2061         void *entry;
2062         struct mlx5_aso_mtr *aso_mtr;
2063
2064         if (!priv->mtr_en)
2065                 return 0;
2066         if (priv->sh->meter_aso_en) {
2067                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2068                         return -rte_errno;
2069                 if (priv->mtr_idx_tbl) {
2070                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2071                                 mtr_idx = *(uint32_t *)entry;
2072                                 if (mtr_idx) {
2073                                         aso_mtr =
2074                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2075                                         fm = &aso_mtr->fm;
2076                                         (void)mlx5_flow_meter_params_flush(dev,
2077                                                 fm, mtr_idx);
2078                                 }
2079                         }
2080                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2081                         priv->mtr_idx_tbl = NULL;
2082                 }
2083         } else {
2084                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2085                         fm = &legacy_fm->fm;
2086                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2087                                 return -rte_mtr_error_set(error, EINVAL,
2088                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2089                                 NULL, "MTR object meter profile invalid.");
2090                 }
2091         }
2092         if (priv->policy_idx_tbl) {
2093                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2094                         policy_idx = *(uint32_t *)entry;
2095                         sub_policy = mlx5_ipool_get
2096                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2097                                 policy_idx);
2098                         if (!sub_policy)
2099                                 return -rte_mtr_error_set(error,
2100                                                 EINVAL,
2101                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2102                                                 NULL, "MTR object "
2103                                                 "meter policy invalid.");
2104                         if (__mlx5_flow_meter_policy_delete(dev, i,
2105                                                 sub_policy->main_policy,
2106                                                 error, true))
2107                                 return -rte_mtr_error_set(error,
2108                                                 EINVAL,
2109                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2110                                                 NULL, "MTR object "
2111                                                 "meter policy invalid.");
2112                         mlx5_free(sub_policy->main_policy);
2113                 }
2114                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2115                 priv->policy_idx_tbl = NULL;
2116         }
2117         if (priv->mtr_profile_tbl) {
2118                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2119                         fmp = entry;
2120                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2121                                                            error))
2122                                 return -rte_mtr_error_set(error, EINVAL,
2123                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2124                                                 NULL, "Fail to destroy "
2125                                                 "meter profile.");
2126                 }
2127                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2128                 priv->mtr_profile_tbl = NULL;
2129         }
2130         /* Delete default policy table. */
2131         mlx5_flow_destroy_def_policy(dev);
2132         if (priv->sh->refcnt == 1)
2133                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2134         return 0;
2135 }