drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
  95         struct mlx5_flow_meter_profile *fmp;
  96
  97         TAILQ_FOREACH(fmp, fmps, next)
  98                 if (meter_profile_id == fmp->id)
  99                         return fmp;
 100         return NULL;
 101 }
 102
 103 /**
 104  * Validate the MTR profile.
 105  *
 106  * @param[in] dev
 107  *   Pointer to Ethernet device.
 108  * @param[in] meter_profile_id
 109  *   Meter profile id.
 110  * @param[in] profile
 111  *   Pointer to meter profile detail.
 112  * @param[out] error
 113  *   Pointer to the error structure.
 114  *
 115  * @return
 116  *   0 on success, a negative errno value otherwise and rte_errno is set.
 117  */
 118 static int
 119 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 120                                  uint32_t meter_profile_id,
 121                                  struct rte_mtr_meter_profile *profile,
 122                                  struct rte_mtr_error *error)
 123 {
 124         struct mlx5_priv *priv = dev->data->dev_private;
 125         struct mlx5_flow_meter_profile *fmp;
 126
 127         /* Profile must not be NULL. */
 128         if (profile == NULL)
 129                 return -rte_mtr_error_set(error, EINVAL,
 130                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 131                                           NULL, "Meter profile is null.");
 132         /* Meter profile ID must be valid. */
 133         if (meter_profile_id == UINT32_MAX)
 134                 return -rte_mtr_error_set(error, EINVAL,
 135                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 136                                           NULL, "Meter profile id not valid.");
 137         /* Meter profile must not exist. */
 138         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 139         if (fmp)
 140                 return -rte_mtr_error_set(error, EEXIST,
 141                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 142                                           NULL,
 143                                           "Meter profile already exists.");
 144         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 145                 if (priv->config.hca_attr.qos.flow_meter_old) {
 146                         /* Verify support for flow meter parameters. */
 147                         if (priv->sh->meter_aso_en && profile->packet_mode) {
 148                                 if (profile->srtcm_rfc2697.cir > 0 &&
 149                                         (profile->srtcm_rfc2697.cir <<
 150                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 151                                         <= MLX5_SRTCM_CIR_MAX &&
 152                                         profile->srtcm_rfc2697.cbs > 0 &&
 153                                         (profile->srtcm_rfc2697.cbs <<
 154                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 155                                         <= MLX5_SRTCM_CBS_MAX &&
 156                                         (profile->srtcm_rfc2697.ebs <<
 157                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 158                                         <= MLX5_SRTCM_EBS_MAX)
 159                                         return 0;
 160                                 return -rte_mtr_error_set
 161                                              (error, ENOTSUP,
 162                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 163                                               NULL,
 164                                               profile->srtcm_rfc2697.ebs ?
 165                                               "Metering value ebs must be 0." :
 166                                               "Invalid metering parameters.");
 167                         }
 168                         if (profile->srtcm_rfc2697.cir > 0 &&
 169                                 profile->srtcm_rfc2697.cir <=
 170                                                 MLX5_SRTCM_CIR_MAX &&
 171                                 profile->srtcm_rfc2697.cbs > 0 &&
 172                                 profile->srtcm_rfc2697.cbs <=
 173                                                 MLX5_SRTCM_CBS_MAX &&
 174                                 profile->srtcm_rfc2697.ebs <=
 175                                                 MLX5_SRTCM_EBS_MAX)
 176                                 return 0;
 177                         return -rte_mtr_error_set(error, ENOTSUP,
 178                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 179                                         NULL,
 180                                         profile->srtcm_rfc2697.ebs ?
 181                                         "Metering value ebs must be 0." :
 182                                         "Invalid metering parameters.");
 183                 }
 184         }
 185         return -rte_mtr_error_set(error, ENOTSUP,
 186                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 187                                   NULL, "Metering algorithm not supported.");
 188 }
 189
 190 /**
 191  * Calculate mantissa and exponent for cir.
 192  *
 193  * @param[in] cir
 194  *   Value to be calculated.
 195  * @param[out] man
 196  *   Pointer to the mantissa.
 197  * @param[out] exp
 198  *   Pointer to the exp.
 199  */
 200 static void
 201 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 202 {
 203         int64_t _cir;
 204         int64_t delta = INT64_MAX;
 205         uint8_t _man = 0;
 206         uint8_t _exp = 0;
 207         uint64_t m, e;
 208
 209         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 210                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 211                         _cir = (1000000000ULL * m) >> e;
 212                         if (llabs(cir - _cir) <= delta) {
 213                                 delta = llabs(cir - _cir);
 214                                 _man = m;
 215                                 _exp = e;
 216                         }
 217                 }
 218         }
 219         *man = _man;
 220         *exp = _exp;
 221 }
 222
 223 /**
 224  * Calculate mantissa and exponent for xbs.
 225  *
 226  * @param[in] xbs
 227  *   Value to be calculated.
 228  * @param[out] man
 229  *   Pointer to the mantissa.
 230  * @param[out] exp
 231  *   Pointer to the exp.
 232  */
 233 static void
 234 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 235 {
 236         int _exp;
 237         double _man;
 238
 239         /* Special case xbs == 0 ? both exp and matissa are 0. */
 240         if (xbs == 0) {
 241                 *man = 0;
 242                 *exp = 0;
 243                 return;
 244         }
 245         /* xbs = xbs_mantissa * 2^xbs_exponent */
 246         _man = frexp(xbs, &_exp);
 247         _man = _man * pow(2, MLX5_MAN_WIDTH);
 248         _exp = _exp - MLX5_MAN_WIDTH;
 249         *man = (uint8_t)ceil(_man);
 250         *exp = _exp;
 251 }
 252
 253 /**
 254  * Fill the prm meter parameter.
 255  *
 256  * @param[in,out] fmp
 257  *   Pointer to meter profie to be converted.
 258  * @param[out] error
 259  *   Pointer to the error structure.
 260  *
 261  * @return
 262  *   0 on success, a negative errno value otherwise and rte_errno is set.
 263  */
 264 static int
 265 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 266                         struct mlx5_priv *priv, struct rte_mtr_error *error)
 267 {
 268         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 269         uint8_t man, exp;
 270         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 271         uint32_t ebs_exp, ebs_man;
 272         uint64_t cir, cbs, ebs;
 273
 274         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 275                 return -rte_mtr_error_set(error, ENOTSUP,
 276                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 277                                 NULL, "Metering algorithm not supported.");
 278         if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
 279                 return -rte_mtr_error_set(error, ENOTSUP,
 280                         RTE_MTR_ERROR_TYPE_METER_PROFILE,
 281                         NULL, "Metering algorithm packet mode not supported.");
 282         if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
 283                 cir = fmp->profile.srtcm_rfc2697.cir <<
 284                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 285                 cbs = fmp->profile.srtcm_rfc2697.cbs <<
 286                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 287                 ebs = fmp->profile.srtcm_rfc2697.ebs <<
 288                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 289         } else {
 290                 cir = fmp->profile.srtcm_rfc2697.cir;
 291                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 292                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 293         }
 294         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 295         mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
 296         /* Check if cir mantissa is too large. */
 297         if (exp > ASO_DSEG_CIR_EXP_MASK)
 298                 return -rte_mtr_error_set(error, ENOTSUP,
 299                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 300                                           "meter profile parameter cir is"
 301                                           " not supported.");
 302         cir_man = man;
 303         cir_exp = exp;
 304          /* cbs = cbs_mantissa * 2^cbs_exponent */
 305         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 306         /* Check if cbs mantissa is too large. */
 307         if (exp > ASO_DSEG_EXP_MASK)
 308                 return -rte_mtr_error_set(error, ENOTSUP,
 309                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 310                                           "meter profile parameter cbs is"
 311                                           " not supported.");
 312         cbs_man = man;
 313         cbs_exp = exp;
 314         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 315                                 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 316                                 cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 317                                 cir_man);
 318         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 319         /* Check if ebs mantissa is too large. */
 320         if (exp > ASO_DSEG_EXP_MASK)
 321                 return -rte_mtr_error_set(error, ENOTSUP,
 322                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 323                                           "meter profile parameter ebs is"
 324                                           " not supported.");
 325         ebs_man = man;
 326         ebs_exp = exp;
 327         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 328                                         ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 329         return 0;
 330 }
 331
 332 /**
 333  * Callback to get MTR capabilities.
 334  *
 335  * @param[in] dev
 336  *   Pointer to Ethernet device.
 337  * @param[out] cap
 338  *   Pointer to save MTR capabilities.
 339  * @param[out] error
 340  *   Pointer to the error structure.
 341  *
 342  * @return
 343  *   0 on success, a negative errno value otherwise and rte_errno is set.
 344  */
 345 static int
 346 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 347                  struct rte_mtr_capabilities *cap,
 348                  struct rte_mtr_error *error __rte_unused)
 349 {
 350         struct mlx5_priv *priv = dev->data->dev_private;
 351         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 352
 353         if (!priv->mtr_en)
 354                 return -rte_mtr_error_set(error, ENOTSUP,
 355                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 356                                           "Meter is not supported");
 357         memset(cap, 0, sizeof(*cap));
 358         if (priv->sh->meter_aso_en) {
 359                 /* 2 meters per one ASO cache line. */
 360                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 361                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 362         } else {
 363                 cap->n_max = 1 << qattr->log_max_flow_meter;
 364                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 365         }
 366         cap->srtcm_rfc2697_byte_mode_supported = 1;
 367         cap->n_shared_max = cap->n_max;
 368         cap->identical = 1;
 369         cap->shared_identical = 1;
 370         cap->shared_n_flows_per_mtr_max = 4 << 20;
 371         /* 2M flows can share the same meter. */
 372         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 373         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 374         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 375         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 376                           RTE_MTR_STATS_N_PKTS_DROPPED;
 377         return 0;
 378 }
 379
 380 /**
 381  * Callback to add MTR profile.
 382  *
 383  * @param[in] dev
 384  *   Pointer to Ethernet device.
 385  * @param[in] meter_profile_id
 386  *   Meter profile id.
 387  * @param[in] profile
 388  *   Pointer to meter profile detail.
 389  * @param[out] error
 390  *   Pointer to the error structure.
 391  *
 392  * @return
 393  *   0 on success, a negative errno value otherwise and rte_errno is set.
 394  */
 395 static int
 396 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 397                        uint32_t meter_profile_id,
 398                        struct rte_mtr_meter_profile *profile,
 399                        struct rte_mtr_error *error)
 400 {
 401         struct mlx5_priv *priv = dev->data->dev_private;
 402         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
 403         struct mlx5_flow_meter_profile *fmp;
 404         int ret;
 405
 406         if (!priv->mtr_en)
 407                 return -rte_mtr_error_set(error, ENOTSUP,
 408                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 409                                           "Meter is not supported");
 410         /* Check input params. */
 411         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 412                                                profile, error);
 413         if (ret)
 414                 return ret;
 415         /* Meter profile memory allocation. */
 416         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 417                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 418         if (fmp == NULL)
 419                 return -rte_mtr_error_set(error, ENOMEM,
 420                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 421                                           NULL, "Meter profile memory "
 422                                           "alloc failed.");
 423         /* Fill profile info. */
 424         fmp->id = meter_profile_id;
 425         fmp->profile = *profile;
 426         /* Fill the flow meter parameters for the PRM. */
 427         ret = mlx5_flow_meter_param_fill(fmp, priv, error);
 428         if (ret)
 429                 goto error;
 430         /* Add to list. */
 431         TAILQ_INSERT_TAIL(fmps, fmp, next);
 432         return 0;
 433 error:
 434         mlx5_free(fmp);
 435         return ret;
 436 }
 437
 438 /**
 439  * Callback to delete MTR profile.
 440  *
 441  * @param[in] dev
 442  *   Pointer to Ethernet device.
 443  * @param[in] meter_profile_id
 444  *   Meter profile id.
 445  * @param[out] error
 446  *   Pointer to the error structure.
 447  *
 448  * @return
 449  *   0 on success, a negative errno value otherwise and rte_errno is set.
 450  */
 451 static int
 452 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 453                           uint32_t meter_profile_id,
 454                           struct rte_mtr_error *error)
 455 {
 456         struct mlx5_priv *priv = dev->data->dev_private;
 457         struct mlx5_flow_meter_profile *fmp;
 458
 459         if (!priv->mtr_en)
 460                 return -rte_mtr_error_set(error, ENOTSUP,
 461                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 462                                           "Meter is not supported");
 463         /* Meter profile must exist. */
 464         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 465         if (fmp == NULL)
 466                 return -rte_mtr_error_set(error, ENOENT,
 467                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 468                                           &meter_profile_id,
 469                                           "Meter profile id is invalid.");
 470         /* Check profile is unused. */
 471         if (fmp->ref_cnt)
 472                 return -rte_mtr_error_set(error, EBUSY,
 473                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 474                                           NULL, "Meter profile is in use.");
 475         /* Remove from list. */
 476         TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
 477         mlx5_free(fmp);
 478         return 0;
 479 }
 480
 481 /**
 482  * Find policy by id.
 483  *
 484  * @param[in] dev
 485  *   Pointer to Ethernet device.
 486  * @param policy_id
 487  *   Policy id.
 488  *
 489  * @return
 490  *   Pointer to the policy found on success, NULL otherwise.
 491  */
 492 struct mlx5_flow_meter_policy *
 493 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 494                             uint32_t policy_id,
 495                             uint32_t *policy_idx)
 496 {
 497         struct mlx5_priv *priv = dev->data->dev_private;
 498         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 499         union mlx5_l3t_data data;
 500
 501         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM ||
 502                 !priv->sh->mtrmng->policy_idx_tbl)
 503                 return NULL;
 504         if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl,
 505                                 policy_id, &data) ||
 506                                 !data.dword)
 507                 return NULL;
 508         if (policy_idx)
 509                 *policy_idx = data.dword;
 510         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 511                                         data.dword);
 512         /* Remove reference taken by the mlx5_l3t_get_entry. */
 513         mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 514                                 policy_id);
 515         if (sub_policy)
 516                 if (sub_policy->main_policy_id)
 517                         return sub_policy->main_policy;
 518         return NULL;
 519 }
 520
 521 /**
 522  * Callback to check MTR policy action validate
 523  *
 524  * @param[in] dev
 525  *   Pointer to Ethernet device.
 526  * @param[in] actions
 527  *   Pointer to meter policy action detail.
 528  * @param[out] error
 529  *   Pointer to the error structure.
 530  *
 531  * @return
 532  *   0 on success, a negative errno value otherwise and rte_errno is set.
 533  */
 534 static int
 535 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 536         struct rte_mtr_meter_policy_params *policy,
 537         struct rte_mtr_error *error)
 538 {
 539         struct mlx5_priv *priv = dev->data->dev_private;
 540         struct rte_flow_attr attr = { .transfer =
 541                         priv->config.dv_esw_en ? 1 : 0};
 542         bool is_rss = false;
 543         bool is_def_policy = false;
 544         uint8_t domain_bitmap;
 545         int ret;
 546
 547         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 548                 return -rte_mtr_error_set(error, ENOTSUP,
 549                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 550                                 NULL, "meter policy unsupported.");
 551         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 552                         &is_rss, &domain_bitmap, &is_def_policy, error);
 553         if (ret)
 554                 return ret;
 555         return 0;
 556 }
 557
 558 static int
 559 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 560                         uint32_t policy_id,
 561                         struct mlx5_flow_meter_policy *mtr_policy,
 562                         struct rte_mtr_error *error)
 563 {
 564         struct mlx5_priv *priv = dev->data->dev_private;
 565         struct mlx5_flow_meter_sub_policy *sub_policy;
 566         uint32_t i, j;
 567         uint16_t sub_policy_num;
 568
 569         rte_spinlock_lock(&mtr_policy->sl);
 570         if (mtr_policy->ref_cnt) {
 571                 rte_spinlock_unlock(&mtr_policy->sl);
 572                 return -rte_mtr_error_set(error, EBUSY,
 573                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 574                                  NULL,
 575                                 "Meter policy object is being used.");
 576         }
 577         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 578         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 579         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 580                 sub_policy_num = (mtr_policy->sub_policy_num >>
 581                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 582                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 583                 if (sub_policy_num) {
 584                         for (j = 0; j < sub_policy_num; j++) {
 585                                 sub_policy = mtr_policy->sub_policys[i][j];
 586                                 if (sub_policy)
 587                                         mlx5_ipool_free
 588                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 589                                         sub_policy->idx);
 590                         }
 591                 }
 592         }
 593         if (priv->sh->mtrmng->policy_idx_tbl) {
 594                 if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 595                                         policy_id)) {
 596                         rte_spinlock_unlock(&mtr_policy->sl);
 597                         return -rte_mtr_error_set(error, ENOTSUP,
 598                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 599                                 "Fail to delete policy in index table.");
 600                 }
 601         }
 602         rte_spinlock_unlock(&mtr_policy->sl);
 603         return 0;
 604 }
 605
 606 /**
 607  * Callback to add MTR policy.
 608  *
 609  * @param[in] dev
 610  *   Pointer to Ethernet device.
 611  * @param[out] policy_id
 612  *   Pointer to policy id
 613  * @param[in] actions
 614  *   Pointer to meter policy action detail.
 615  * @param[out] error
 616  *   Pointer to the error structure.
 617  *
 618  * @return
 619  *   0 on success, a negative errno value otherwise and rte_errno is set.
 620  */
 621 static int
 622 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 623                         uint32_t policy_id,
 624                         struct rte_mtr_meter_policy_params *policy,
 625                         struct rte_mtr_error *error)
 626 {
 627         struct mlx5_priv *priv = dev->data->dev_private;
 628         struct rte_flow_attr attr = { .transfer =
 629                         priv->config.dv_esw_en ? 1 : 0};
 630         uint32_t sub_policy_idx = 0;
 631         uint32_t policy_idx = 0;
 632         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 633         struct mlx5_flow_meter_sub_policy *sub_policy;
 634         bool is_rss = false;
 635         bool is_def_policy = false;
 636         uint32_t i;
 637         int ret;
 638         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 639         uint16_t sub_policy_num;
 640         uint8_t domain_bitmap = 0;
 641         union mlx5_l3t_data data;
 642
 643         if (!priv->mtr_en)
 644                 return -rte_mtr_error_set(error, ENOTSUP,
 645                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 646                                           NULL, "meter policy unsupported.");
 647         if (policy_id == MLX5_INVALID_POLICY_ID)
 648                 return -rte_mtr_error_set(error, ENOTSUP,
 649                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 650                         "policy ID is invalid. ");
 651         if (policy_id == priv->sh->mtrmng->def_policy_id)
 652                 return -rte_mtr_error_set(error, EEXIST,
 653                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 654                         "policy ID exists. ");
 655         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id,
 656                                 &policy_idx);
 657         if (mtr_policy)
 658                 return -rte_mtr_error_set(error, EEXIST,
 659                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 660                         "policy ID exists. ");
 661         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 662                         &is_rss, &domain_bitmap, &is_def_policy, error);
 663         if (ret)
 664                 return ret;
 665         if (!domain_bitmap)
 666                 return -rte_mtr_error_set(error, ENOTSUP,
 667                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 668                                 NULL, "fail to find policy domain.");
 669         if (is_def_policy) {
 670                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 671                         return -rte_mtr_error_set(error, EEXIST,
 672                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 673                                 NULL, "a policy with similar actions "
 674                                 "is already configured");
 675                 if (mlx5_flow_create_def_policy(dev))
 676                         return -rte_mtr_error_set(error, ENOTSUP,
 677                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 678                                 NULL,
 679                                 "fail to create non-terminated policy.");
 680                 priv->sh->mtrmng->def_policy_id = policy_id;
 681                 return 0;
 682         }
 683         if (!priv->sh->meter_aso_en)
 684                 return -rte_mtr_error_set(error, ENOTSUP,
 685                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 686                         "no ASO capability to support the policy ");
 687         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 688                 if (!(domain_bitmap & (1 << i)))
 689                         continue;
 690                 if (is_rss) {
 691                         policy_size +=
 692                         sizeof(struct mlx5_flow_meter_sub_policy *) *
 693                         MLX5_MTR_RSS_MAX_SUB_POLICY;
 694                         break;
 695                 }
 696                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 697         }
 698         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 699                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 700         if (!mtr_policy)
 701                 return -rte_mtr_error_set(error, ENOMEM,
 702                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 703                                 "Memory alloc failed for meter policy.");
 704         policy_size = sizeof(struct mlx5_flow_meter_policy);
 705         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 706                 if (!(domain_bitmap & (1 << i)))
 707                         continue;
 708                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 709                         mtr_policy->ingress = 1;
 710                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 711                         mtr_policy->egress = 1;
 712                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 713                         mtr_policy->transfer = 1;
 714                 sub_policy = mlx5_ipool_zmalloc
 715                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 716                                 &sub_policy_idx);
 717                 if (!sub_policy)
 718                         goto policy_add_err;
 719                 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 720                         goto policy_add_err;
 721                 sub_policy->idx = sub_policy_idx;
 722                 sub_policy->main_policy = mtr_policy;
 723                 if (!policy_idx) {
 724                         policy_idx = sub_policy_idx;
 725                         sub_policy->main_policy_id = 1;
 726                 }
 727                 mtr_policy->sub_policys[i] =
 728                 (struct mlx5_flow_meter_sub_policy **)
 729                         ((uint8_t *)mtr_policy + policy_size);
 730                 mtr_policy->sub_policys[i][0] = sub_policy;
 731                 sub_policy_num = (mtr_policy->sub_policy_num >>
 732                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 733                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 734                 sub_policy_num++;
 735                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 736                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 737                 mtr_policy->sub_policy_num |=
 738                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 739                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 740                 if (is_rss) {
 741                         mtr_policy->is_rss = 1;
 742                         break;
 743                 }
 744                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy  *);
 745         }
 746         rte_spinlock_init(&mtr_policy->sl);
 747         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 748                                         policy->actions, error);
 749         if (ret)
 750                 goto policy_add_err;
 751         if (!is_rss) {
 752                 /* Create policy rules in HW. */
 753                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 754                 if (ret)
 755                         goto policy_add_err;
 756         }
 757         data.dword = policy_idx;
 758         if (!priv->sh->mtrmng->policy_idx_tbl) {
 759                 priv->sh->mtrmng->policy_idx_tbl =
 760                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 761                 if (!priv->sh->mtrmng->policy_idx_tbl)
 762                         goto policy_add_err;
 763         }
 764         if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl,
 765                                 policy_id, &data))
 766                 goto policy_add_err;
 767         return 0;
 768 policy_add_err:
 769         if (mtr_policy) {
 770                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 771                         mtr_policy, error);
 772                 mlx5_free(mtr_policy);
 773                 if (ret)
 774                         return ret;
 775         }
 776         return -rte_mtr_error_set(error, ENOTSUP,
 777                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 778                                   NULL, "Failed to create devx policy.");
 779 }
 780
 781 /**
 782  * Callback to delete MTR policy.
 783  *
 784  * @param[in] dev
 785  *   Pointer to Ethernet device.
 786  * @param[in] policy_id
 787  *   Meter policy id.
 788  * @param[out] error
 789  *   Pointer to the error structure.
 790  *
 791  * @return
 792  *   0 on success, a negative errno value otherwise and rte_errno is set.
 793  */
 794 static int
 795 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 796                           uint32_t policy_id,
 797                           struct rte_mtr_error *error)
 798 {
 799         struct mlx5_priv *priv = dev->data->dev_private;
 800         struct mlx5_flow_meter_policy *mtr_policy;
 801         uint32_t policy_idx;
 802         int ret;
 803
 804         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 805                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 806                         return -rte_mtr_error_set(error, ENOTSUP,
 807                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 808                                 "Meter policy object is being used.");
 809                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 810                 return 0;
 811         }
 812         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 813         if (!mtr_policy)
 814                 return -rte_mtr_error_set(error, ENOTSUP,
 815                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 816                         "Meter policy id is invalid. ");
 817         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 818                                                 error);
 819         if (ret)
 820                 return ret;
 821         mlx5_free(mtr_policy);
 822         return 0;
 823 }
 824
 825 /**
 826  * Check meter validation.
 827  *
 828  * @param[in] priv
 829  *   Pointer to mlx5 private data structure.
 830  * @param[in] meter_id
 831  *   Meter id.
 832  * @param[in] params
 833  *   Pointer to rte meter parameters.
 834  * @param[out] error
 835  *   Pointer to rte meter error structure.
 836  *
 837  * @return
 838  *   0 on success, a negative errno value otherwise and rte_errno is set.
 839  */
 840 static int
 841 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 842                          struct rte_mtr_params *params,
 843                          struct rte_mtr_error *error)
 844 {
 845         /* Meter must use global drop action. */
 846         if (!priv->sh->dr_drop_action)
 847                 return -rte_mtr_error_set(error, ENOTSUP,
 848                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 849                                           NULL,
 850                                           "No drop action ready for meter.");
 851         /* Meter params must not be NULL. */
 852         if (params == NULL)
 853                 return -rte_mtr_error_set(error, EINVAL,
 854                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 855                                           NULL, "Meter object params null.");
 856         /* Previous meter color is not supported. */
 857         if (params->use_prev_mtr_color)
 858                 return -rte_mtr_error_set(error, ENOTSUP,
 859                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 860                                           NULL,
 861                                           "Previous meter color "
 862                                           "not supported.");
 863         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 864                 return -rte_mtr_error_set(error, ENOENT,
 865                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 866                                 NULL, "Meter policy id not valid.");
 867         /* Validate meter id. */
 868         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 869                 return -rte_mtr_error_set(error, EEXIST,
 870                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 871                         "Meter object already exists.");
 872         return 0;
 873 }
 874
 875 /**
 876  * Modify the flow meter action.
 877  *
 878  * @param[in] priv
 879  *   Pointer to mlx5 private data structure.
 880  * @param[in] fm
 881  *   Pointer to flow meter to be modified.
 882  * @param[in] srtcm
 883  *   Pointer to meter srtcm description parameter.
 884  * @param[in] modify_bits
 885  *   The bit in srtcm to be updated.
 886  * @param[in] active_state
 887  *   The state to be updated.
 888  * @return
 889  *   0 on success, o negative value otherwise.
 890  */
 891 static int
 892 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 893                 struct mlx5_flow_meter_info *fm,
 894                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 895                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 896 {
 897 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 898         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 899         uint32_t *attr;
 900         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 901         int ret;
 902         struct mlx5_aso_mtr *aso_mtr = NULL;
 903         uint32_t cbs_cir, ebs_eir, val;
 904
 905         if (priv->sh->meter_aso_en) {
 906                 fm->is_enable = !!is_enable;
 907                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 908                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 909                 if (ret)
 910                         return ret;
 911                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 912                 if (ret)
 913                         return ret;
 914         } else {
 915                 /* Fill command parameters. */
 916                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 917                 mod_attr.flow_meter_parameter = in;
 918                 mod_attr.flow_meter_parameter_sz =
 919                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 920                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 921                         mod_attr.active = !!active_state;
 922                 else
 923                         mod_attr.active = 0;
 924                 attr = in;
 925                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 926                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 927                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 928                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 929                                 ASO_DSEG_EXP_MASK;
 930                         MLX5_SET(flow_meter_parameters, attr,
 931                                 cbs_exponent, val);
 932                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 933                                 ASO_DSEG_MAN_MASK;
 934                         MLX5_SET(flow_meter_parameters, attr,
 935                                 cbs_mantissa, val);
 936                 }
 937                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
 938                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
 939                                 ASO_DSEG_EXP_MASK;
 940                         MLX5_SET(flow_meter_parameters, attr,
 941                                 cir_exponent, val);
 942                         val = cbs_cir & ASO_DSEG_MAN_MASK;
 943                         MLX5_SET(flow_meter_parameters, attr,
 944                                 cir_mantissa, val);
 945                 }
 946                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
 947                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
 948                                 ASO_DSEG_EXP_MASK;
 949                         MLX5_SET(flow_meter_parameters, attr,
 950                                 ebs_exponent, val);
 951                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
 952                                 ASO_DSEG_MAN_MASK;
 953                         MLX5_SET(flow_meter_parameters, attr,
 954                                 ebs_mantissa, val);
 955                 }
 956                 /* Apply modifications to meter only if it was created. */
 957                 if (fm->meter_action) {
 958                         ret = mlx5_glue->dv_modify_flow_action_meter
 959                                         (fm->meter_action, &mod_attr,
 960                                         rte_cpu_to_be_64(modify_bits));
 961                         if (ret)
 962                                 return ret;
 963                 }
 964                 /* Update succeedded modify meter parameters. */
 965                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 966                         fm->active_state = !!active_state;
 967         }
 968         return 0;
 969 #else
 970         (void)priv;
 971         (void)fm;
 972         (void)srtcm;
 973         (void)modify_bits;
 974         (void)active_state;
 975         (void)is_enable;
 976         return -ENOTSUP;
 977 #endif
 978 }
 979
 980 static int
 981 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
 982                                 struct mlx5_flow_meter_info *fm,
 983                                 uint64_t stats_mask)
 984 {
 985         fm->bytes_dropped =
 986                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
 987         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
 988         if (fm->bytes_dropped || fm->pkts_dropped) {
 989                 if (!fm->drop_cnt) {
 990                         /* Alloc policer counters. */
 991                         fm->drop_cnt = mlx5_counter_alloc(dev);
 992                         if (!fm->drop_cnt)
 993                                 return -1;
 994                 }
 995         } else {
 996                 if (fm->drop_cnt) {
 997                         mlx5_counter_free(dev, fm->drop_cnt);
 998                         fm->drop_cnt = 0;
 999                 }
1000         }
1001         return 0;
1002 }
1003
1004 /**
1005  * Create meter rules.
1006  *
1007  * @param[in] dev
1008  *   Pointer to Ethernet device.
1009  * @param[in] meter_id
1010  *   Meter id.
1011  * @param[in] params
1012  *   Pointer to rte meter parameters.
1013  * @param[in] shared
1014  *   Meter shared with other flow or not.
1015  * @param[out] error
1016  *   Pointer to rte meter error structure.
1017  *
1018  * @return
1019  *   0 on success, a negative errno value otherwise and rte_errno is set.
1020  */
1021 static int
1022 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1023                        struct rte_mtr_params *params, int shared,
1024                        struct rte_mtr_error *error)
1025 {
1026         struct mlx5_priv *priv = dev->data->dev_private;
1027         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1028         struct mlx5_flow_meter_profile *fmp;
1029         struct mlx5_flow_meter_info *fm;
1030         struct mlx5_legacy_flow_meter *legacy_fm;
1031         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1032         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1033                 .size = 0,
1034                 .trunk_size = 64,
1035                 .need_lock = 1,
1036                 .type = "mlx5_flow_mtr_flow_id_pool",
1037         };
1038         struct mlx5_aso_mtr *aso_mtr;
1039         uint32_t mtr_idx, policy_idx;
1040         union mlx5_l3t_data data;
1041         int ret;
1042         uint8_t domain_bitmap;
1043         uint8_t mtr_id_bits;
1044         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1045                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1046
1047         if (!priv->mtr_en)
1048                 return -rte_mtr_error_set(error, ENOTSUP,
1049                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1050                                         "Meter is not supported");
1051         /* Validate the parameters. */
1052         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1053         if (ret)
1054                 return ret;
1055         /* Meter profile must exist. */
1056         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1057         if (fmp == NULL)
1058                 return -rte_mtr_error_set(error, ENOENT,
1059                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1060                         NULL, "Meter profile id not valid.");
1061         /* Meter policy must exist. */
1062         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1063                 __atomic_add_fetch
1064                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1065                         1, __ATOMIC_RELAXED);
1066                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1067                 if (!priv->config.dv_esw_en)
1068                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1069         } else {
1070                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1071                                 params->meter_policy_id, &policy_idx);
1072                 if (!priv->sh->meter_aso_en)
1073                         return -rte_mtr_error_set(error, ENOTSUP,
1074                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1075                                 "Part of the policies cannot be "
1076                                 "supported without ASO ");
1077                 if (!mtr_policy)
1078                         return -rte_mtr_error_set(error, ENOENT,
1079                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1080                                 NULL, "Meter policy id not valid.");
1081                 domain_bitmap = (mtr_policy->ingress ?
1082                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1083                                 (mtr_policy->egress ?
1084                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1085                                 (mtr_policy->transfer ?
1086                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1087         }
1088         /* Allocate the flow meter memory. */
1089         if (priv->sh->meter_aso_en) {
1090                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1091                 if (!mtr_idx)
1092                         return -rte_mtr_error_set(error, ENOMEM,
1093                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1094                                 "Memory alloc failed for meter.");
1095                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1096                 fm = &aso_mtr->fm;
1097         } else {
1098                 legacy_fm = mlx5_ipool_zmalloc
1099                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1100                 if (legacy_fm == NULL)
1101                         return -rte_mtr_error_set(error, ENOMEM,
1102                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1103                                 "Memory alloc failed for meter.");
1104                 legacy_fm->idx = mtr_idx;
1105                 fm = &legacy_fm->fm;
1106         }
1107         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1108         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1109             mtr_reg_bits) {
1110                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1111                 goto error;
1112         }
1113         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1114                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1115         /* Fill the flow meter parameters. */
1116         fm->meter_id = meter_id;
1117         fm->policy_id = params->meter_policy_id;
1118         fm->profile = fmp;
1119         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1120                 goto error;
1121         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1122                 goto error;
1123         /* Add to the flow meter list. */
1124         if (!priv->sh->meter_aso_en)
1125                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1126         /* Add to the flow meter list. */
1127         fm->active_state = 1; /* Config meter starts as active. */
1128         fm->is_enable = 1;
1129         fm->shared = !!shared;
1130         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1131         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1132                 fm->def_policy = 1;
1133                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1134                 if (!fm->flow_ipool)
1135                         goto error;
1136         }
1137         rte_spinlock_init(&fm->sl);
1138         /* If ASO meter supported, update ASO flow meter by wqe. */
1139         if (priv->sh->meter_aso_en) {
1140                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1141                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1142                 if (ret)
1143                         goto error;
1144                 if (!priv->mtr_idx_tbl) {
1145                         priv->mtr_idx_tbl =
1146                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1147                         if (!priv->mtr_idx_tbl)
1148                                 goto error;
1149                 }
1150                 data.dword = mtr_idx;
1151                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1152                         goto error;
1153         }
1154         if (mtr_policy)
1155                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1156         return 0;
1157 error:
1158         mlx5_flow_destroy_mtr_tbls(dev, fm);
1159         /* Free policer counters. */
1160         if (fm->drop_cnt)
1161                 mlx5_counter_free(dev, fm->drop_cnt);
1162         if (priv->sh->meter_aso_en)
1163                 mlx5_flow_mtr_free(dev, mtr_idx);
1164         else
1165                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1166         return -rte_mtr_error_set(error, ENOTSUP,
1167                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1168                 NULL, "Failed to create devx meter.");
1169 }
1170
1171 static int
1172 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1173                         struct mlx5_flow_meter_info *fm,
1174                         uint32_t mtr_idx)
1175 {
1176         struct mlx5_priv *priv = dev->data->dev_private;
1177         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1178         struct mlx5_flow_meter_profile *fmp;
1179         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1180         struct mlx5_flow_meter_policy *mtr_policy;
1181
1182         /* Meter object must not have any owner. */
1183         MLX5_ASSERT(!fm->ref_cnt);
1184         /* Get meter profile. */
1185         fmp = fm->profile;
1186         if (fmp == NULL)
1187                 return -1;
1188         /* Update dependencies. */
1189         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1190         fm->profile = NULL;
1191         /* Remove from list. */
1192         if (!priv->sh->meter_aso_en) {
1193                 legacy_fm = container_of(fm,
1194                         struct mlx5_legacy_flow_meter, fm);
1195                 TAILQ_REMOVE(fms, legacy_fm, next);
1196         }
1197         /* Free drop counters. */
1198         if (fm->drop_cnt)
1199                 mlx5_counter_free(dev, fm->drop_cnt);
1200         /* Free meter flow table. */
1201         if (fm->flow_ipool) {
1202                 mlx5_ipool_destroy(fm->flow_ipool);
1203                 fm->flow_ipool = 0;
1204         }
1205         mlx5_flow_destroy_mtr_tbls(dev, fm);
1206         if (fm->def_policy)
1207                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1208                                 1, __ATOMIC_RELAXED);
1209         if (priv->sh->meter_aso_en) {
1210                 if (!fm->def_policy) {
1211                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1212                                                 fm->policy_id, NULL);
1213                         if (mtr_policy)
1214                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1215                                                 1, __ATOMIC_RELAXED);
1216                         fm->policy_id = 0;
1217                 }
1218                 fm->def_policy = 0;
1219                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1220                         return -1;
1221                 mlx5_flow_mtr_free(dev, mtr_idx);
1222         } else {
1223                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1224                                         legacy_fm->idx);
1225         }
1226         return 0;
1227 }
1228
1229 /**
1230  * Destroy meter rules.
1231  *
1232  * @param[in] dev
1233  *   Pointer to Ethernet device.
1234  * @param[in] meter_id
1235  *   Meter id.
1236  * @param[out] error
1237  *   Pointer to rte meter error structure.
1238  *
1239  * @return
1240  *   0 on success, a negative errno value otherwise and rte_errno is set.
1241  */
1242 static int
1243 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1244                         struct rte_mtr_error *error)
1245 {
1246         struct mlx5_priv *priv = dev->data->dev_private;
1247         struct mlx5_flow_meter_info *fm;
1248         uint32_t mtr_idx = 0;
1249
1250         if (!priv->mtr_en)
1251                 return -rte_mtr_error_set(error, ENOTSUP,
1252                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1253                                           NULL,
1254                                           "Meter is not supported");
1255         /* Meter object must exist. */
1256         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1257         if (fm == NULL)
1258                 return -rte_mtr_error_set(error, ENOENT,
1259                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1260                                           NULL,
1261                                           "Meter object id not valid.");
1262         /* Meter object must not have any owner. */
1263         if (fm->ref_cnt > 0)
1264                 return -rte_mtr_error_set(error, EBUSY,
1265                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1266                                           NULL,
1267                                           "Meter object is being used.");
1268         /* Destroy the meter profile. */
1269         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1270                 return -rte_mtr_error_set(error, EINVAL,
1271                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1272                                         NULL,
1273                                         "MTR object meter profile invalid.");
1274         return 0;
1275 }
1276
1277 /**
1278  * Modify meter state.
1279  *
1280  * @param[in] priv
1281  *   Pointer to mlx5 private data structure.
1282  * @param[in] fm
1283  *   Pointer to flow meter.
1284  * @param[in] new_state
1285  *   New state to update.
1286  * @param[out] error
1287  *   Pointer to rte meter error structure.
1288  *
1289  * @return
1290  *   0 on success, a negative errno value otherwise and rte_errno is set.
1291  */
1292 static int
1293 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1294                              struct mlx5_flow_meter_info *fm,
1295                              uint32_t new_state,
1296                              struct rte_mtr_error *error)
1297 {
1298         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1299                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1300                 .ebs_eir = 0,
1301         };
1302         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1303                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1304         int ret;
1305
1306         if (new_state == MLX5_FLOW_METER_DISABLE)
1307                 ret = mlx5_flow_meter_action_modify(priv, fm,
1308                                 &srtcm, modify_bits, 0, 0);
1309         else
1310                 ret = mlx5_flow_meter_action_modify(priv, fm,
1311                                                    &fm->profile->srtcm_prm,
1312                                                     modify_bits, 0, 1);
1313         if (ret)
1314                 return -rte_mtr_error_set(error, -ret,
1315                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1316                                           NULL,
1317                                           new_state ?
1318                                           "Failed to enable meter." :
1319                                           "Failed to disable meter.");
1320         return 0;
1321 }
1322
1323 /**
1324  * Callback to enable flow meter.
1325  *
1326  * @param[in] dev
1327  *   Pointer to Ethernet device.
1328  * @param[in] meter_id
1329  *   Meter id.
1330  * @param[out] error
1331  *   Pointer to rte meter error structure.
1332  *
1333  * @return
1334  *   0 on success, a negative errno value otherwise and rte_errno is set.
1335  */
1336 static int
1337 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1338                        uint32_t meter_id,
1339                        struct rte_mtr_error *error)
1340 {
1341         struct mlx5_priv *priv = dev->data->dev_private;
1342         struct mlx5_flow_meter_info *fm;
1343         int ret;
1344
1345         if (!priv->mtr_en)
1346                 return -rte_mtr_error_set(error, ENOTSUP,
1347                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1348                                           "Meter is not supported");
1349         /* Meter object must exist. */
1350         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1351         if (fm == NULL)
1352                 return -rte_mtr_error_set(error, ENOENT,
1353                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1354                                           NULL, "Meter not found.");
1355         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1356                 return 0;
1357         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1358                                            error);
1359         if (!ret)
1360                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1361         return ret;
1362 }
1363
1364 /**
1365  * Callback to disable flow meter.
1366  *
1367  * @param[in] dev
1368  *   Pointer to Ethernet device.
1369  * @param[in] meter_id
1370  *   Meter id.
1371  * @param[out] error
1372  *   Pointer to rte meter error structure.
1373  *
1374  * @return
1375  *   0 on success, a negative errno value otherwise and rte_errno is set.
1376  */
1377 static int
1378 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1379                         uint32_t meter_id,
1380                         struct rte_mtr_error *error)
1381 {
1382         struct mlx5_priv *priv = dev->data->dev_private;
1383         struct mlx5_flow_meter_info *fm;
1384         int ret;
1385
1386         if (!priv->mtr_en)
1387                 return -rte_mtr_error_set(error, ENOTSUP,
1388                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1389                                           "Meter is not supported");
1390         /* Meter object must exist. */
1391         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1392         if (fm == NULL)
1393                 return -rte_mtr_error_set(error, ENOENT,
1394                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1395                                           NULL, "Meter not found.");
1396         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1397                 return 0;
1398         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1399                                            error);
1400         if (!ret)
1401                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1402         return ret;
1403 }
1404
1405 /**
1406  * Callback to update meter profile.
1407  *
1408  * @param[in] dev
1409  *   Pointer to Ethernet device.
1410  * @param[in] meter_id
1411  *   Meter id.
1412  * @param[in] meter_profile_id
1413  *   To be updated meter profile id.
1414  * @param[out] error
1415  *   Pointer to rte meter error structure.
1416  *
1417  * @return
1418  *   0 on success, a negative errno value otherwise and rte_errno is set.
1419  */
1420 static int
1421 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1422                                uint32_t meter_id,
1423                                uint32_t meter_profile_id,
1424                                struct rte_mtr_error *error)
1425 {
1426         struct mlx5_priv *priv = dev->data->dev_private;
1427         struct mlx5_flow_meter_profile *fmp;
1428         struct mlx5_flow_meter_profile *old_fmp;
1429         struct mlx5_flow_meter_info *fm;
1430         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1431                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1432         int ret;
1433
1434         if (!priv->mtr_en)
1435                 return -rte_mtr_error_set(error, ENOTSUP,
1436                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1437                                           "Meter is not supported");
1438         /* Meter profile must exist. */
1439         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1440         if (fmp == NULL)
1441                 return -rte_mtr_error_set(error, ENOENT,
1442                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1443                                           NULL, "Meter profile not found.");
1444         /* Meter object must exist. */
1445         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1446         if (fm == NULL)
1447                 return -rte_mtr_error_set(error, ENOENT,
1448                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1449                                           NULL, "Meter not found.");
1450         /* MTR object already set to meter profile id. */
1451         old_fmp = fm->profile;
1452         if (fmp == old_fmp)
1453                 return 0;
1454         /* Update the profile. */
1455         fm->profile = fmp;
1456         /* Update meter params in HW (if not disabled). */
1457         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1458                 return 0;
1459         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1460                                               modify_bits, fm->active_state, 1);
1461         if (ret) {
1462                 fm->profile = old_fmp;
1463                 return -rte_mtr_error_set(error, -ret,
1464                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1465                                           NULL, "Failed to update meter"
1466                                           " parmeters in hardware.");
1467         }
1468         old_fmp->ref_cnt--;
1469         fmp->ref_cnt++;
1470         return 0;
1471 }
1472
1473 /**
1474  * Callback to update meter stats mask.
1475  *
1476  * @param[in] dev
1477  *   Pointer to Ethernet device.
1478  * @param[in] meter_id
1479  *   Meter id.
1480  * @param[in] stats_mask
1481  *   To be updated stats_mask.
1482  * @param[out] error
1483  *   Pointer to rte meter error structure.
1484  *
1485  * @return
1486  *   0 on success, a negative errno value otherwise and rte_errno is set.
1487  */
1488 static int
1489 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1490                              uint32_t meter_id,
1491                              uint64_t stats_mask,
1492                              struct rte_mtr_error *error)
1493 {
1494         struct mlx5_priv *priv = dev->data->dev_private;
1495         struct mlx5_flow_meter_info *fm;
1496
1497         if (!priv->mtr_en)
1498                 return -rte_mtr_error_set(error, ENOTSUP,
1499                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1500                                           "Meter is not supported");
1501         /* Meter object must exist. */
1502         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1503         if (fm == NULL)
1504                 return -rte_mtr_error_set(error, ENOENT,
1505                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1506                                           NULL, "Meter object id not valid.");
1507         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1508                 return -rte_mtr_error_set(error, ENOENT,
1509                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1510                                           NULL, "Fail to allocate "
1511                                           "counter for meter.");
1512         return 0;
1513 }
1514
1515 /**
1516  * Callback to read meter statistics.
1517  *
1518  * @param[in] dev
1519  *   Pointer to Ethernet device.
1520  * @param[in] meter_id
1521  *   Meter id.
1522  * @param[out] stats
1523  *   Pointer to store the statistics.
1524  * @param[out] stats_mask
1525  *   Pointer to store the stats_mask.
1526  * @param[in] clear
1527  *   Statistic to be cleared after read or not.
1528  * @param[out] error
1529  *   Pointer to rte meter error structure.
1530  *
1531  * @return
1532  *   0 on success, a negative errno value otherwise and rte_errno is set.
1533  */
1534 static int
1535 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1536                            uint32_t meter_id,
1537                            struct rte_mtr_stats *stats,
1538                            uint64_t *stats_mask,
1539                            int clear,
1540                            struct rte_mtr_error *error)
1541 {
1542         struct mlx5_priv *priv = dev->data->dev_private;
1543         struct mlx5_flow_meter_info *fm;
1544         uint64_t pkts;
1545         uint64_t bytes;
1546         int ret = 0;
1547
1548         if (!priv->mtr_en)
1549                 return -rte_mtr_error_set(error, ENOTSUP,
1550                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1551                                           "Meter is not supported");
1552         /* Meter object must exist. */
1553         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1554         if (fm == NULL)
1555                 return -rte_mtr_error_set(error, ENOENT,
1556                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1557                                           NULL, "Meter object id not valid.");
1558         *stats_mask = 0;
1559         if (fm->bytes_dropped)
1560                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1561         if (fm->pkts_dropped)
1562                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1563         memset(stats, 0, sizeof(*stats));
1564         if (fm->drop_cnt) {
1565                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1566                                                  &bytes);
1567                 if (ret)
1568                         goto error;
1569                 /* If need to read the packets, set it. */
1570                 if (fm->pkts_dropped)
1571                         stats->n_pkts_dropped = pkts;
1572                 /* If need to read the bytes, set it. */
1573                 if (fm->bytes_dropped)
1574                         stats->n_bytes_dropped = bytes;
1575         }
1576         return 0;
1577 error:
1578         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1579                                  "Failed to read meter drop counters.");
1580 }
1581
1582 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1583         .capabilities_get = mlx5_flow_mtr_cap_get,
1584         .meter_profile_add = mlx5_flow_meter_profile_add,
1585         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1586         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1587         .meter_policy_add = mlx5_flow_meter_policy_add,
1588         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1589         .create = mlx5_flow_meter_create,
1590         .destroy = mlx5_flow_meter_destroy,
1591         .meter_enable = mlx5_flow_meter_enable,
1592         .meter_disable = mlx5_flow_meter_disable,
1593         .meter_profile_update = mlx5_flow_meter_profile_update,
1594         .meter_dscp_table_update = NULL,
1595         .stats_update = mlx5_flow_meter_stats_update,
1596         .stats_read = mlx5_flow_meter_stats_read,
1597 };
1598
1599 /**
1600  * Get meter operations.
1601  *
1602  * @param dev
1603  *   Pointer to Ethernet device structure.
1604  * @param arg
1605  *   Pointer to set the mtr operations.
1606  *
1607  * @return
1608  *   Always 0.
1609  */
1610 int
1611 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1612 {
1613         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1614         return 0;
1615 }
1616
1617 /**
1618  * Find meter by id.
1619  *
1620  * @param priv
1621  *   Pointer to mlx5_priv.
1622  * @param meter_id
1623  *   Meter id.
1624  * @param mtr_idx
1625  *   Pointer to Meter index.
1626  *
1627  * @return
1628  *   Pointer to the meter info found on success, NULL otherwise.
1629  */
1630 struct mlx5_flow_meter_info *
1631 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1632                 uint32_t *mtr_idx)
1633 {
1634         struct mlx5_legacy_flow_meter *legacy_fm;
1635         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1636         struct mlx5_aso_mtr *aso_mtr;
1637         struct mlx5_aso_mtr_pools_mng *pools_mng =
1638                                 &priv->sh->mtrmng->pools_mng;
1639         union mlx5_l3t_data data;
1640
1641         if (priv->sh->meter_aso_en) {
1642                 rte_spinlock_lock(&pools_mng->mtrsl);
1643                 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1644                         rte_spinlock_unlock(&pools_mng->mtrsl);
1645                         return NULL;
1646                 }
1647                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1648                         !data.dword) {
1649                         rte_spinlock_unlock(&pools_mng->mtrsl);
1650                         return NULL;
1651                 }
1652                 if (mtr_idx)
1653                         *mtr_idx = data.dword;
1654                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1655                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1656                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1657                 rte_spinlock_unlock(&pools_mng->mtrsl);
1658                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1659                         return NULL;
1660                 return &aso_mtr->fm;
1661         }
1662         TAILQ_FOREACH(legacy_fm, fms, next)
1663                 if (meter_id == legacy_fm->fm.meter_id) {
1664                         if (mtr_idx)
1665                                 *mtr_idx = legacy_fm->idx;
1666                         return &legacy_fm->fm;
1667                 }
1668         return NULL;
1669 }
1670
1671 /**
1672  * Find meter by index.
1673  *
1674  * @param priv
1675  *   Pointer to mlx5_priv.
1676  * @param idx
1677  *   Meter index.
1678  *
1679  * @return
1680  *   Pointer to the meter info found on success, NULL otherwise.
1681  */
1682 struct mlx5_flow_meter_info *
1683 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1684 {
1685         struct mlx5_aso_mtr *aso_mtr;
1686
1687         if (priv->sh->meter_aso_en) {
1688                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1689                 if (!aso_mtr)
1690                         return NULL;
1691                 return &aso_mtr->fm;
1692         } else {
1693                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1694         }
1695 }
1696
1697 /**
1698  * Attach meter to flow.
1699  * Unidirectional Meter creation can only be done
1700  * when flow direction is known, i.e. when calling meter_attach.
1701  *
1702  * @param [in] priv
1703  *  Pointer to mlx5 private data.
1704  * @param[in] fm
1705  *   Pointer to flow meter.
1706  * @param [in] attr
1707  *  Pointer to flow attributes.
1708  * @param [out] error
1709  *  Pointer to error structure.
1710  *
1711  * @return
1712  *   0 on success, a negative errno value otherwise and rte_errno is set.
1713  */
1714 int
1715 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1716                        struct mlx5_flow_meter_info *fm,
1717                        const struct rte_flow_attr *attr,
1718                        struct rte_flow_error *error)
1719 {
1720         int ret = 0;
1721
1722         if (priv->sh->meter_aso_en) {
1723                 struct mlx5_aso_mtr *aso_mtr;
1724
1725                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1726                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1727                         return rte_flow_error_set(error, ENOENT,
1728                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1729                                         NULL,
1730                                         "Timeout in meter configuration");
1731                 }
1732                 rte_spinlock_lock(&fm->sl);
1733                 if (fm->shared || !fm->ref_cnt) {
1734                         fm->ref_cnt++;
1735                 } else {
1736                         rte_flow_error_set(error, EINVAL,
1737                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1738                                    "Meter cannot be shared");
1739                         ret = -1;
1740                 }
1741                 rte_spinlock_unlock(&fm->sl);
1742         } else {
1743                 rte_spinlock_lock(&fm->sl);
1744                 if (fm->meter_action) {
1745                         if (fm->shared &&
1746                             attr->transfer == fm->transfer &&
1747                             attr->ingress == fm->ingress &&
1748                             attr->egress == fm->egress) {
1749                                 fm->ref_cnt++;
1750                         } else {
1751                                 rte_flow_error_set(error, EINVAL,
1752                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1753                                         fm->shared ?
1754                                         "Meter attr not match." :
1755                                         "Meter cannot be shared.");
1756                                 ret = -1;
1757                         }
1758                 } else {
1759                         fm->ingress = attr->ingress;
1760                         fm->egress = attr->egress;
1761                         fm->transfer = attr->transfer;
1762                         fm->ref_cnt = 1;
1763                         /* This also creates the meter object. */
1764                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1765                                                                          fm);
1766                         if (!fm->meter_action) {
1767                                 fm->ref_cnt = 0;
1768                                 fm->ingress = 0;
1769                                 fm->egress = 0;
1770                                 fm->transfer = 0;
1771                                 rte_flow_error_set(error, EINVAL,
1772                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1773                                         "Meter action create failed.");
1774                                 ret = -1;
1775                         }
1776                 }
1777                 rte_spinlock_unlock(&fm->sl);
1778         }
1779         return ret ? -rte_errno : 0;
1780 }
1781
1782 /**
1783  * Detach meter from flow.
1784  *
1785  * @param [in] priv
1786  *  Pointer to mlx5 private data.
1787  * @param [in] fm
1788  *  Pointer to flow meter.
1789  */
1790 void
1791 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1792                        struct mlx5_flow_meter_info *fm)
1793 {
1794 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1795         rte_spinlock_lock(&fm->sl);
1796         MLX5_ASSERT(fm->ref_cnt);
1797         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1798                 mlx5_glue->destroy_flow_action(fm->meter_action);
1799                 fm->meter_action = NULL;
1800                 fm->ingress = 0;
1801                 fm->egress = 0;
1802                 fm->transfer = 0;
1803         }
1804         rte_spinlock_unlock(&fm->sl);
1805 #else
1806         (void)priv;
1807         (void)fm;
1808 #endif
1809 }
1810
1811 /**
1812  * Flush meter configuration.
1813  *
1814  * @param[in] dev
1815  *   Pointer to Ethernet device.
1816  * @param[out] error
1817  *   Pointer to rte meter error structure.
1818  *
1819  * @return
1820  *   0 on success, a negative errno value otherwise and rte_errno is set.
1821  */
1822 int
1823 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1824 {
1825         struct mlx5_priv *priv = dev->data->dev_private;
1826         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1827         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
1828         struct mlx5_flow_meter_profile *fmp;
1829         struct mlx5_legacy_flow_meter *legacy_fm;
1830         struct mlx5_flow_meter_info *fm;
1831         struct mlx5_flow_meter_sub_policy *sub_policy;
1832         void *tmp;
1833         uint32_t i, mtr_idx, policy_idx;
1834         void *entry;
1835         struct mlx5_aso_mtr *aso_mtr;
1836
1837         if (!priv->mtr_en)
1838                 return 0;
1839         if (priv->sh->meter_aso_en) {
1840                 if (priv->mtr_idx_tbl) {
1841                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
1842                                 mtr_idx = *(uint32_t *)entry;
1843                                 if (mtr_idx) {
1844                                         aso_mtr =
1845                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
1846                                         fm = &aso_mtr->fm;
1847                                         (void)mlx5_flow_meter_params_flush(dev,
1848                                                 fm, mtr_idx);
1849                                 }
1850                         }
1851                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
1852                         priv->mtr_idx_tbl = NULL;
1853                 }
1854         } else {
1855                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1856                         fm = &legacy_fm->fm;
1857                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
1858                                 return -rte_mtr_error_set(error, EINVAL,
1859                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1860                                 NULL, "MTR object meter profile invalid.");
1861                 }
1862         }
1863         if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1864                 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1865                                         i, entry) {
1866                         policy_idx = *(uint32_t *)entry;
1867                         sub_policy = mlx5_ipool_get
1868                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1869                                 policy_idx);
1870                         if (!sub_policy)
1871                                 return -rte_mtr_error_set(error,
1872                                                 EINVAL,
1873                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1874                                                 NULL, "MTR object "
1875                                                 "meter policy invalid.");
1876                         if (__mlx5_flow_meter_policy_delete(dev, i,
1877                                                 sub_policy->main_policy,
1878                                                 error))
1879                                 return -rte_mtr_error_set(error,
1880                                                 EINVAL,
1881                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1882                                                 NULL, "MTR object "
1883                                                 "meter policy invalid.");
1884                         mlx5_free(sub_policy->main_policy);
1885                 }
1886                 mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl);
1887                 priv->sh->mtrmng->policy_idx_tbl = NULL;
1888         }
1889         TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) {
1890                 /* Check unused. */
1891                 MLX5_ASSERT(!fmp->ref_cnt);
1892                 /* Remove from list. */
1893                 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
1894                 mlx5_free(fmp);
1895         }
1896         /* Delete default policy table. */
1897         mlx5_flow_destroy_def_policy(dev);
1898         if (priv->sh->refcnt == 1)
1899                 mlx5_flow_destroy_mtr_drop_tbls(dev);
1900         return 0;
1901 }