drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_flow_meter_profile *fmp;
  95         union mlx5_l3t_data data;
  96         int32_t ret;
  97
  98         if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
  99                                meter_profile_id, &data) || !data.ptr)
 100                 return NULL;
 101         fmp = data.ptr;
 102         /* Remove reference taken by the mlx5_l3t_get_entry. */
 103         ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
 104                                    meter_profile_id);
 105         if (!ret || ret == -1)
 106                 return NULL;
 107         return fmp;
 108 }
 109
 110 /**
 111  * Validate the MTR profile.
 112  *
 113  * @param[in] dev
 114  *   Pointer to Ethernet device.
 115  * @param[in] meter_profile_id
 116  *   Meter profile id.
 117  * @param[in] profile
 118  *   Pointer to meter profile detail.
 119  * @param[out] error
 120  *   Pointer to the error structure.
 121  *
 122  * @return
 123  *   0 on success, a negative errno value otherwise and rte_errno is set.
 124  */
 125 static int
 126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 127                                  uint32_t meter_profile_id,
 128                                  struct rte_mtr_meter_profile *profile,
 129                                  struct rte_mtr_error *error)
 130 {
 131         struct mlx5_priv *priv = dev->data->dev_private;
 132         struct mlx5_flow_meter_profile *fmp;
 133
 134         /* Profile must not be NULL. */
 135         if (profile == NULL)
 136                 return -rte_mtr_error_set(error, EINVAL,
 137                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 138                                           NULL, "Meter profile is null.");
 139         /* Meter profile ID must be valid. */
 140         if (meter_profile_id == UINT32_MAX)
 141                 return -rte_mtr_error_set(error, EINVAL,
 142                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 143                                           NULL, "Meter profile id not valid.");
 144         /* Meter profile must not exist. */
 145         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 146         if (fmp)
 147                 return -rte_mtr_error_set(error, EEXIST,
 148                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 149                                           NULL,
 150                                           "Meter profile already exists.");
 151         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 152                 if (priv->config.hca_attr.qos.flow_meter_old) {
 153                         /* Verify support for flow meter parameters. */
 154                         if (priv->sh->meter_aso_en && profile->packet_mode) {
 155                                 if (profile->srtcm_rfc2697.cir > 0 &&
 156                                         (profile->srtcm_rfc2697.cir <<
 157                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 158                                         <= MLX5_SRTCM_CIR_MAX &&
 159                                         profile->srtcm_rfc2697.cbs > 0 &&
 160                                         (profile->srtcm_rfc2697.cbs <<
 161                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 162                                         <= MLX5_SRTCM_CBS_MAX &&
 163                                         (profile->srtcm_rfc2697.ebs <<
 164                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 165                                         <= MLX5_SRTCM_EBS_MAX)
 166                                         return 0;
 167                                 return -rte_mtr_error_set
 168                                              (error, ENOTSUP,
 169                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 170                                               NULL,
 171                                               profile->srtcm_rfc2697.ebs ?
 172                                               "Metering value ebs must be 0." :
 173                                               "Invalid metering parameters.");
 174                         }
 175                         if (profile->srtcm_rfc2697.cir > 0 &&
 176                                 profile->srtcm_rfc2697.cir <=
 177                                                 MLX5_SRTCM_CIR_MAX &&
 178                                 profile->srtcm_rfc2697.cbs > 0 &&
 179                                 profile->srtcm_rfc2697.cbs <=
 180                                                 MLX5_SRTCM_CBS_MAX &&
 181                                 profile->srtcm_rfc2697.ebs <=
 182                                                 MLX5_SRTCM_EBS_MAX)
 183                                 return 0;
 184                         return -rte_mtr_error_set(error, ENOTSUP,
 185                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 186                                         NULL,
 187                                         profile->srtcm_rfc2697.ebs ?
 188                                         "Metering value ebs must be 0." :
 189                                         "Invalid metering parameters.");
 190                 }
 191         }
 192         return -rte_mtr_error_set(error, ENOTSUP,
 193                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 194                                   NULL, "Metering algorithm not supported.");
 195 }
 196
 197 /**
 198  * Calculate mantissa and exponent for cir.
 199  *
 200  * @param[in] cir
 201  *   Value to be calculated.
 202  * @param[out] man
 203  *   Pointer to the mantissa.
 204  * @param[out] exp
 205  *   Pointer to the exp.
 206  */
 207 static void
 208 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 209 {
 210         int64_t _cir;
 211         int64_t delta = INT64_MAX;
 212         uint8_t _man = 0;
 213         uint8_t _exp = 0;
 214         uint64_t m, e;
 215
 216         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 217                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 218                         _cir = (1000000000ULL * m) >> e;
 219                         if (llabs(cir - _cir) <= delta) {
 220                                 delta = llabs(cir - _cir);
 221                                 _man = m;
 222                                 _exp = e;
 223                         }
 224                 }
 225         }
 226         *man = _man;
 227         *exp = _exp;
 228 }
 229
 230 /**
 231  * Calculate mantissa and exponent for xbs.
 232  *
 233  * @param[in] xbs
 234  *   Value to be calculated.
 235  * @param[out] man
 236  *   Pointer to the mantissa.
 237  * @param[out] exp
 238  *   Pointer to the exp.
 239  */
 240 static void
 241 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 242 {
 243         int _exp;
 244         double _man;
 245
 246         /* Special case xbs == 0 ? both exp and matissa are 0. */
 247         if (xbs == 0) {
 248                 *man = 0;
 249                 *exp = 0;
 250                 return;
 251         }
 252         /* xbs = xbs_mantissa * 2^xbs_exponent */
 253         _man = frexp(xbs, &_exp);
 254         _man = _man * pow(2, MLX5_MAN_WIDTH);
 255         _exp = _exp - MLX5_MAN_WIDTH;
 256         *man = (uint8_t)ceil(_man);
 257         *exp = _exp;
 258 }
 259
 260 /**
 261  * Fill the prm meter parameter.
 262  *
 263  * @param[in,out] fmp
 264  *   Pointer to meter profie to be converted.
 265  * @param[out] error
 266  *   Pointer to the error structure.
 267  *
 268  * @return
 269  *   0 on success, a negative errno value otherwise and rte_errno is set.
 270  */
 271 static int
 272 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 273                         struct mlx5_priv *priv, struct rte_mtr_error *error)
 274 {
 275         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 276         uint8_t man, exp;
 277         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 278         uint32_t ebs_exp, ebs_man;
 279         uint64_t cir, cbs, ebs;
 280
 281         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 282                 return -rte_mtr_error_set(error, ENOTSUP,
 283                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 284                                 NULL, "Metering algorithm not supported.");
 285         if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
 286                 return -rte_mtr_error_set(error, ENOTSUP,
 287                         RTE_MTR_ERROR_TYPE_METER_PROFILE,
 288                         NULL, "Metering algorithm packet mode not supported.");
 289         if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
 290                 cir = fmp->profile.srtcm_rfc2697.cir <<
 291                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 292                 cbs = fmp->profile.srtcm_rfc2697.cbs <<
 293                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 294                 ebs = fmp->profile.srtcm_rfc2697.ebs <<
 295                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 296         } else {
 297                 cir = fmp->profile.srtcm_rfc2697.cir;
 298                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 299                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 300         }
 301         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 302         mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
 303         /* Check if cir mantissa is too large. */
 304         if (exp > ASO_DSEG_CIR_EXP_MASK)
 305                 return -rte_mtr_error_set(error, ENOTSUP,
 306                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 307                                           "meter profile parameter cir is"
 308                                           " not supported.");
 309         cir_man = man;
 310         cir_exp = exp;
 311          /* cbs = cbs_mantissa * 2^cbs_exponent */
 312         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 313         /* Check if cbs mantissa is too large. */
 314         if (exp > ASO_DSEG_EXP_MASK)
 315                 return -rte_mtr_error_set(error, ENOTSUP,
 316                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 317                                           "meter profile parameter cbs is"
 318                                           " not supported.");
 319         cbs_man = man;
 320         cbs_exp = exp;
 321         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 322                                           cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 323                                           cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 324                                           cir_man);
 325         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 326         /* Check if ebs mantissa is too large. */
 327         if (exp > ASO_DSEG_EXP_MASK)
 328                 return -rte_mtr_error_set(error, ENOTSUP,
 329                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 330                                           "meter profile parameter ebs is"
 331                                           " not supported.");
 332         ebs_man = man;
 333         ebs_exp = exp;
 334         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 335                                           ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 336         if (srtcm->cbs_cir)
 337                 fmp->g_support = 1;
 338         if (srtcm->ebs_eir)
 339                 fmp->y_support = 1;
 340         return 0;
 341 }
 342
 343 /**
 344  * Callback to get MTR capabilities.
 345  *
 346  * @param[in] dev
 347  *   Pointer to Ethernet device.
 348  * @param[out] cap
 349  *   Pointer to save MTR capabilities.
 350  * @param[out] error
 351  *   Pointer to the error structure.
 352  *
 353  * @return
 354  *   0 on success, a negative errno value otherwise and rte_errno is set.
 355  */
 356 static int
 357 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 358                  struct rte_mtr_capabilities *cap,
 359                  struct rte_mtr_error *error __rte_unused)
 360 {
 361         struct mlx5_priv *priv = dev->data->dev_private;
 362         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 363
 364         if (!priv->mtr_en)
 365                 return -rte_mtr_error_set(error, ENOTSUP,
 366                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 367                                           "Meter is not supported");
 368         memset(cap, 0, sizeof(*cap));
 369         if (priv->sh->meter_aso_en) {
 370                 /* 2 meters per one ASO cache line. */
 371                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 372                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 373         } else {
 374                 cap->n_max = 1 << qattr->log_max_flow_meter;
 375                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 376         }
 377         cap->srtcm_rfc2697_byte_mode_supported = 1;
 378         cap->n_shared_max = cap->n_max;
 379         cap->identical = 1;
 380         cap->shared_identical = 1;
 381         cap->shared_n_flows_per_mtr_max = 4 << 20;
 382         /* 2M flows can share the same meter. */
 383         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 384         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 385         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 386         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 387                           RTE_MTR_STATS_N_PKTS_DROPPED;
 388         return 0;
 389 }
 390
 391 /**
 392  * Callback to add MTR profile.
 393  *
 394  * @param[in] dev
 395  *   Pointer to Ethernet device.
 396  * @param[in] meter_profile_id
 397  *   Meter profile id.
 398  * @param[in] profile
 399  *   Pointer to meter profile detail.
 400  * @param[out] error
 401  *   Pointer to the error structure.
 402  *
 403  * @return
 404  *   0 on success, a negative errno value otherwise and rte_errno is set.
 405  */
 406 static int
 407 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 408                        uint32_t meter_profile_id,
 409                        struct rte_mtr_meter_profile *profile,
 410                        struct rte_mtr_error *error)
 411 {
 412         struct mlx5_priv *priv = dev->data->dev_private;
 413         struct mlx5_flow_meter_profile *fmp;
 414         union mlx5_l3t_data data;
 415         int ret;
 416
 417         if (!priv->mtr_en)
 418                 return -rte_mtr_error_set(error, ENOTSUP,
 419                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 420                                           "Meter is not supported");
 421         /* Check input params. */
 422         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 423                                                profile, error);
 424         if (ret)
 425                 return ret;
 426         /* Meter profile memory allocation. */
 427         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 428                           RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 429         if (fmp == NULL)
 430                 return -rte_mtr_error_set(error, ENOMEM,
 431                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 432                                           NULL, "Meter profile memory "
 433                                           "alloc failed.");
 434         /* Fill profile info. */
 435         fmp->id = meter_profile_id;
 436         fmp->profile = *profile;
 437         /* Fill the flow meter parameters for the PRM. */
 438         ret = mlx5_flow_meter_param_fill(fmp, priv, error);
 439         if (ret)
 440                 goto error;
 441         data.ptr = fmp;
 442         ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
 443                                  meter_profile_id, &data);
 444         if (ret)
 445                 return -rte_mtr_error_set(error, ENOTSUP,
 446                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 447                                           NULL, "Meter profile insert fail.");
 448         return 0;
 449 error:
 450         mlx5_free(fmp);
 451         return ret;
 452 }
 453
 454 /**
 455  * Callback to delete MTR profile.
 456  *
 457  * @param[in] dev
 458  *   Pointer to Ethernet device.
 459  * @param[in] meter_profile_id
 460  *   Meter profile id.
 461  * @param[out] error
 462  *   Pointer to the error structure.
 463  *
 464  * @return
 465  *   0 on success, a negative errno value otherwise and rte_errno is set.
 466  */
 467 static int
 468 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 469                           uint32_t meter_profile_id,
 470                           struct rte_mtr_error *error)
 471 {
 472         struct mlx5_priv *priv = dev->data->dev_private;
 473         struct mlx5_flow_meter_profile *fmp;
 474
 475         if (!priv->mtr_en)
 476                 return -rte_mtr_error_set(error, ENOTSUP,
 477                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 478                                           "Meter is not supported");
 479         /* Meter profile must exist. */
 480         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 481         if (fmp == NULL)
 482                 return -rte_mtr_error_set(error, ENOENT,
 483                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 484                                           &meter_profile_id,
 485                                           "Meter profile id is invalid.");
 486         /* Check profile is unused. */
 487         if (fmp->ref_cnt)
 488                 return -rte_mtr_error_set(error, EBUSY,
 489                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 490                                           NULL, "Meter profile is in use.");
 491         if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
 492                 return -rte_mtr_error_set(error, EBUSY,
 493                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 494                                           NULL, "Meter profile remove fail.");
 495         mlx5_free(fmp);
 496         return 0;
 497 }
 498
 499 /**
 500  * Find policy by id.
 501  *
 502  * @param[in] dev
 503  *   Pointer to Ethernet device.
 504  * @param policy_id
 505  *   Policy id.
 506  *
 507  * @return
 508  *   Pointer to the policy found on success, NULL otherwise.
 509  */
 510 struct mlx5_flow_meter_policy *
 511 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 512                             uint32_t policy_id,
 513                             uint32_t *policy_idx)
 514 {
 515         struct mlx5_priv *priv = dev->data->dev_private;
 516         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 517         union mlx5_l3t_data data;
 518
 519         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
 520                 return NULL;
 521         if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
 522                                 !data.dword)
 523                 return NULL;
 524         if (policy_idx)
 525                 *policy_idx = data.dword;
 526         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 527                                         data.dword);
 528         /* Remove reference taken by the mlx5_l3t_get_entry. */
 529         mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
 530         if (sub_policy)
 531                 if (sub_policy->main_policy_id)
 532                         return sub_policy->main_policy;
 533         return NULL;
 534 }
 535
 536 /**
 537  * Get the last meter's policy from one meter's policy in hierarchy.
 538  *
 539  * @param[in] dev
 540  *   Pointer to Ethernet device.
 541  * @param[in] policy
 542  *   Pointer to flow meter policy.
 543  *
 544  * @return
 545  *   Pointer to the final meter's policy, or NULL when fail.
 546  */
 547 struct mlx5_flow_meter_policy *
 548 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
 549                                         struct mlx5_flow_meter_policy *policy)
 550 {
 551         struct mlx5_priv *priv = dev->data->dev_private;
 552         struct mlx5_flow_meter_info *next_fm;
 553         struct mlx5_flow_meter_policy *next_policy = policy;
 554
 555         while (next_policy->is_hierarchy) {
 556                 next_fm = mlx5_flow_meter_find(priv,
 557                        next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
 558                 if (!next_fm || next_fm->def_policy)
 559                         return NULL;
 560                 next_policy = mlx5_flow_meter_policy_find(dev,
 561                                                 next_fm->policy_id, NULL);
 562                 MLX5_ASSERT(next_policy);
 563         }
 564         return next_policy;
 565 }
 566
 567 /**
 568  * Callback to check MTR policy action validate
 569  *
 570  * @param[in] dev
 571  *   Pointer to Ethernet device.
 572  * @param[in] actions
 573  *   Pointer to meter policy action detail.
 574  * @param[out] error
 575  *   Pointer to the error structure.
 576  *
 577  * @return
 578  *   0 on success, a negative errno value otherwise and rte_errno is set.
 579  */
 580 static int
 581 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 582         struct rte_mtr_meter_policy_params *policy,
 583         struct rte_mtr_error *error)
 584 {
 585         struct mlx5_priv *priv = dev->data->dev_private;
 586         struct rte_flow_attr attr = { .transfer =
 587                         priv->config.dv_esw_en ? 1 : 0};
 588         bool is_rss = false;
 589         uint8_t policy_mode;
 590         uint8_t domain_bitmap;
 591         int ret;
 592
 593         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 594                 return -rte_mtr_error_set(error, ENOTSUP,
 595                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 596                                 NULL, "meter policy unsupported.");
 597         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 598                         &is_rss, &domain_bitmap, &policy_mode, error);
 599         if (ret)
 600                 return ret;
 601         return 0;
 602 }
 603
 604 static int
 605 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 606                         uint32_t policy_id,
 607                         struct mlx5_flow_meter_policy *mtr_policy,
 608                         struct rte_mtr_error *error,
 609                         bool clear_l3t)
 610 {
 611         struct mlx5_priv *priv = dev->data->dev_private;
 612         struct mlx5_flow_meter_sub_policy *sub_policy;
 613         uint32_t i, j;
 614         uint16_t sub_policy_num;
 615
 616         rte_spinlock_lock(&mtr_policy->sl);
 617         if (mtr_policy->ref_cnt) {
 618                 rte_spinlock_unlock(&mtr_policy->sl);
 619                 return -rte_mtr_error_set(error, EBUSY,
 620                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 621                                  NULL,
 622                                 "Meter policy object is being used.");
 623         }
 624         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 625         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 626         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 627                 sub_policy_num = (mtr_policy->sub_policy_num >>
 628                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 629                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 630                 if (sub_policy_num) {
 631                         for (j = 0; j < sub_policy_num; j++) {
 632                                 sub_policy = mtr_policy->sub_policys[i][j];
 633                                 if (sub_policy)
 634                                         mlx5_ipool_free
 635                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 636                                         sub_policy->idx);
 637                         }
 638                 }
 639         }
 640         if (priv->policy_idx_tbl && clear_l3t) {
 641                 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
 642                         rte_spinlock_unlock(&mtr_policy->sl);
 643                         return -rte_mtr_error_set(error, ENOTSUP,
 644                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 645                                 "Fail to delete policy in index table.");
 646                 }
 647         }
 648         rte_spinlock_unlock(&mtr_policy->sl);
 649         return 0;
 650 }
 651
 652 /**
 653  * Callback to add MTR policy.
 654  *
 655  * @param[in] dev
 656  *   Pointer to Ethernet device.
 657  * @param[out] policy_id
 658  *   Pointer to policy id
 659  * @param[in] actions
 660  *   Pointer to meter policy action detail.
 661  * @param[out] error
 662  *   Pointer to the error structure.
 663  *
 664  * @return
 665  *   0 on success, a negative errno value otherwise and rte_errno is set.
 666  */
 667 static int
 668 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 669                         uint32_t policy_id,
 670                         struct rte_mtr_meter_policy_params *policy,
 671                         struct rte_mtr_error *error)
 672 {
 673         struct mlx5_priv *priv = dev->data->dev_private;
 674         struct rte_flow_attr attr = { .transfer =
 675                         priv->config.dv_esw_en ? 1 : 0};
 676         uint32_t sub_policy_idx = 0;
 677         uint32_t policy_idx = 0;
 678         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 679         struct mlx5_flow_meter_sub_policy *sub_policy;
 680         bool is_rss = false;
 681         uint8_t policy_mode;
 682         uint32_t i;
 683         int ret;
 684         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 685         uint16_t sub_policy_num;
 686         uint8_t domain_bitmap = 0;
 687         union mlx5_l3t_data data;
 688         bool skip_rule = false;
 689
 690         if (!priv->mtr_en)
 691                 return -rte_mtr_error_set(error, ENOTSUP,
 692                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 693                                           NULL, "meter policy unsupported. ");
 694         if (policy_id == MLX5_INVALID_POLICY_ID)
 695                 return -rte_mtr_error_set(error, ENOTSUP,
 696                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 697                                           NULL, "policy ID is invalid. ");
 698         if (policy_id == priv->sh->mtrmng->def_policy_id)
 699                 return -rte_mtr_error_set(error, EEXIST,
 700                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 701                                           NULL, "default policy ID exists. ");
 702         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 703         if (mtr_policy)
 704                 return -rte_mtr_error_set(error, EEXIST,
 705                                           RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 706                                           NULL, "policy ID exists. ");
 707         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 708                                           &is_rss, &domain_bitmap,
 709                                           &policy_mode, error);
 710         if (ret)
 711                 return ret;
 712         if (!domain_bitmap)
 713                 return -rte_mtr_error_set(error, ENOTSUP,
 714                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 715                                           NULL, "fail to find policy domain.");
 716         if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
 717                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 718                         return -rte_mtr_error_set(error, EEXIST,
 719                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 720                                 NULL, "a policy with similar actions "
 721                                 "is already configured");
 722                 if (mlx5_flow_create_def_policy(dev))
 723                         return -rte_mtr_error_set(error, ENOTSUP,
 724                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 725                                 NULL,
 726                                 "fail to create non-terminated policy.");
 727                 priv->sh->mtrmng->def_policy_id = policy_id;
 728                 return 0;
 729         }
 730         if (!priv->sh->meter_aso_en)
 731                 return -rte_mtr_error_set(error, ENOTSUP,
 732                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 733                         "no ASO capability to support the policy ");
 734         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 735                 if (!(domain_bitmap & (1 << i)))
 736                         continue;
 737                 /*
 738                  * If RSS is found, it means that only the ingress domain can
 739                  * be supported. It is invalid to support RSS for one color
 740                  * and egress / transfer domain actions for another. Drop and
 741                  * jump action should have no impact.
 742                  */
 743                 if (is_rss) {
 744                         policy_size +=
 745                                 sizeof(struct mlx5_flow_meter_sub_policy *) *
 746                                 MLX5_MTR_RSS_MAX_SUB_POLICY;
 747                         break;
 748                 }
 749                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 750         }
 751         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 752                                  RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 753         if (!mtr_policy)
 754                 return -rte_mtr_error_set(error, ENOMEM,
 755                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 756                                 "Memory alloc failed for meter policy.");
 757         if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
 758                 mtr_policy->skip_y = 1;
 759         else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
 760                 mtr_policy->skip_g = 1;
 761         policy_size = sizeof(struct mlx5_flow_meter_policy);
 762         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 763                 if (!(domain_bitmap & (1 << i)))
 764                         continue;
 765                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 766                         mtr_policy->ingress = 1;
 767                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 768                         mtr_policy->egress = 1;
 769                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 770                         mtr_policy->transfer = 1;
 771                 sub_policy = mlx5_ipool_zmalloc
 772                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 773                                  &sub_policy_idx);
 774                 if (!sub_policy ||
 775                     sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 776                         goto policy_add_err;
 777                 sub_policy->idx = sub_policy_idx;
 778                 sub_policy->main_policy = mtr_policy;
 779                 if (!policy_idx) {
 780                         policy_idx = sub_policy_idx;
 781                         sub_policy->main_policy_id = 1;
 782                 }
 783                 mtr_policy->sub_policys[i] =
 784                         (struct mlx5_flow_meter_sub_policy **)
 785                         ((uint8_t *)mtr_policy + policy_size);
 786                 mtr_policy->sub_policys[i][0] = sub_policy;
 787                 sub_policy_num = (mtr_policy->sub_policy_num >>
 788                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 789                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 790                 sub_policy_num++;
 791                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 792                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 793                 mtr_policy->sub_policy_num |=
 794                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 795                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 796                 /*
 797                  * If RSS is found, it means that only the ingress domain can
 798                  * be supported. It is invalid to support RSS for one color
 799                  * and egress / transfer domain actions for another. Drop and
 800                  * jump action should have no impact.
 801                  */
 802                 if (is_rss) {
 803                         mtr_policy->is_rss = 1;
 804                         break;
 805                 }
 806                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 807         }
 808         rte_spinlock_init(&mtr_policy->sl);
 809         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 810                                         policy->actions, error);
 811         if (ret)
 812                 goto policy_add_err;
 813         if (mtr_policy->is_hierarchy) {
 814                 struct mlx5_flow_meter_policy *final_policy;
 815
 816                 final_policy =
 817                 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
 818                 if (!final_policy)
 819                         goto policy_add_err;
 820                 skip_rule = (final_policy->is_rss || final_policy->is_queue);
 821         }
 822         /*
 823          * If either Green or Yellow has queue / RSS action, all the policy
 824          * rules will be created later in the flow splitting stage.
 825          */
 826         if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
 827                 /* Create policy rules in HW. */
 828                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 829                 if (ret)
 830                         goto policy_add_err;
 831         }
 832         data.dword = policy_idx;
 833         if (!priv->policy_idx_tbl) {
 834                 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 835                 if (!priv->policy_idx_tbl)
 836                         goto policy_add_err;
 837         }
 838         if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
 839                 goto policy_add_err;
 840         return 0;
 841 policy_add_err:
 842         if (mtr_policy) {
 843                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 844                         mtr_policy, error, false);
 845                 mlx5_free(mtr_policy);
 846                 if (ret)
 847                         return ret;
 848         }
 849         return -rte_mtr_error_set(error, ENOTSUP,
 850                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 851                                   NULL, "Failed to create devx policy.");
 852 }
 853
 854 /**
 855  * Callback to delete MTR policy.
 856  *
 857  * @param[in] dev
 858  *   Pointer to Ethernet device.
 859  * @param[in] policy_id
 860  *   Meter policy id.
 861  * @param[out] error
 862  *   Pointer to the error structure.
 863  *
 864  * @return
 865  *   0 on success, a negative errno value otherwise and rte_errno is set.
 866  */
 867 static int
 868 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 869                           uint32_t policy_id,
 870                           struct rte_mtr_error *error)
 871 {
 872         struct mlx5_priv *priv = dev->data->dev_private;
 873         struct mlx5_flow_meter_policy *mtr_policy;
 874         uint32_t policy_idx;
 875         int ret;
 876
 877         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 878                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 879                         return -rte_mtr_error_set(error, ENOTSUP,
 880                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 881                                 "Meter policy object is being used.");
 882                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 883                 return 0;
 884         }
 885         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 886         if (!mtr_policy)
 887                 return -rte_mtr_error_set(error, ENOTSUP,
 888                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 889                         "Meter policy id is invalid. ");
 890         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 891                                                 error, true);
 892         if (ret)
 893                 return ret;
 894         mlx5_free(mtr_policy);
 895         return 0;
 896 }
 897
 898 /**
 899  * Check meter validation.
 900  *
 901  * @param[in] priv
 902  *   Pointer to mlx5 private data structure.
 903  * @param[in] meter_id
 904  *   Meter id.
 905  * @param[in] params
 906  *   Pointer to rte meter parameters.
 907  * @param[out] error
 908  *   Pointer to rte meter error structure.
 909  *
 910  * @return
 911  *   0 on success, a negative errno value otherwise and rte_errno is set.
 912  */
 913 static int
 914 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 915                          struct rte_mtr_params *params,
 916                          struct rte_mtr_error *error)
 917 {
 918         /* Meter must use global drop action. */
 919         if (!priv->sh->dr_drop_action)
 920                 return -rte_mtr_error_set(error, ENOTSUP,
 921                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 922                                           NULL,
 923                                           "No drop action ready for meter.");
 924         /* Meter params must not be NULL. */
 925         if (params == NULL)
 926                 return -rte_mtr_error_set(error, EINVAL,
 927                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 928                                           NULL, "Meter object params null.");
 929         /* Previous meter color is not supported. */
 930         if (params->use_prev_mtr_color)
 931                 return -rte_mtr_error_set(error, ENOTSUP,
 932                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 933                                           NULL,
 934                                           "Previous meter color "
 935                                           "not supported.");
 936         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 937                 return -rte_mtr_error_set(error, ENOENT,
 938                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 939                                 NULL, "Meter policy id not valid.");
 940         /* Validate meter id. */
 941         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 942                 return -rte_mtr_error_set(error, EEXIST,
 943                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 944                         "Meter object already exists.");
 945         return 0;
 946 }
 947
 948 /**
 949  * Modify the flow meter action.
 950  *
 951  * @param[in] priv
 952  *   Pointer to mlx5 private data structure.
 953  * @param[in] fm
 954  *   Pointer to flow meter to be modified.
 955  * @param[in] srtcm
 956  *   Pointer to meter srtcm description parameter.
 957  * @param[in] modify_bits
 958  *   The bit in srtcm to be updated.
 959  * @param[in] active_state
 960  *   The state to be updated.
 961  * @return
 962  *   0 on success, o negative value otherwise.
 963  */
 964 static int
 965 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 966                 struct mlx5_flow_meter_info *fm,
 967                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 968                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 969 {
 970 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 971         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 972         uint32_t *attr;
 973         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 974         int ret;
 975         struct mlx5_aso_mtr *aso_mtr = NULL;
 976         uint32_t cbs_cir, ebs_eir, val;
 977
 978         if (priv->sh->meter_aso_en) {
 979                 fm->is_enable = !!is_enable;
 980                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 981                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 982                 if (ret)
 983                         return ret;
 984                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 985                 if (ret)
 986                         return ret;
 987         } else {
 988                 /* Fill command parameters. */
 989                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 990                 mod_attr.flow_meter_parameter = in;
 991                 mod_attr.flow_meter_parameter_sz =
 992                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 993                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 994                         mod_attr.active = !!active_state;
 995                 else
 996                         mod_attr.active = 0;
 997                 attr = in;
 998                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 999                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1000                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1001                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1002                                 ASO_DSEG_EXP_MASK;
1003                         MLX5_SET(flow_meter_parameters, attr,
1004                                 cbs_exponent, val);
1005                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1006                                 ASO_DSEG_MAN_MASK;
1007                         MLX5_SET(flow_meter_parameters, attr,
1008                                 cbs_mantissa, val);
1009                 }
1010                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1011                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
1012                                 ASO_DSEG_EXP_MASK;
1013                         MLX5_SET(flow_meter_parameters, attr,
1014                                 cir_exponent, val);
1015                         val = cbs_cir & ASO_DSEG_MAN_MASK;
1016                         MLX5_SET(flow_meter_parameters, attr,
1017                                 cir_mantissa, val);
1018                 }
1019                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1020                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1021                                 ASO_DSEG_EXP_MASK;
1022                         MLX5_SET(flow_meter_parameters, attr,
1023                                 ebs_exponent, val);
1024                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1025                                 ASO_DSEG_MAN_MASK;
1026                         MLX5_SET(flow_meter_parameters, attr,
1027                                 ebs_mantissa, val);
1028                 }
1029                 /* Apply modifications to meter only if it was created. */
1030                 if (fm->meter_action) {
1031                         ret = mlx5_glue->dv_modify_flow_action_meter
1032                                         (fm->meter_action, &mod_attr,
1033                                         rte_cpu_to_be_64(modify_bits));
1034                         if (ret)
1035                                 return ret;
1036                 }
1037                 /* Update succeedded modify meter parameters. */
1038                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1039                         fm->active_state = !!active_state;
1040         }
1041         return 0;
1042 #else
1043         (void)priv;
1044         (void)fm;
1045         (void)srtcm;
1046         (void)modify_bits;
1047         (void)active_state;
1048         (void)is_enable;
1049         return -ENOTSUP;
1050 #endif
1051 }
1052
1053 static int
1054 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1055                                 struct mlx5_flow_meter_info *fm,
1056                                 uint64_t stats_mask)
1057 {
1058         fm->bytes_dropped =
1059                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1060         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1061         if (fm->bytes_dropped || fm->pkts_dropped) {
1062                 if (!fm->drop_cnt) {
1063                         /* Alloc policer counters. */
1064                         fm->drop_cnt = mlx5_counter_alloc(dev);
1065                         if (!fm->drop_cnt)
1066                                 return -1;
1067                 }
1068         } else {
1069                 if (fm->drop_cnt) {
1070                         mlx5_counter_free(dev, fm->drop_cnt);
1071                         fm->drop_cnt = 0;
1072                 }
1073         }
1074         return 0;
1075 }
1076
1077 /**
1078  * Create meter rules.
1079  *
1080  * @param[in] dev
1081  *   Pointer to Ethernet device.
1082  * @param[in] meter_id
1083  *   Meter id.
1084  * @param[in] params
1085  *   Pointer to rte meter parameters.
1086  * @param[in] shared
1087  *   Meter shared with other flow or not.
1088  * @param[out] error
1089  *   Pointer to rte meter error structure.
1090  *
1091  * @return
1092  *   0 on success, a negative errno value otherwise and rte_errno is set.
1093  */
1094 static int
1095 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1096                        struct rte_mtr_params *params, int shared,
1097                        struct rte_mtr_error *error)
1098 {
1099         struct mlx5_priv *priv = dev->data->dev_private;
1100         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1101         struct mlx5_flow_meter_profile *fmp;
1102         struct mlx5_flow_meter_info *fm;
1103         struct mlx5_legacy_flow_meter *legacy_fm;
1104         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1105         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1106                 .size = 0,
1107                 .trunk_size = 64,
1108                 .need_lock = 1,
1109                 .type = "mlx5_flow_mtr_flow_id_pool",
1110         };
1111         struct mlx5_aso_mtr *aso_mtr;
1112         uint32_t mtr_idx, policy_idx;
1113         union mlx5_l3t_data data;
1114         int ret;
1115         uint8_t domain_bitmap;
1116         uint8_t mtr_id_bits;
1117         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1118                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1119
1120         if (!priv->mtr_en)
1121                 return -rte_mtr_error_set(error, ENOTSUP,
1122                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1123                                         "Meter is not supported");
1124         /* Validate the parameters. */
1125         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1126         if (ret)
1127                 return ret;
1128         /* Meter profile must exist. */
1129         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1130         if (fmp == NULL)
1131                 return -rte_mtr_error_set(error, ENOENT,
1132                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1133                         NULL, "Meter profile id not valid.");
1134         /* Meter policy must exist. */
1135         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1136                 __atomic_add_fetch
1137                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1138                         1, __ATOMIC_RELAXED);
1139                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1140                 if (!priv->config.dv_esw_en)
1141                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1142         } else {
1143                 if (!priv->sh->meter_aso_en)
1144                         return -rte_mtr_error_set(error, ENOTSUP,
1145                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1146                                 "Part of the policies cannot be "
1147                                 "supported without ASO ");
1148                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1149                                 params->meter_policy_id, &policy_idx);
1150                 if (!mtr_policy)
1151                         return -rte_mtr_error_set(error, ENOENT,
1152                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1153                                 NULL, "Meter policy id not valid.");
1154                 domain_bitmap = (mtr_policy->ingress ?
1155                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1156                                 (mtr_policy->egress ?
1157                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1158                                 (mtr_policy->transfer ?
1159                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1160                 if (fmp->g_support && mtr_policy->skip_g)
1161                         return -rte_mtr_error_set(error, ENOTSUP,
1162                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1163                                         NULL, "Meter green policy is empty.");
1164                 if (fmp->y_support && mtr_policy->skip_y)
1165                         return -rte_mtr_error_set(error, ENOTSUP,
1166                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1167                                         NULL, "Meter yellow policy is empty.");
1168         }
1169         /* Allocate the flow meter memory. */
1170         if (priv->sh->meter_aso_en) {
1171                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1172                 if (!mtr_idx)
1173                         return -rte_mtr_error_set(error, ENOMEM,
1174                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1175                                 "Memory alloc failed for meter.");
1176                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1177                 fm = &aso_mtr->fm;
1178         } else {
1179                 legacy_fm = mlx5_ipool_zmalloc
1180                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1181                 if (legacy_fm == NULL)
1182                         return -rte_mtr_error_set(error, ENOMEM,
1183                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1184                                 "Memory alloc failed for meter.");
1185                 legacy_fm->idx = mtr_idx;
1186                 fm = &legacy_fm->fm;
1187         }
1188         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1189         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1190             mtr_reg_bits) {
1191                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1192                 goto error;
1193         }
1194         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1195                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1196         /* Fill the flow meter parameters. */
1197         fm->meter_id = meter_id;
1198         fm->policy_id = params->meter_policy_id;
1199         fm->profile = fmp;
1200         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1201                 goto error;
1202         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1203                 goto error;
1204         /* Add to the flow meter list. */
1205         if (!priv->sh->meter_aso_en)
1206                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1207         /* Add to the flow meter list. */
1208         fm->active_state = 1; /* Config meter starts as active. */
1209         fm->is_enable = 1;
1210         fm->shared = !!shared;
1211         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1212         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1213                 fm->def_policy = 1;
1214                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1215                 if (!fm->flow_ipool)
1216                         goto error;
1217         }
1218         rte_spinlock_init(&fm->sl);
1219         /* If ASO meter supported, update ASO flow meter by wqe. */
1220         if (priv->sh->meter_aso_en) {
1221                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1222                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1223                 if (ret)
1224                         goto error;
1225                 if (!priv->mtr_idx_tbl) {
1226                         priv->mtr_idx_tbl =
1227                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1228                         if (!priv->mtr_idx_tbl)
1229                                 goto error;
1230                 }
1231                 data.dword = mtr_idx;
1232                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1233                         goto error;
1234         }
1235         if (mtr_policy)
1236                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1237         return 0;
1238 error:
1239         mlx5_flow_destroy_mtr_tbls(dev, fm);
1240         /* Free policer counters. */
1241         if (fm->drop_cnt)
1242                 mlx5_counter_free(dev, fm->drop_cnt);
1243         if (priv->sh->meter_aso_en)
1244                 mlx5_flow_mtr_free(dev, mtr_idx);
1245         else
1246                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1247         return -rte_mtr_error_set(error, ENOTSUP,
1248                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1249                 NULL, "Failed to create devx meter.");
1250 }
1251
1252 static int
1253 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1254                         struct mlx5_flow_meter_info *fm,
1255                         uint32_t mtr_idx)
1256 {
1257         struct mlx5_priv *priv = dev->data->dev_private;
1258         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1259         struct mlx5_flow_meter_profile *fmp;
1260         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1261         struct mlx5_flow_meter_policy *mtr_policy;
1262
1263         /* Meter object must not have any owner. */
1264         MLX5_ASSERT(!fm->ref_cnt);
1265         /* Get meter profile. */
1266         fmp = fm->profile;
1267         if (fmp == NULL)
1268                 return -1;
1269         /* Update dependencies. */
1270         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1271         fm->profile = NULL;
1272         /* Remove from list. */
1273         if (!priv->sh->meter_aso_en) {
1274                 legacy_fm = container_of(fm,
1275                         struct mlx5_legacy_flow_meter, fm);
1276                 TAILQ_REMOVE(fms, legacy_fm, next);
1277         }
1278         /* Free drop counters. */
1279         if (fm->drop_cnt)
1280                 mlx5_counter_free(dev, fm->drop_cnt);
1281         /* Free meter flow table. */
1282         if (fm->flow_ipool) {
1283                 mlx5_ipool_destroy(fm->flow_ipool);
1284                 fm->flow_ipool = 0;
1285         }
1286         mlx5_flow_destroy_mtr_tbls(dev, fm);
1287         if (fm->def_policy)
1288                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1289                                 1, __ATOMIC_RELAXED);
1290         if (priv->sh->meter_aso_en) {
1291                 if (!fm->def_policy) {
1292                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1293                                                 fm->policy_id, NULL);
1294                         if (mtr_policy)
1295                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1296                                                 1, __ATOMIC_RELAXED);
1297                         fm->policy_id = 0;
1298                 }
1299                 fm->def_policy = 0;
1300                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1301                         return -1;
1302                 mlx5_flow_mtr_free(dev, mtr_idx);
1303         } else {
1304                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1305                                         legacy_fm->idx);
1306         }
1307         return 0;
1308 }
1309
1310 /**
1311  * Destroy meter rules.
1312  *
1313  * @param[in] dev
1314  *   Pointer to Ethernet device.
1315  * @param[in] meter_id
1316  *   Meter id.
1317  * @param[out] error
1318  *   Pointer to rte meter error structure.
1319  *
1320  * @return
1321  *   0 on success, a negative errno value otherwise and rte_errno is set.
1322  */
1323 static int
1324 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1325                         struct rte_mtr_error *error)
1326 {
1327         struct mlx5_priv *priv = dev->data->dev_private;
1328         struct mlx5_flow_meter_info *fm;
1329         uint32_t mtr_idx = 0;
1330
1331         if (!priv->mtr_en)
1332                 return -rte_mtr_error_set(error, ENOTSUP,
1333                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1334                                           NULL,
1335                                           "Meter is not supported");
1336         /* Meter object must exist. */
1337         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1338         if (fm == NULL)
1339                 return -rte_mtr_error_set(error, ENOENT,
1340                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1341                                           NULL,
1342                                           "Meter object id not valid.");
1343         /* Meter object must not have any owner. */
1344         if (fm->ref_cnt > 0)
1345                 return -rte_mtr_error_set(error, EBUSY,
1346                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1347                                           NULL,
1348                                           "Meter object is being used.");
1349         /* Destroy the meter profile. */
1350         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1351                 return -rte_mtr_error_set(error, EINVAL,
1352                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1353                                         NULL,
1354                                         "MTR object meter profile invalid.");
1355         return 0;
1356 }
1357
1358 /**
1359  * Modify meter state.
1360  *
1361  * @param[in] priv
1362  *   Pointer to mlx5 private data structure.
1363  * @param[in] fm
1364  *   Pointer to flow meter.
1365  * @param[in] new_state
1366  *   New state to update.
1367  * @param[out] error
1368  *   Pointer to rte meter error structure.
1369  *
1370  * @return
1371  *   0 on success, a negative errno value otherwise and rte_errno is set.
1372  */
1373 static int
1374 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1375                              struct mlx5_flow_meter_info *fm,
1376                              uint32_t new_state,
1377                              struct rte_mtr_error *error)
1378 {
1379         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1380                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1381                 .ebs_eir = 0,
1382         };
1383         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1384                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1385         int ret;
1386
1387         if (new_state == MLX5_FLOW_METER_DISABLE)
1388                 ret = mlx5_flow_meter_action_modify(priv, fm,
1389                                 &srtcm, modify_bits, 0, 0);
1390         else
1391                 ret = mlx5_flow_meter_action_modify(priv, fm,
1392                                                    &fm->profile->srtcm_prm,
1393                                                     modify_bits, 0, 1);
1394         if (ret)
1395                 return -rte_mtr_error_set(error, -ret,
1396                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1397                                           NULL,
1398                                           new_state ?
1399                                           "Failed to enable meter." :
1400                                           "Failed to disable meter.");
1401         return 0;
1402 }
1403
1404 /**
1405  * Callback to enable flow meter.
1406  *
1407  * @param[in] dev
1408  *   Pointer to Ethernet device.
1409  * @param[in] meter_id
1410  *   Meter id.
1411  * @param[out] error
1412  *   Pointer to rte meter error structure.
1413  *
1414  * @return
1415  *   0 on success, a negative errno value otherwise and rte_errno is set.
1416  */
1417 static int
1418 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1419                        uint32_t meter_id,
1420                        struct rte_mtr_error *error)
1421 {
1422         struct mlx5_priv *priv = dev->data->dev_private;
1423         struct mlx5_flow_meter_info *fm;
1424         int ret;
1425
1426         if (!priv->mtr_en)
1427                 return -rte_mtr_error_set(error, ENOTSUP,
1428                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1429                                           "Meter is not supported");
1430         /* Meter object must exist. */
1431         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1432         if (fm == NULL)
1433                 return -rte_mtr_error_set(error, ENOENT,
1434                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1435                                           NULL, "Meter not found.");
1436         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1437                 return 0;
1438         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1439                                            error);
1440         if (!ret)
1441                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1442         return ret;
1443 }
1444
1445 /**
1446  * Callback to disable flow meter.
1447  *
1448  * @param[in] dev
1449  *   Pointer to Ethernet device.
1450  * @param[in] meter_id
1451  *   Meter id.
1452  * @param[out] error
1453  *   Pointer to rte meter error structure.
1454  *
1455  * @return
1456  *   0 on success, a negative errno value otherwise and rte_errno is set.
1457  */
1458 static int
1459 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1460                         uint32_t meter_id,
1461                         struct rte_mtr_error *error)
1462 {
1463         struct mlx5_priv *priv = dev->data->dev_private;
1464         struct mlx5_flow_meter_info *fm;
1465         int ret;
1466
1467         if (!priv->mtr_en)
1468                 return -rte_mtr_error_set(error, ENOTSUP,
1469                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1470                                           "Meter is not supported");
1471         /* Meter object must exist. */
1472         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1473         if (fm == NULL)
1474                 return -rte_mtr_error_set(error, ENOENT,
1475                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1476                                           NULL, "Meter not found.");
1477         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1478                 return 0;
1479         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1480                                            error);
1481         if (!ret)
1482                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1483         return ret;
1484 }
1485
1486 /**
1487  * Callback to update meter profile.
1488  *
1489  * @param[in] dev
1490  *   Pointer to Ethernet device.
1491  * @param[in] meter_id
1492  *   Meter id.
1493  * @param[in] meter_profile_id
1494  *   To be updated meter profile id.
1495  * @param[out] error
1496  *   Pointer to rte meter error structure.
1497  *
1498  * @return
1499  *   0 on success, a negative errno value otherwise and rte_errno is set.
1500  */
1501 static int
1502 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1503                                uint32_t meter_id,
1504                                uint32_t meter_profile_id,
1505                                struct rte_mtr_error *error)
1506 {
1507         struct mlx5_priv *priv = dev->data->dev_private;
1508         struct mlx5_flow_meter_profile *fmp;
1509         struct mlx5_flow_meter_profile *old_fmp;
1510         struct mlx5_flow_meter_info *fm;
1511         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1512                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1513         int ret;
1514
1515         if (!priv->mtr_en)
1516                 return -rte_mtr_error_set(error, ENOTSUP,
1517                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1518                                           "Meter is not supported");
1519         /* Meter profile must exist. */
1520         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1521         if (fmp == NULL)
1522                 return -rte_mtr_error_set(error, ENOENT,
1523                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1524                                           NULL, "Meter profile not found.");
1525         /* Meter object must exist. */
1526         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1527         if (fm == NULL)
1528                 return -rte_mtr_error_set(error, ENOENT,
1529                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1530                                           NULL, "Meter not found.");
1531         /* MTR object already set to meter profile id. */
1532         old_fmp = fm->profile;
1533         if (fmp == old_fmp)
1534                 return 0;
1535         /* Update the profile. */
1536         fm->profile = fmp;
1537         /* Update meter params in HW (if not disabled). */
1538         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1539                 return 0;
1540         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1541                                               modify_bits, fm->active_state, 1);
1542         if (ret) {
1543                 fm->profile = old_fmp;
1544                 return -rte_mtr_error_set(error, -ret,
1545                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1546                                           NULL, "Failed to update meter"
1547                                           " parmeters in hardware.");
1548         }
1549         old_fmp->ref_cnt--;
1550         fmp->ref_cnt++;
1551         return 0;
1552 }
1553
1554 /**
1555  * Callback to update meter stats mask.
1556  *
1557  * @param[in] dev
1558  *   Pointer to Ethernet device.
1559  * @param[in] meter_id
1560  *   Meter id.
1561  * @param[in] stats_mask
1562  *   To be updated stats_mask.
1563  * @param[out] error
1564  *   Pointer to rte meter error structure.
1565  *
1566  * @return
1567  *   0 on success, a negative errno value otherwise and rte_errno is set.
1568  */
1569 static int
1570 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1571                              uint32_t meter_id,
1572                              uint64_t stats_mask,
1573                              struct rte_mtr_error *error)
1574 {
1575         struct mlx5_priv *priv = dev->data->dev_private;
1576         struct mlx5_flow_meter_info *fm;
1577
1578         if (!priv->mtr_en)
1579                 return -rte_mtr_error_set(error, ENOTSUP,
1580                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1581                                           "Meter is not supported");
1582         /* Meter object must exist. */
1583         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1584         if (fm == NULL)
1585                 return -rte_mtr_error_set(error, ENOENT,
1586                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1587                                           NULL, "Meter object id not valid.");
1588         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1589                 return -rte_mtr_error_set(error, ENOENT,
1590                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1591                                           NULL, "Fail to allocate "
1592                                           "counter for meter.");
1593         return 0;
1594 }
1595
1596 /**
1597  * Callback to read meter statistics.
1598  *
1599  * @param[in] dev
1600  *   Pointer to Ethernet device.
1601  * @param[in] meter_id
1602  *   Meter id.
1603  * @param[out] stats
1604  *   Pointer to store the statistics.
1605  * @param[out] stats_mask
1606  *   Pointer to store the stats_mask.
1607  * @param[in] clear
1608  *   Statistic to be cleared after read or not.
1609  * @param[out] error
1610  *   Pointer to rte meter error structure.
1611  *
1612  * @return
1613  *   0 on success, a negative errno value otherwise and rte_errno is set.
1614  */
1615 static int
1616 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1617                            uint32_t meter_id,
1618                            struct rte_mtr_stats *stats,
1619                            uint64_t *stats_mask,
1620                            int clear,
1621                            struct rte_mtr_error *error)
1622 {
1623         struct mlx5_priv *priv = dev->data->dev_private;
1624         struct mlx5_flow_meter_info *fm;
1625         uint64_t pkts;
1626         uint64_t bytes;
1627         int ret = 0;
1628
1629         if (!priv->mtr_en)
1630                 return -rte_mtr_error_set(error, ENOTSUP,
1631                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1632                                           "Meter is not supported");
1633         /* Meter object must exist. */
1634         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1635         if (fm == NULL)
1636                 return -rte_mtr_error_set(error, ENOENT,
1637                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1638                                           NULL, "Meter object id not valid.");
1639         *stats_mask = 0;
1640         if (fm->bytes_dropped)
1641                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1642         if (fm->pkts_dropped)
1643                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1644         memset(stats, 0, sizeof(*stats));
1645         if (fm->drop_cnt) {
1646                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1647                                                  &bytes);
1648                 if (ret)
1649                         goto error;
1650                 /* If need to read the packets, set it. */
1651                 if (fm->pkts_dropped)
1652                         stats->n_pkts_dropped = pkts;
1653                 /* If need to read the bytes, set it. */
1654                 if (fm->bytes_dropped)
1655                         stats->n_bytes_dropped = bytes;
1656         }
1657         return 0;
1658 error:
1659         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1660                                  "Failed to read meter drop counters.");
1661 }
1662
1663 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1664         .capabilities_get = mlx5_flow_mtr_cap_get,
1665         .meter_profile_add = mlx5_flow_meter_profile_add,
1666         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1667         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1668         .meter_policy_add = mlx5_flow_meter_policy_add,
1669         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1670         .create = mlx5_flow_meter_create,
1671         .destroy = mlx5_flow_meter_destroy,
1672         .meter_enable = mlx5_flow_meter_enable,
1673         .meter_disable = mlx5_flow_meter_disable,
1674         .meter_profile_update = mlx5_flow_meter_profile_update,
1675         .meter_dscp_table_update = NULL,
1676         .stats_update = mlx5_flow_meter_stats_update,
1677         .stats_read = mlx5_flow_meter_stats_read,
1678 };
1679
1680 /**
1681  * Get meter operations.
1682  *
1683  * @param dev
1684  *   Pointer to Ethernet device structure.
1685  * @param arg
1686  *   Pointer to set the mtr operations.
1687  *
1688  * @return
1689  *   Always 0.
1690  */
1691 int
1692 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1693 {
1694         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1695         return 0;
1696 }
1697
1698 /**
1699  * Find meter by id.
1700  *
1701  * @param priv
1702  *   Pointer to mlx5_priv.
1703  * @param meter_id
1704  *   Meter id.
1705  * @param mtr_idx
1706  *   Pointer to Meter index.
1707  *
1708  * @return
1709  *   Pointer to the meter info found on success, NULL otherwise.
1710  */
1711 struct mlx5_flow_meter_info *
1712 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1713                 uint32_t *mtr_idx)
1714 {
1715         struct mlx5_legacy_flow_meter *legacy_fm;
1716         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1717         struct mlx5_aso_mtr *aso_mtr;
1718         struct mlx5_aso_mtr_pools_mng *pools_mng =
1719                                 &priv->sh->mtrmng->pools_mng;
1720         union mlx5_l3t_data data;
1721
1722         if (priv->sh->meter_aso_en) {
1723                 rte_spinlock_lock(&pools_mng->mtrsl);
1724                 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1725                         rte_spinlock_unlock(&pools_mng->mtrsl);
1726                         return NULL;
1727                 }
1728                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1729                         !data.dword) {
1730                         rte_spinlock_unlock(&pools_mng->mtrsl);
1731                         return NULL;
1732                 }
1733                 if (mtr_idx)
1734                         *mtr_idx = data.dword;
1735                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1736                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1737                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1738                 rte_spinlock_unlock(&pools_mng->mtrsl);
1739                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1740                         return NULL;
1741                 return &aso_mtr->fm;
1742         }
1743         TAILQ_FOREACH(legacy_fm, fms, next)
1744                 if (meter_id == legacy_fm->fm.meter_id) {
1745                         if (mtr_idx)
1746                                 *mtr_idx = legacy_fm->idx;
1747                         return &legacy_fm->fm;
1748                 }
1749         return NULL;
1750 }
1751
1752 /**
1753  * Find meter by index.
1754  *
1755  * @param priv
1756  *   Pointer to mlx5_priv.
1757  * @param idx
1758  *   Meter index.
1759  *
1760  * @return
1761  *   Pointer to the meter info found on success, NULL otherwise.
1762  */
1763 struct mlx5_flow_meter_info *
1764 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1765 {
1766         struct mlx5_aso_mtr *aso_mtr;
1767
1768         if (priv->sh->meter_aso_en) {
1769                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1770                 if (!aso_mtr)
1771                         return NULL;
1772                 return &aso_mtr->fm;
1773         } else {
1774                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1775         }
1776 }
1777
1778 /**
1779  * Attach meter to flow.
1780  * Unidirectional Meter creation can only be done
1781  * when flow direction is known, i.e. when calling meter_attach.
1782  *
1783  * @param [in] priv
1784  *  Pointer to mlx5 private data.
1785  * @param[in] fm
1786  *   Pointer to flow meter.
1787  * @param [in] attr
1788  *  Pointer to flow attributes.
1789  * @param [out] error
1790  *  Pointer to error structure.
1791  *
1792  * @return
1793  *   0 on success, a negative errno value otherwise and rte_errno is set.
1794  */
1795 int
1796 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1797                        struct mlx5_flow_meter_info *fm,
1798                        const struct rte_flow_attr *attr,
1799                        struct rte_flow_error *error)
1800 {
1801         int ret = 0;
1802
1803         if (priv->sh->meter_aso_en) {
1804                 struct mlx5_aso_mtr *aso_mtr;
1805
1806                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1807                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1808                         return rte_flow_error_set(error, ENOENT,
1809                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1810                                         NULL,
1811                                         "Timeout in meter configuration");
1812                 }
1813                 rte_spinlock_lock(&fm->sl);
1814                 if (fm->shared || !fm->ref_cnt) {
1815                         fm->ref_cnt++;
1816                 } else {
1817                         rte_flow_error_set(error, EINVAL,
1818                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1819                                    "Meter cannot be shared");
1820                         ret = -1;
1821                 }
1822                 rte_spinlock_unlock(&fm->sl);
1823         } else {
1824                 rte_spinlock_lock(&fm->sl);
1825                 if (fm->meter_action) {
1826                         if (fm->shared &&
1827                             attr->transfer == fm->transfer &&
1828                             attr->ingress == fm->ingress &&
1829                             attr->egress == fm->egress) {
1830                                 fm->ref_cnt++;
1831                         } else {
1832                                 rte_flow_error_set(error, EINVAL,
1833                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1834                                         fm->shared ?
1835                                         "Meter attr not match." :
1836                                         "Meter cannot be shared.");
1837                                 ret = -1;
1838                         }
1839                 } else {
1840                         fm->ingress = attr->ingress;
1841                         fm->egress = attr->egress;
1842                         fm->transfer = attr->transfer;
1843                         fm->ref_cnt = 1;
1844                         /* This also creates the meter object. */
1845                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1846                                                                          fm);
1847                         if (!fm->meter_action) {
1848                                 fm->ref_cnt = 0;
1849                                 fm->ingress = 0;
1850                                 fm->egress = 0;
1851                                 fm->transfer = 0;
1852                                 rte_flow_error_set(error, EINVAL,
1853                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1854                                         "Meter action create failed.");
1855                                 ret = -1;
1856                         }
1857                 }
1858                 rte_spinlock_unlock(&fm->sl);
1859         }
1860         return ret ? -rte_errno : 0;
1861 }
1862
1863 /**
1864  * Detach meter from flow.
1865  *
1866  * @param [in] priv
1867  *  Pointer to mlx5 private data.
1868  * @param [in] fm
1869  *  Pointer to flow meter.
1870  */
1871 void
1872 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1873                        struct mlx5_flow_meter_info *fm)
1874 {
1875 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1876         rte_spinlock_lock(&fm->sl);
1877         MLX5_ASSERT(fm->ref_cnt);
1878         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1879                 mlx5_glue->destroy_flow_action(fm->meter_action);
1880                 fm->meter_action = NULL;
1881                 fm->ingress = 0;
1882                 fm->egress = 0;
1883                 fm->transfer = 0;
1884         }
1885         rte_spinlock_unlock(&fm->sl);
1886 #else
1887         (void)priv;
1888         (void)fm;
1889 #endif
1890 }
1891
1892 /**
1893  * Flush meter with Rx queue configuration.
1894  *
1895  * @param[in] dev
1896  *   Pointer to Ethernet device.
1897  */
1898 void
1899 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1900 {
1901         struct mlx5_priv *priv = dev->data->dev_private;
1902         struct mlx5_flow_meter_sub_policy *sub_policy;
1903         struct mlx5_flow_meter_policy *mtr_policy;
1904         void *entry;
1905         uint32_t i, policy_idx;
1906
1907         if (!priv->mtr_en)
1908                 return;
1909         if (priv->policy_idx_tbl) {
1910                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1911                         policy_idx = *(uint32_t *)entry;
1912                         sub_policy = mlx5_ipool_get
1913                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1914                                 policy_idx);
1915                         if (!sub_policy || !sub_policy->main_policy)
1916                                 continue;
1917                         mtr_policy = sub_policy->main_policy;
1918                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1919                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1920                                         mtr_policy);
1921                 }
1922         }
1923 }
1924
1925 /**
1926  * Iterate a meter hierarchy and flush all meters and policies if possible.
1927  *
1928  * @param[in] dev
1929  *   Pointer to Ethernet device.
1930  * @param[in] fm
1931  *   Pointer to flow meter.
1932  * @param[in] mtr_idx
1933  *   .Meter's index
1934  * @param[out] error
1935  *   Pointer to rte meter error structure.
1936  *
1937  * @return
1938  *   0 on success, a negative errno value otherwise and rte_errno is set.
1939  */
1940 static int
1941 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
1942                                 struct mlx5_flow_meter_info *fm,
1943                                 uint32_t mtr_idx,
1944                                 struct rte_mtr_error *error)
1945 {
1946         struct mlx5_priv *priv = dev->data->dev_private;
1947         struct mlx5_flow_meter_policy *policy;
1948         uint32_t policy_id;
1949         struct mlx5_flow_meter_info *next_fm;
1950         uint32_t next_mtr_idx;
1951         struct mlx5_flow_meter_policy *next_policy = NULL;
1952
1953         policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
1954         MLX5_ASSERT(policy);
1955         while (!fm->ref_cnt && policy->is_hierarchy) {
1956                 policy_id = fm->policy_id;
1957                 next_fm = mlx5_flow_meter_find(priv,
1958                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
1959                                 &next_mtr_idx);
1960                 if (next_fm) {
1961                         next_policy = mlx5_flow_meter_policy_find(dev,
1962                                                         next_fm->policy_id,
1963                                                         NULL);
1964                         MLX5_ASSERT(next_policy);
1965                 }
1966                 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1967                         return -rte_mtr_error_set(error, ENOTSUP,
1968                                                 RTE_MTR_ERROR_TYPE_MTR_ID,
1969                                                 NULL,
1970                                                 "Failed to flush meter.");
1971                 if (policy->ref_cnt)
1972                         break;
1973                 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
1974                                                 policy, error, true))
1975                         return -rte_errno;
1976                 mlx5_free(policy);
1977                 if (!next_fm || !next_policy)
1978                         break;
1979                 fm = next_fm;
1980                 mtr_idx = next_mtr_idx;
1981                 policy = next_policy;
1982         }
1983         return 0;
1984 }
1985
1986 /**
1987  * Flush all the hierarchy meters and their policies.
1988  *
1989  * @param[in] dev
1990  *   Pointer to Ethernet device.
1991  * @param[out] error
1992  *   Pointer to rte meter error structure.
1993  *
1994  * @return
1995  *   0 on success, a negative errno value otherwise and rte_errno is set.
1996  */
1997 static int
1998 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
1999                                       struct rte_mtr_error *error)
2000 {
2001         struct mlx5_priv *priv = dev->data->dev_private;
2002         struct mlx5_flow_meter_info *fm;
2003         struct mlx5_flow_meter_policy *policy;
2004         struct mlx5_flow_meter_sub_policy *sub_policy;
2005         struct mlx5_flow_meter_info *next_fm;
2006         struct mlx5_aso_mtr *aso_mtr;
2007         uint32_t mtr_idx = 0;
2008         uint32_t i, policy_idx;
2009         void *entry;
2010
2011         if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2012                 return 0;
2013         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2014                 mtr_idx = *(uint32_t *)entry;
2015                 if (!mtr_idx)
2016                         continue;
2017                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2018                 fm = &aso_mtr->fm;
2019                 if (fm->ref_cnt || fm->def_policy)
2020                         continue;
2021                 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2022                         return -rte_errno;
2023         }
2024         MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2025                 policy_idx = *(uint32_t *)entry;
2026                 sub_policy = mlx5_ipool_get
2027                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2028                                 policy_idx);
2029                 if (!sub_policy)
2030                         return -rte_mtr_error_set(error,
2031                                         EINVAL,
2032                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2033                                         NULL, "Meter policy invalid.");
2034                 policy = sub_policy->main_policy;
2035                 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2036                         continue;
2037                 next_fm = mlx5_flow_meter_find(priv,
2038                                 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2039                                 &mtr_idx);
2040                 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2041                                                     error, true))
2042                         return -rte_mtr_error_set(error,
2043                                         EINVAL,
2044                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2045                                         NULL, "Meter policy invalid.");
2046                 mlx5_free(policy);
2047                 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2048                         continue;
2049                 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2050                                                     mtr_idx, error))
2051                         return -rte_errno;
2052         }
2053         return 0;
2054 }
2055 /**
2056  * Flush meter configuration.
2057  *
2058  * @param[in] dev
2059  *   Pointer to Ethernet device.
2060  * @param[out] error
2061  *   Pointer to rte meter error structure.
2062  *
2063  * @return
2064  *   0 on success, a negative errno value otherwise and rte_errno is set.
2065  */
2066 int
2067 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2068 {
2069         struct mlx5_priv *priv = dev->data->dev_private;
2070         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2071         struct mlx5_flow_meter_profile *fmp;
2072         struct mlx5_legacy_flow_meter *legacy_fm;
2073         struct mlx5_flow_meter_info *fm;
2074         struct mlx5_flow_meter_sub_policy *sub_policy;
2075         void *tmp;
2076         uint32_t i, mtr_idx, policy_idx;
2077         void *entry;
2078         struct mlx5_aso_mtr *aso_mtr;
2079
2080         if (!priv->mtr_en)
2081                 return 0;
2082         if (priv->sh->meter_aso_en) {
2083                 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2084                         return -rte_errno;
2085                 if (priv->mtr_idx_tbl) {
2086                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2087                                 mtr_idx = *(uint32_t *)entry;
2088                                 if (mtr_idx) {
2089                                         aso_mtr =
2090                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
2091                                         fm = &aso_mtr->fm;
2092                                         (void)mlx5_flow_meter_params_flush(dev,
2093                                                 fm, mtr_idx);
2094                                 }
2095                         }
2096                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
2097                         priv->mtr_idx_tbl = NULL;
2098                 }
2099         } else {
2100                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2101                         fm = &legacy_fm->fm;
2102                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
2103                                 return -rte_mtr_error_set(error, EINVAL,
2104                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2105                                 NULL, "MTR object meter profile invalid.");
2106                 }
2107         }
2108         if (priv->policy_idx_tbl) {
2109                 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2110                         policy_idx = *(uint32_t *)entry;
2111                         sub_policy = mlx5_ipool_get
2112                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2113                                 policy_idx);
2114                         if (!sub_policy)
2115                                 return -rte_mtr_error_set(error,
2116                                                 EINVAL,
2117                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2118                                                 NULL, "MTR object "
2119                                                 "meter policy invalid.");
2120                         if (__mlx5_flow_meter_policy_delete(dev, i,
2121                                                 sub_policy->main_policy,
2122                                                 error, true))
2123                                 return -rte_mtr_error_set(error,
2124                                                 EINVAL,
2125                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2126                                                 NULL, "MTR object "
2127                                                 "meter policy invalid.");
2128                         mlx5_free(sub_policy->main_policy);
2129                 }
2130                 mlx5_l3t_destroy(priv->policy_idx_tbl);
2131                 priv->policy_idx_tbl = NULL;
2132         }
2133         if (priv->mtr_profile_tbl) {
2134                 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2135                         fmp = entry;
2136                         if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2137                                                            error))
2138                                 return -rte_mtr_error_set(error, EINVAL,
2139                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2140                                                 NULL, "Fail to destroy "
2141                                                 "meter profile.");
2142                 }
2143                 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2144                 priv->mtr_profile_tbl = NULL;
2145         }
2146         /* Delete default policy table. */
2147         mlx5_flow_destroy_def_policy(dev);
2148         if (priv->sh->refcnt == 1)
2149                 mlx5_flow_destroy_mtr_drop_tbls(dev);
2150         return 0;
2151 }