drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
  95         struct mlx5_flow_meter_profile *fmp;
  96
  97         TAILQ_FOREACH(fmp, fmps, next)
  98                 if (meter_profile_id == fmp->id)
  99                         return fmp;
 100         return NULL;
 101 }
 102
 103 /**
 104  * Validate the MTR profile.
 105  *
 106  * @param[in] dev
 107  *   Pointer to Ethernet device.
 108  * @param[in] meter_profile_id
 109  *   Meter profile id.
 110  * @param[in] profile
 111  *   Pointer to meter profile detail.
 112  * @param[out] error
 113  *   Pointer to the error structure.
 114  *
 115  * @return
 116  *   0 on success, a negative errno value otherwise and rte_errno is set.
 117  */
 118 static int
 119 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 120                                  uint32_t meter_profile_id,
 121                                  struct rte_mtr_meter_profile *profile,
 122                                  struct rte_mtr_error *error)
 123 {
 124         struct mlx5_priv *priv = dev->data->dev_private;
 125         struct mlx5_flow_meter_profile *fmp;
 126
 127         /* Profile must not be NULL. */
 128         if (profile == NULL)
 129                 return -rte_mtr_error_set(error, EINVAL,
 130                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 131                                           NULL, "Meter profile is null.");
 132         /* Meter profile ID must be valid. */
 133         if (meter_profile_id == UINT32_MAX)
 134                 return -rte_mtr_error_set(error, EINVAL,
 135                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 136                                           NULL, "Meter profile id not valid.");
 137         /* Meter profile must not exist. */
 138         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 139         if (fmp)
 140                 return -rte_mtr_error_set(error, EEXIST,
 141                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 142                                           NULL,
 143                                           "Meter profile already exists.");
 144         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 145                 if (priv->config.hca_attr.qos.flow_meter_old) {
 146                         /* Verify support for flow meter parameters. */
 147                         if (priv->sh->meter_aso_en && profile->packet_mode) {
 148                                 if (profile->srtcm_rfc2697.cir > 0 &&
 149                                         (profile->srtcm_rfc2697.cir <<
 150                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 151                                         <= MLX5_SRTCM_CIR_MAX &&
 152                                         profile->srtcm_rfc2697.cbs > 0 &&
 153                                         (profile->srtcm_rfc2697.cbs <<
 154                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 155                                         <= MLX5_SRTCM_CBS_MAX &&
 156                                         (profile->srtcm_rfc2697.ebs <<
 157                                         MLX5_MTRS_PPS_MAP_BPS_SHIFT)
 158                                         <= MLX5_SRTCM_EBS_MAX)
 159                                         return 0;
 160                                 return -rte_mtr_error_set
 161                                              (error, ENOTSUP,
 162                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 163                                               NULL,
 164                                               profile->srtcm_rfc2697.ebs ?
 165                                               "Metering value ebs must be 0." :
 166                                               "Invalid metering parameters.");
 167                         }
 168                         if (profile->srtcm_rfc2697.cir > 0 &&
 169                                 profile->srtcm_rfc2697.cir <=
 170                                                 MLX5_SRTCM_CIR_MAX &&
 171                                 profile->srtcm_rfc2697.cbs > 0 &&
 172                                 profile->srtcm_rfc2697.cbs <=
 173                                                 MLX5_SRTCM_CBS_MAX &&
 174                                 profile->srtcm_rfc2697.ebs <=
 175                                                 MLX5_SRTCM_EBS_MAX)
 176                                 return 0;
 177                         return -rte_mtr_error_set(error, ENOTSUP,
 178                                         RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 179                                         NULL,
 180                                         profile->srtcm_rfc2697.ebs ?
 181                                         "Metering value ebs must be 0." :
 182                                         "Invalid metering parameters.");
 183                 }
 184         }
 185         return -rte_mtr_error_set(error, ENOTSUP,
 186                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 187                                   NULL, "Metering algorithm not supported.");
 188 }
 189
 190 /**
 191  * Calculate mantissa and exponent for cir.
 192  *
 193  * @param[in] cir
 194  *   Value to be calculated.
 195  * @param[out] man
 196  *   Pointer to the mantissa.
 197  * @param[out] exp
 198  *   Pointer to the exp.
 199  */
 200 static void
 201 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 202 {
 203         int64_t _cir;
 204         int64_t delta = INT64_MAX;
 205         uint8_t _man = 0;
 206         uint8_t _exp = 0;
 207         uint64_t m, e;
 208
 209         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 210                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 211                         _cir = (1000000000ULL * m) >> e;
 212                         if (llabs(cir - _cir) <= delta) {
 213                                 delta = llabs(cir - _cir);
 214                                 _man = m;
 215                                 _exp = e;
 216                         }
 217                 }
 218         }
 219         *man = _man;
 220         *exp = _exp;
 221 }
 222
 223 /**
 224  * Calculate mantissa and exponent for xbs.
 225  *
 226  * @param[in] xbs
 227  *   Value to be calculated.
 228  * @param[out] man
 229  *   Pointer to the mantissa.
 230  * @param[out] exp
 231  *   Pointer to the exp.
 232  */
 233 static void
 234 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 235 {
 236         int _exp;
 237         double _man;
 238
 239         /* Special case xbs == 0 ? both exp and matissa are 0. */
 240         if (xbs == 0) {
 241                 *man = 0;
 242                 *exp = 0;
 243                 return;
 244         }
 245         /* xbs = xbs_mantissa * 2^xbs_exponent */
 246         _man = frexp(xbs, &_exp);
 247         _man = _man * pow(2, MLX5_MAN_WIDTH);
 248         _exp = _exp - MLX5_MAN_WIDTH;
 249         *man = (uint8_t)ceil(_man);
 250         *exp = _exp;
 251 }
 252
 253 /**
 254  * Fill the prm meter parameter.
 255  *
 256  * @param[in,out] fmp
 257  *   Pointer to meter profie to be converted.
 258  * @param[out] error
 259  *   Pointer to the error structure.
 260  *
 261  * @return
 262  *   0 on success, a negative errno value otherwise and rte_errno is set.
 263  */
 264 static int
 265 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 266                         struct mlx5_priv *priv, struct rte_mtr_error *error)
 267 {
 268         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 269         uint8_t man, exp;
 270         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 271         uint32_t ebs_exp, ebs_man;
 272         uint64_t cir, cbs, ebs;
 273
 274         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 275                 return -rte_mtr_error_set(error, ENOTSUP,
 276                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 277                                 NULL, "Metering algorithm not supported.");
 278         if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
 279                 return -rte_mtr_error_set(error, ENOTSUP,
 280                         RTE_MTR_ERROR_TYPE_METER_PROFILE,
 281                         NULL, "Metering algorithm packet mode not supported.");
 282         if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
 283                 cir = fmp->profile.srtcm_rfc2697.cir <<
 284                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 285                 cbs = fmp->profile.srtcm_rfc2697.cbs <<
 286                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 287                 ebs = fmp->profile.srtcm_rfc2697.ebs <<
 288                                 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
 289         } else {
 290                 cir = fmp->profile.srtcm_rfc2697.cir;
 291                 cbs = fmp->profile.srtcm_rfc2697.cbs;
 292                 ebs = fmp->profile.srtcm_rfc2697.ebs;
 293         }
 294         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 295         mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
 296         /* Check if cir mantissa is too large. */
 297         if (exp > ASO_DSEG_CIR_EXP_MASK)
 298                 return -rte_mtr_error_set(error, ENOTSUP,
 299                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 300                                           "meter profile parameter cir is"
 301                                           " not supported.");
 302         cir_man = man;
 303         cir_exp = exp;
 304          /* cbs = cbs_mantissa * 2^cbs_exponent */
 305         mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
 306         /* Check if cbs mantissa is too large. */
 307         if (exp > ASO_DSEG_EXP_MASK)
 308                 return -rte_mtr_error_set(error, ENOTSUP,
 309                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 310                                           "meter profile parameter cbs is"
 311                                           " not supported.");
 312         cbs_man = man;
 313         cbs_exp = exp;
 314         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 315                                 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 316                                 cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 317                                 cir_man);
 318         mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
 319         /* Check if ebs mantissa is too large. */
 320         if (exp > ASO_DSEG_EXP_MASK)
 321                 return -rte_mtr_error_set(error, ENOTSUP,
 322                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 323                                           "meter profile parameter ebs is"
 324                                           " not supported.");
 325         ebs_man = man;
 326         ebs_exp = exp;
 327         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 328                                         ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 329         return 0;
 330 }
 331
 332 /**
 333  * Callback to get MTR capabilities.
 334  *
 335  * @param[in] dev
 336  *   Pointer to Ethernet device.
 337  * @param[out] cap
 338  *   Pointer to save MTR capabilities.
 339  * @param[out] error
 340  *   Pointer to the error structure.
 341  *
 342  * @return
 343  *   0 on success, a negative errno value otherwise and rte_errno is set.
 344  */
 345 static int
 346 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 347                  struct rte_mtr_capabilities *cap,
 348                  struct rte_mtr_error *error __rte_unused)
 349 {
 350         struct mlx5_priv *priv = dev->data->dev_private;
 351         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 352
 353         if (!priv->mtr_en)
 354                 return -rte_mtr_error_set(error, ENOTSUP,
 355                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 356                                           "Meter is not supported");
 357         memset(cap, 0, sizeof(*cap));
 358         if (priv->sh->meter_aso_en) {
 359                 /* 2 meters per one ASO cache line. */
 360                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 361                 cap->srtcm_rfc2697_packet_mode_supported = 1;
 362         } else {
 363                 cap->n_max = 1 << qattr->log_max_flow_meter;
 364                 cap->srtcm_rfc2697_packet_mode_supported = 0;
 365         }
 366         cap->srtcm_rfc2697_byte_mode_supported = 1;
 367         cap->n_shared_max = cap->n_max;
 368         cap->identical = 1;
 369         cap->shared_identical = 1;
 370         cap->shared_n_flows_per_mtr_max = 4 << 20;
 371         /* 2M flows can share the same meter. */
 372         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 373         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 374         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 375         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 376                           RTE_MTR_STATS_N_PKTS_DROPPED;
 377         return 0;
 378 }
 379
 380 /**
 381  * Callback to add MTR profile.
 382  *
 383  * @param[in] dev
 384  *   Pointer to Ethernet device.
 385  * @param[in] meter_profile_id
 386  *   Meter profile id.
 387  * @param[in] profile
 388  *   Pointer to meter profile detail.
 389  * @param[out] error
 390  *   Pointer to the error structure.
 391  *
 392  * @return
 393  *   0 on success, a negative errno value otherwise and rte_errno is set.
 394  */
 395 static int
 396 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 397                        uint32_t meter_profile_id,
 398                        struct rte_mtr_meter_profile *profile,
 399                        struct rte_mtr_error *error)
 400 {
 401         struct mlx5_priv *priv = dev->data->dev_private;
 402         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
 403         struct mlx5_flow_meter_profile *fmp;
 404         int ret;
 405
 406         if (!priv->mtr_en)
 407                 return -rte_mtr_error_set(error, ENOTSUP,
 408                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 409                                           "Meter is not supported");
 410         /* Check input params. */
 411         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 412                                                profile, error);
 413         if (ret)
 414                 return ret;
 415         /* Meter profile memory allocation. */
 416         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 417                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 418         if (fmp == NULL)
 419                 return -rte_mtr_error_set(error, ENOMEM,
 420                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 421                                           NULL, "Meter profile memory "
 422                                           "alloc failed.");
 423         /* Fill profile info. */
 424         fmp->id = meter_profile_id;
 425         fmp->profile = *profile;
 426         /* Fill the flow meter parameters for the PRM. */
 427         ret = mlx5_flow_meter_param_fill(fmp, priv, error);
 428         if (ret)
 429                 goto error;
 430         /* Add to list. */
 431         TAILQ_INSERT_TAIL(fmps, fmp, next);
 432         return 0;
 433 error:
 434         mlx5_free(fmp);
 435         return ret;
 436 }
 437
 438 /**
 439  * Callback to delete MTR profile.
 440  *
 441  * @param[in] dev
 442  *   Pointer to Ethernet device.
 443  * @param[in] meter_profile_id
 444  *   Meter profile id.
 445  * @param[out] error
 446  *   Pointer to the error structure.
 447  *
 448  * @return
 449  *   0 on success, a negative errno value otherwise and rte_errno is set.
 450  */
 451 static int
 452 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 453                           uint32_t meter_profile_id,
 454                           struct rte_mtr_error *error)
 455 {
 456         struct mlx5_priv *priv = dev->data->dev_private;
 457         struct mlx5_flow_meter_profile *fmp;
 458
 459         if (!priv->mtr_en)
 460                 return -rte_mtr_error_set(error, ENOTSUP,
 461                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 462                                           "Meter is not supported");
 463         /* Meter profile must exist. */
 464         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 465         if (fmp == NULL)
 466                 return -rte_mtr_error_set(error, ENOENT,
 467                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 468                                           &meter_profile_id,
 469                                           "Meter profile id is invalid.");
 470         /* Check profile is unused. */
 471         if (fmp->ref_cnt)
 472                 return -rte_mtr_error_set(error, EBUSY,
 473                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 474                                           NULL, "Meter profile is in use.");
 475         /* Remove from list. */
 476         TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
 477         mlx5_free(fmp);
 478         return 0;
 479 }
 480
 481 /**
 482  * Find policy by id.
 483  *
 484  * @param[in] dev
 485  *   Pointer to Ethernet device.
 486  * @param policy_id
 487  *   Policy id.
 488  *
 489  * @return
 490  *   Pointer to the policy found on success, NULL otherwise.
 491  */
 492 struct mlx5_flow_meter_policy *
 493 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 494                             uint32_t policy_id,
 495                             uint32_t *policy_idx)
 496 {
 497         struct mlx5_priv *priv = dev->data->dev_private;
 498         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 499         union mlx5_l3t_data data;
 500
 501         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM ||
 502                 !priv->sh->mtrmng->policy_idx_tbl)
 503                 return NULL;
 504         if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl,
 505                                 policy_id, &data) ||
 506                                 !data.dword)
 507                 return NULL;
 508         if (policy_idx)
 509                 *policy_idx = data.dword;
 510         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 511                                         data.dword);
 512         /* Remove reference taken by the mlx5_l3t_get_entry. */
 513         mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 514                                 policy_id);
 515         if (sub_policy)
 516                 if (sub_policy->main_policy_id)
 517                         return sub_policy->main_policy;
 518         return NULL;
 519 }
 520
 521 /**
 522  * Callback to check MTR policy action validate
 523  *
 524  * @param[in] dev
 525  *   Pointer to Ethernet device.
 526  * @param[in] actions
 527  *   Pointer to meter policy action detail.
 528  * @param[out] error
 529  *   Pointer to the error structure.
 530  *
 531  * @return
 532  *   0 on success, a negative errno value otherwise and rte_errno is set.
 533  */
 534 static int
 535 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 536         struct rte_mtr_meter_policy_params *policy,
 537         struct rte_mtr_error *error)
 538 {
 539         struct mlx5_priv *priv = dev->data->dev_private;
 540         struct rte_flow_attr attr = { .transfer =
 541                         priv->config.dv_esw_en ? 1 : 0};
 542         bool is_rss = false;
 543         bool is_def_policy = false;
 544         uint8_t domain_bitmap;
 545         int ret;
 546
 547         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 548                 return -rte_mtr_error_set(error, ENOTSUP,
 549                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 550                                 NULL, "meter policy unsupported.");
 551         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 552                         &is_rss, &domain_bitmap, &is_def_policy, error);
 553         if (ret)
 554                 return ret;
 555         return 0;
 556 }
 557
 558 static int
 559 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 560                         uint32_t policy_id,
 561                         struct mlx5_flow_meter_policy *mtr_policy,
 562                         struct rte_mtr_error *error,
 563                         bool clear_l3t)
 564 {
 565         struct mlx5_priv *priv = dev->data->dev_private;
 566         struct mlx5_flow_meter_sub_policy *sub_policy;
 567         uint32_t i, j;
 568         uint16_t sub_policy_num;
 569
 570         rte_spinlock_lock(&mtr_policy->sl);
 571         if (mtr_policy->ref_cnt) {
 572                 rte_spinlock_unlock(&mtr_policy->sl);
 573                 return -rte_mtr_error_set(error, EBUSY,
 574                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 575                                  NULL,
 576                                 "Meter policy object is being used.");
 577         }
 578         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 579         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 580         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 581                 sub_policy_num = (mtr_policy->sub_policy_num >>
 582                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 583                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 584                 if (sub_policy_num) {
 585                         for (j = 0; j < sub_policy_num; j++) {
 586                                 sub_policy = mtr_policy->sub_policys[i][j];
 587                                 if (sub_policy)
 588                                         mlx5_ipool_free
 589                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 590                                         sub_policy->idx);
 591                         }
 592                 }
 593         }
 594         if (priv->sh->mtrmng->policy_idx_tbl && clear_l3t) {
 595                 if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 596                                         policy_id)) {
 597                         rte_spinlock_unlock(&mtr_policy->sl);
 598                         return -rte_mtr_error_set(error, ENOTSUP,
 599                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 600                                 "Fail to delete policy in index table.");
 601                 }
 602         }
 603         rte_spinlock_unlock(&mtr_policy->sl);
 604         return 0;
 605 }
 606
 607 /**
 608  * Callback to add MTR policy.
 609  *
 610  * @param[in] dev
 611  *   Pointer to Ethernet device.
 612  * @param[out] policy_id
 613  *   Pointer to policy id
 614  * @param[in] actions
 615  *   Pointer to meter policy action detail.
 616  * @param[out] error
 617  *   Pointer to the error structure.
 618  *
 619  * @return
 620  *   0 on success, a negative errno value otherwise and rte_errno is set.
 621  */
 622 static int
 623 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 624                         uint32_t policy_id,
 625                         struct rte_mtr_meter_policy_params *policy,
 626                         struct rte_mtr_error *error)
 627 {
 628         struct mlx5_priv *priv = dev->data->dev_private;
 629         struct rte_flow_attr attr = { .transfer =
 630                         priv->config.dv_esw_en ? 1 : 0};
 631         uint32_t sub_policy_idx = 0;
 632         uint32_t policy_idx = 0;
 633         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 634         struct mlx5_flow_meter_sub_policy *sub_policy;
 635         bool is_rss = false;
 636         bool is_def_policy = false;
 637         uint32_t i;
 638         int ret;
 639         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 640         uint16_t sub_policy_num;
 641         uint8_t domain_bitmap = 0;
 642         union mlx5_l3t_data data;
 643
 644         if (!priv->mtr_en)
 645                 return -rte_mtr_error_set(error, ENOTSUP,
 646                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 647                                           NULL, "meter policy unsupported.");
 648         if (policy_id == MLX5_INVALID_POLICY_ID)
 649                 return -rte_mtr_error_set(error, ENOTSUP,
 650                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 651                         "policy ID is invalid. ");
 652         if (policy_id == priv->sh->mtrmng->def_policy_id)
 653                 return -rte_mtr_error_set(error, EEXIST,
 654                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 655                         "policy ID exists. ");
 656         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id,
 657                                 &policy_idx);
 658         if (mtr_policy)
 659                 return -rte_mtr_error_set(error, EEXIST,
 660                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 661                         "policy ID exists. ");
 662         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 663                         &is_rss, &domain_bitmap, &is_def_policy, error);
 664         if (ret)
 665                 return ret;
 666         if (!domain_bitmap)
 667                 return -rte_mtr_error_set(error, ENOTSUP,
 668                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 669                                 NULL, "fail to find policy domain.");
 670         if (is_def_policy) {
 671                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 672                         return -rte_mtr_error_set(error, EEXIST,
 673                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 674                                 NULL, "a policy with similar actions "
 675                                 "is already configured");
 676                 if (mlx5_flow_create_def_policy(dev))
 677                         return -rte_mtr_error_set(error, ENOTSUP,
 678                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 679                                 NULL,
 680                                 "fail to create non-terminated policy.");
 681                 priv->sh->mtrmng->def_policy_id = policy_id;
 682                 return 0;
 683         }
 684         if (!priv->sh->meter_aso_en)
 685                 return -rte_mtr_error_set(error, ENOTSUP,
 686                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 687                         "no ASO capability to support the policy ");
 688         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 689                 if (!(domain_bitmap & (1 << i)))
 690                         continue;
 691                 if (is_rss) {
 692                         policy_size +=
 693                         sizeof(struct mlx5_flow_meter_sub_policy *) *
 694                         MLX5_MTR_RSS_MAX_SUB_POLICY;
 695                         break;
 696                 }
 697                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 698         }
 699         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 700                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 701         if (!mtr_policy)
 702                 return -rte_mtr_error_set(error, ENOMEM,
 703                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 704                                 "Memory alloc failed for meter policy.");
 705         policy_size = sizeof(struct mlx5_flow_meter_policy);
 706         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 707                 if (!(domain_bitmap & (1 << i)))
 708                         continue;
 709                 if (i == MLX5_MTR_DOMAIN_INGRESS)
 710                         mtr_policy->ingress = 1;
 711                 if (i == MLX5_MTR_DOMAIN_EGRESS)
 712                         mtr_policy->egress = 1;
 713                 if (i == MLX5_MTR_DOMAIN_TRANSFER)
 714                         mtr_policy->transfer = 1;
 715                 sub_policy = mlx5_ipool_zmalloc
 716                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 717                                 &sub_policy_idx);
 718                 if (!sub_policy)
 719                         goto policy_add_err;
 720                 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 721                         goto policy_add_err;
 722                 sub_policy->idx = sub_policy_idx;
 723                 sub_policy->main_policy = mtr_policy;
 724                 if (!policy_idx) {
 725                         policy_idx = sub_policy_idx;
 726                         sub_policy->main_policy_id = 1;
 727                 }
 728                 mtr_policy->sub_policys[i] =
 729                 (struct mlx5_flow_meter_sub_policy **)
 730                         ((uint8_t *)mtr_policy + policy_size);
 731                 mtr_policy->sub_policys[i][0] = sub_policy;
 732                 sub_policy_num = (mtr_policy->sub_policy_num >>
 733                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 734                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 735                 sub_policy_num++;
 736                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 737                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 738                 mtr_policy->sub_policy_num |=
 739                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 740                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 741                 if (is_rss) {
 742                         mtr_policy->is_rss = 1;
 743                         break;
 744                 }
 745                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy  *);
 746         }
 747         rte_spinlock_init(&mtr_policy->sl);
 748         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 749                                         policy->actions, error);
 750         if (ret)
 751                 goto policy_add_err;
 752         if (!is_rss && !mtr_policy->is_queue) {
 753                 /* Create policy rules in HW. */
 754                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 755                 if (ret)
 756                         goto policy_add_err;
 757         }
 758         data.dword = policy_idx;
 759         if (!priv->sh->mtrmng->policy_idx_tbl) {
 760                 priv->sh->mtrmng->policy_idx_tbl =
 761                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 762                 if (!priv->sh->mtrmng->policy_idx_tbl)
 763                         goto policy_add_err;
 764         }
 765         if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl,
 766                                 policy_id, &data))
 767                 goto policy_add_err;
 768         return 0;
 769 policy_add_err:
 770         if (mtr_policy) {
 771                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 772                         mtr_policy, error, false);
 773                 mlx5_free(mtr_policy);
 774                 if (ret)
 775                         return ret;
 776         }
 777         return -rte_mtr_error_set(error, ENOTSUP,
 778                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 779                                   NULL, "Failed to create devx policy.");
 780 }
 781
 782 /**
 783  * Callback to delete MTR policy.
 784  *
 785  * @param[in] dev
 786  *   Pointer to Ethernet device.
 787  * @param[in] policy_id
 788  *   Meter policy id.
 789  * @param[out] error
 790  *   Pointer to the error structure.
 791  *
 792  * @return
 793  *   0 on success, a negative errno value otherwise and rte_errno is set.
 794  */
 795 static int
 796 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 797                           uint32_t policy_id,
 798                           struct rte_mtr_error *error)
 799 {
 800         struct mlx5_priv *priv = dev->data->dev_private;
 801         struct mlx5_flow_meter_policy *mtr_policy;
 802         uint32_t policy_idx;
 803         int ret;
 804
 805         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 806                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 807                         return -rte_mtr_error_set(error, ENOTSUP,
 808                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 809                                 "Meter policy object is being used.");
 810                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 811                 return 0;
 812         }
 813         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 814         if (!mtr_policy)
 815                 return -rte_mtr_error_set(error, ENOTSUP,
 816                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 817                         "Meter policy id is invalid. ");
 818         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 819                                                 error, true);
 820         if (ret)
 821                 return ret;
 822         mlx5_free(mtr_policy);
 823         return 0;
 824 }
 825
 826 /**
 827  * Check meter validation.
 828  *
 829  * @param[in] priv
 830  *   Pointer to mlx5 private data structure.
 831  * @param[in] meter_id
 832  *   Meter id.
 833  * @param[in] params
 834  *   Pointer to rte meter parameters.
 835  * @param[out] error
 836  *   Pointer to rte meter error structure.
 837  *
 838  * @return
 839  *   0 on success, a negative errno value otherwise and rte_errno is set.
 840  */
 841 static int
 842 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 843                          struct rte_mtr_params *params,
 844                          struct rte_mtr_error *error)
 845 {
 846         /* Meter must use global drop action. */
 847         if (!priv->sh->dr_drop_action)
 848                 return -rte_mtr_error_set(error, ENOTSUP,
 849                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 850                                           NULL,
 851                                           "No drop action ready for meter.");
 852         /* Meter params must not be NULL. */
 853         if (params == NULL)
 854                 return -rte_mtr_error_set(error, EINVAL,
 855                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 856                                           NULL, "Meter object params null.");
 857         /* Previous meter color is not supported. */
 858         if (params->use_prev_mtr_color)
 859                 return -rte_mtr_error_set(error, ENOTSUP,
 860                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 861                                           NULL,
 862                                           "Previous meter color "
 863                                           "not supported.");
 864         if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
 865                 return -rte_mtr_error_set(error, ENOENT,
 866                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 867                                 NULL, "Meter policy id not valid.");
 868         /* Validate meter id. */
 869         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 870                 return -rte_mtr_error_set(error, EEXIST,
 871                         RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 872                         "Meter object already exists.");
 873         return 0;
 874 }
 875
 876 /**
 877  * Modify the flow meter action.
 878  *
 879  * @param[in] priv
 880  *   Pointer to mlx5 private data structure.
 881  * @param[in] fm
 882  *   Pointer to flow meter to be modified.
 883  * @param[in] srtcm
 884  *   Pointer to meter srtcm description parameter.
 885  * @param[in] modify_bits
 886  *   The bit in srtcm to be updated.
 887  * @param[in] active_state
 888  *   The state to be updated.
 889  * @return
 890  *   0 on success, o negative value otherwise.
 891  */
 892 static int
 893 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 894                 struct mlx5_flow_meter_info *fm,
 895                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 896                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 897 {
 898 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 899         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 900         uint32_t *attr;
 901         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 902         int ret;
 903         struct mlx5_aso_mtr *aso_mtr = NULL;
 904         uint32_t cbs_cir, ebs_eir, val;
 905
 906         if (priv->sh->meter_aso_en) {
 907                 fm->is_enable = !!is_enable;
 908                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 909                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 910                 if (ret)
 911                         return ret;
 912                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 913                 if (ret)
 914                         return ret;
 915         } else {
 916                 /* Fill command parameters. */
 917                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 918                 mod_attr.flow_meter_parameter = in;
 919                 mod_attr.flow_meter_parameter_sz =
 920                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 921                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 922                         mod_attr.active = !!active_state;
 923                 else
 924                         mod_attr.active = 0;
 925                 attr = in;
 926                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 927                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 928                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 929                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 930                                 ASO_DSEG_EXP_MASK;
 931                         MLX5_SET(flow_meter_parameters, attr,
 932                                 cbs_exponent, val);
 933                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 934                                 ASO_DSEG_MAN_MASK;
 935                         MLX5_SET(flow_meter_parameters, attr,
 936                                 cbs_mantissa, val);
 937                 }
 938                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
 939                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
 940                                 ASO_DSEG_EXP_MASK;
 941                         MLX5_SET(flow_meter_parameters, attr,
 942                                 cir_exponent, val);
 943                         val = cbs_cir & ASO_DSEG_MAN_MASK;
 944                         MLX5_SET(flow_meter_parameters, attr,
 945                                 cir_mantissa, val);
 946                 }
 947                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
 948                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
 949                                 ASO_DSEG_EXP_MASK;
 950                         MLX5_SET(flow_meter_parameters, attr,
 951                                 ebs_exponent, val);
 952                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
 953                                 ASO_DSEG_MAN_MASK;
 954                         MLX5_SET(flow_meter_parameters, attr,
 955                                 ebs_mantissa, val);
 956                 }
 957                 /* Apply modifications to meter only if it was created. */
 958                 if (fm->meter_action) {
 959                         ret = mlx5_glue->dv_modify_flow_action_meter
 960                                         (fm->meter_action, &mod_attr,
 961                                         rte_cpu_to_be_64(modify_bits));
 962                         if (ret)
 963                                 return ret;
 964                 }
 965                 /* Update succeedded modify meter parameters. */
 966                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 967                         fm->active_state = !!active_state;
 968         }
 969         return 0;
 970 #else
 971         (void)priv;
 972         (void)fm;
 973         (void)srtcm;
 974         (void)modify_bits;
 975         (void)active_state;
 976         (void)is_enable;
 977         return -ENOTSUP;
 978 #endif
 979 }
 980
 981 static int
 982 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
 983                                 struct mlx5_flow_meter_info *fm,
 984                                 uint64_t stats_mask)
 985 {
 986         fm->bytes_dropped =
 987                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
 988         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
 989         if (fm->bytes_dropped || fm->pkts_dropped) {
 990                 if (!fm->drop_cnt) {
 991                         /* Alloc policer counters. */
 992                         fm->drop_cnt = mlx5_counter_alloc(dev);
 993                         if (!fm->drop_cnt)
 994                                 return -1;
 995                 }
 996         } else {
 997                 if (fm->drop_cnt) {
 998                         mlx5_counter_free(dev, fm->drop_cnt);
 999                         fm->drop_cnt = 0;
1000                 }
1001         }
1002         return 0;
1003 }
1004
1005 /**
1006  * Create meter rules.
1007  *
1008  * @param[in] dev
1009  *   Pointer to Ethernet device.
1010  * @param[in] meter_id
1011  *   Meter id.
1012  * @param[in] params
1013  *   Pointer to rte meter parameters.
1014  * @param[in] shared
1015  *   Meter shared with other flow or not.
1016  * @param[out] error
1017  *   Pointer to rte meter error structure.
1018  *
1019  * @return
1020  *   0 on success, a negative errno value otherwise and rte_errno is set.
1021  */
1022 static int
1023 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1024                        struct rte_mtr_params *params, int shared,
1025                        struct rte_mtr_error *error)
1026 {
1027         struct mlx5_priv *priv = dev->data->dev_private;
1028         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1029         struct mlx5_flow_meter_profile *fmp;
1030         struct mlx5_flow_meter_info *fm;
1031         struct mlx5_legacy_flow_meter *legacy_fm;
1032         struct mlx5_flow_meter_policy *mtr_policy = NULL;
1033         struct mlx5_indexed_pool_config flow_ipool_cfg = {
1034                 .size = 0,
1035                 .trunk_size = 64,
1036                 .need_lock = 1,
1037                 .type = "mlx5_flow_mtr_flow_id_pool",
1038         };
1039         struct mlx5_aso_mtr *aso_mtr;
1040         uint32_t mtr_idx, policy_idx;
1041         union mlx5_l3t_data data;
1042         int ret;
1043         uint8_t domain_bitmap;
1044         uint8_t mtr_id_bits;
1045         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1046                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1047
1048         if (!priv->mtr_en)
1049                 return -rte_mtr_error_set(error, ENOTSUP,
1050                                         RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1051                                         "Meter is not supported");
1052         /* Validate the parameters. */
1053         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1054         if (ret)
1055                 return ret;
1056         /* Meter profile must exist. */
1057         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1058         if (fmp == NULL)
1059                 return -rte_mtr_error_set(error, ENOENT,
1060                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1061                         NULL, "Meter profile id not valid.");
1062         /* Meter policy must exist. */
1063         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1064                 __atomic_add_fetch
1065                         (&priv->sh->mtrmng->def_policy_ref_cnt,
1066                         1, __ATOMIC_RELAXED);
1067                 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1068                 if (!priv->config.dv_esw_en)
1069                         domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1070         } else {
1071                 mtr_policy = mlx5_flow_meter_policy_find(dev,
1072                                 params->meter_policy_id, &policy_idx);
1073                 if (!priv->sh->meter_aso_en)
1074                         return -rte_mtr_error_set(error, ENOTSUP,
1075                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1076                                 "Part of the policies cannot be "
1077                                 "supported without ASO ");
1078                 if (!mtr_policy)
1079                         return -rte_mtr_error_set(error, ENOENT,
1080                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1081                                 NULL, "Meter policy id not valid.");
1082                 domain_bitmap = (mtr_policy->ingress ?
1083                                         MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1084                                 (mtr_policy->egress ?
1085                                         MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1086                                 (mtr_policy->transfer ?
1087                                         MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1088         }
1089         /* Allocate the flow meter memory. */
1090         if (priv->sh->meter_aso_en) {
1091                 mtr_idx = mlx5_flow_mtr_alloc(dev);
1092                 if (!mtr_idx)
1093                         return -rte_mtr_error_set(error, ENOMEM,
1094                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1095                                 "Memory alloc failed for meter.");
1096                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1097                 fm = &aso_mtr->fm;
1098         } else {
1099                 legacy_fm = mlx5_ipool_zmalloc
1100                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1101                 if (legacy_fm == NULL)
1102                         return -rte_mtr_error_set(error, ENOMEM,
1103                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1104                                 "Memory alloc failed for meter.");
1105                 legacy_fm->idx = mtr_idx;
1106                 fm = &legacy_fm->fm;
1107         }
1108         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1109         if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1110             mtr_reg_bits) {
1111                 DRV_LOG(ERR, "Meter number exceeds max limit.");
1112                 goto error;
1113         }
1114         if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1115                 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1116         /* Fill the flow meter parameters. */
1117         fm->meter_id = meter_id;
1118         fm->policy_id = params->meter_policy_id;
1119         fm->profile = fmp;
1120         if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1121                 goto error;
1122         if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1123                 goto error;
1124         /* Add to the flow meter list. */
1125         if (!priv->sh->meter_aso_en)
1126                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1127         /* Add to the flow meter list. */
1128         fm->active_state = 1; /* Config meter starts as active. */
1129         fm->is_enable = 1;
1130         fm->shared = !!shared;
1131         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1132         if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1133                 fm->def_policy = 1;
1134                 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1135                 if (!fm->flow_ipool)
1136                         goto error;
1137         }
1138         rte_spinlock_init(&fm->sl);
1139         /* If ASO meter supported, update ASO flow meter by wqe. */
1140         if (priv->sh->meter_aso_en) {
1141                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1142                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1143                 if (ret)
1144                         goto error;
1145                 if (!priv->mtr_idx_tbl) {
1146                         priv->mtr_idx_tbl =
1147                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1148                         if (!priv->mtr_idx_tbl)
1149                                 goto error;
1150                 }
1151                 data.dword = mtr_idx;
1152                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1153                         goto error;
1154         }
1155         if (mtr_policy)
1156                 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1157         return 0;
1158 error:
1159         mlx5_flow_destroy_mtr_tbls(dev, fm);
1160         /* Free policer counters. */
1161         if (fm->drop_cnt)
1162                 mlx5_counter_free(dev, fm->drop_cnt);
1163         if (priv->sh->meter_aso_en)
1164                 mlx5_flow_mtr_free(dev, mtr_idx);
1165         else
1166                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1167         return -rte_mtr_error_set(error, ENOTSUP,
1168                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1169                 NULL, "Failed to create devx meter.");
1170 }
1171
1172 static int
1173 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1174                         struct mlx5_flow_meter_info *fm,
1175                         uint32_t mtr_idx)
1176 {
1177         struct mlx5_priv *priv = dev->data->dev_private;
1178         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1179         struct mlx5_flow_meter_profile *fmp;
1180         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1181         struct mlx5_flow_meter_policy *mtr_policy;
1182
1183         /* Meter object must not have any owner. */
1184         MLX5_ASSERT(!fm->ref_cnt);
1185         /* Get meter profile. */
1186         fmp = fm->profile;
1187         if (fmp == NULL)
1188                 return -1;
1189         /* Update dependencies. */
1190         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1191         fm->profile = NULL;
1192         /* Remove from list. */
1193         if (!priv->sh->meter_aso_en) {
1194                 legacy_fm = container_of(fm,
1195                         struct mlx5_legacy_flow_meter, fm);
1196                 TAILQ_REMOVE(fms, legacy_fm, next);
1197         }
1198         /* Free drop counters. */
1199         if (fm->drop_cnt)
1200                 mlx5_counter_free(dev, fm->drop_cnt);
1201         /* Free meter flow table. */
1202         if (fm->flow_ipool) {
1203                 mlx5_ipool_destroy(fm->flow_ipool);
1204                 fm->flow_ipool = 0;
1205         }
1206         mlx5_flow_destroy_mtr_tbls(dev, fm);
1207         if (fm->def_policy)
1208                 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1209                                 1, __ATOMIC_RELAXED);
1210         if (priv->sh->meter_aso_en) {
1211                 if (!fm->def_policy) {
1212                         mtr_policy = mlx5_flow_meter_policy_find(dev,
1213                                                 fm->policy_id, NULL);
1214                         if (mtr_policy)
1215                                 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1216                                                 1, __ATOMIC_RELAXED);
1217                         fm->policy_id = 0;
1218                 }
1219                 fm->def_policy = 0;
1220                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1221                         return -1;
1222                 mlx5_flow_mtr_free(dev, mtr_idx);
1223         } else {
1224                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1225                                         legacy_fm->idx);
1226         }
1227         return 0;
1228 }
1229
1230 /**
1231  * Destroy meter rules.
1232  *
1233  * @param[in] dev
1234  *   Pointer to Ethernet device.
1235  * @param[in] meter_id
1236  *   Meter id.
1237  * @param[out] error
1238  *   Pointer to rte meter error structure.
1239  *
1240  * @return
1241  *   0 on success, a negative errno value otherwise and rte_errno is set.
1242  */
1243 static int
1244 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1245                         struct rte_mtr_error *error)
1246 {
1247         struct mlx5_priv *priv = dev->data->dev_private;
1248         struct mlx5_flow_meter_info *fm;
1249         uint32_t mtr_idx = 0;
1250
1251         if (!priv->mtr_en)
1252                 return -rte_mtr_error_set(error, ENOTSUP,
1253                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1254                                           NULL,
1255                                           "Meter is not supported");
1256         /* Meter object must exist. */
1257         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1258         if (fm == NULL)
1259                 return -rte_mtr_error_set(error, ENOENT,
1260                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1261                                           NULL,
1262                                           "Meter object id not valid.");
1263         /* Meter object must not have any owner. */
1264         if (fm->ref_cnt > 0)
1265                 return -rte_mtr_error_set(error, EBUSY,
1266                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1267                                           NULL,
1268                                           "Meter object is being used.");
1269         /* Destroy the meter profile. */
1270         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1271                 return -rte_mtr_error_set(error, EINVAL,
1272                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1273                                         NULL,
1274                                         "MTR object meter profile invalid.");
1275         return 0;
1276 }
1277
1278 /**
1279  * Modify meter state.
1280  *
1281  * @param[in] priv
1282  *   Pointer to mlx5 private data structure.
1283  * @param[in] fm
1284  *   Pointer to flow meter.
1285  * @param[in] new_state
1286  *   New state to update.
1287  * @param[out] error
1288  *   Pointer to rte meter error structure.
1289  *
1290  * @return
1291  *   0 on success, a negative errno value otherwise and rte_errno is set.
1292  */
1293 static int
1294 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1295                              struct mlx5_flow_meter_info *fm,
1296                              uint32_t new_state,
1297                              struct rte_mtr_error *error)
1298 {
1299         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1300                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1301                 .ebs_eir = 0,
1302         };
1303         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1304                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1305         int ret;
1306
1307         if (new_state == MLX5_FLOW_METER_DISABLE)
1308                 ret = mlx5_flow_meter_action_modify(priv, fm,
1309                                 &srtcm, modify_bits, 0, 0);
1310         else
1311                 ret = mlx5_flow_meter_action_modify(priv, fm,
1312                                                    &fm->profile->srtcm_prm,
1313                                                     modify_bits, 0, 1);
1314         if (ret)
1315                 return -rte_mtr_error_set(error, -ret,
1316                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1317                                           NULL,
1318                                           new_state ?
1319                                           "Failed to enable meter." :
1320                                           "Failed to disable meter.");
1321         return 0;
1322 }
1323
1324 /**
1325  * Callback to enable flow meter.
1326  *
1327  * @param[in] dev
1328  *   Pointer to Ethernet device.
1329  * @param[in] meter_id
1330  *   Meter id.
1331  * @param[out] error
1332  *   Pointer to rte meter error structure.
1333  *
1334  * @return
1335  *   0 on success, a negative errno value otherwise and rte_errno is set.
1336  */
1337 static int
1338 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1339                        uint32_t meter_id,
1340                        struct rte_mtr_error *error)
1341 {
1342         struct mlx5_priv *priv = dev->data->dev_private;
1343         struct mlx5_flow_meter_info *fm;
1344         int ret;
1345
1346         if (!priv->mtr_en)
1347                 return -rte_mtr_error_set(error, ENOTSUP,
1348                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1349                                           "Meter is not supported");
1350         /* Meter object must exist. */
1351         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1352         if (fm == NULL)
1353                 return -rte_mtr_error_set(error, ENOENT,
1354                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1355                                           NULL, "Meter not found.");
1356         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1357                 return 0;
1358         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1359                                            error);
1360         if (!ret)
1361                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1362         return ret;
1363 }
1364
1365 /**
1366  * Callback to disable flow meter.
1367  *
1368  * @param[in] dev
1369  *   Pointer to Ethernet device.
1370  * @param[in] meter_id
1371  *   Meter id.
1372  * @param[out] error
1373  *   Pointer to rte meter error structure.
1374  *
1375  * @return
1376  *   0 on success, a negative errno value otherwise and rte_errno is set.
1377  */
1378 static int
1379 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1380                         uint32_t meter_id,
1381                         struct rte_mtr_error *error)
1382 {
1383         struct mlx5_priv *priv = dev->data->dev_private;
1384         struct mlx5_flow_meter_info *fm;
1385         int ret;
1386
1387         if (!priv->mtr_en)
1388                 return -rte_mtr_error_set(error, ENOTSUP,
1389                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1390                                           "Meter is not supported");
1391         /* Meter object must exist. */
1392         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1393         if (fm == NULL)
1394                 return -rte_mtr_error_set(error, ENOENT,
1395                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1396                                           NULL, "Meter not found.");
1397         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1398                 return 0;
1399         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1400                                            error);
1401         if (!ret)
1402                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1403         return ret;
1404 }
1405
1406 /**
1407  * Callback to update meter profile.
1408  *
1409  * @param[in] dev
1410  *   Pointer to Ethernet device.
1411  * @param[in] meter_id
1412  *   Meter id.
1413  * @param[in] meter_profile_id
1414  *   To be updated meter profile id.
1415  * @param[out] error
1416  *   Pointer to rte meter error structure.
1417  *
1418  * @return
1419  *   0 on success, a negative errno value otherwise and rte_errno is set.
1420  */
1421 static int
1422 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1423                                uint32_t meter_id,
1424                                uint32_t meter_profile_id,
1425                                struct rte_mtr_error *error)
1426 {
1427         struct mlx5_priv *priv = dev->data->dev_private;
1428         struct mlx5_flow_meter_profile *fmp;
1429         struct mlx5_flow_meter_profile *old_fmp;
1430         struct mlx5_flow_meter_info *fm;
1431         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1432                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1433         int ret;
1434
1435         if (!priv->mtr_en)
1436                 return -rte_mtr_error_set(error, ENOTSUP,
1437                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1438                                           "Meter is not supported");
1439         /* Meter profile must exist. */
1440         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1441         if (fmp == NULL)
1442                 return -rte_mtr_error_set(error, ENOENT,
1443                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1444                                           NULL, "Meter profile not found.");
1445         /* Meter object must exist. */
1446         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1447         if (fm == NULL)
1448                 return -rte_mtr_error_set(error, ENOENT,
1449                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1450                                           NULL, "Meter not found.");
1451         /* MTR object already set to meter profile id. */
1452         old_fmp = fm->profile;
1453         if (fmp == old_fmp)
1454                 return 0;
1455         /* Update the profile. */
1456         fm->profile = fmp;
1457         /* Update meter params in HW (if not disabled). */
1458         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1459                 return 0;
1460         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1461                                               modify_bits, fm->active_state, 1);
1462         if (ret) {
1463                 fm->profile = old_fmp;
1464                 return -rte_mtr_error_set(error, -ret,
1465                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1466                                           NULL, "Failed to update meter"
1467                                           " parmeters in hardware.");
1468         }
1469         old_fmp->ref_cnt--;
1470         fmp->ref_cnt++;
1471         return 0;
1472 }
1473
1474 /**
1475  * Callback to update meter stats mask.
1476  *
1477  * @param[in] dev
1478  *   Pointer to Ethernet device.
1479  * @param[in] meter_id
1480  *   Meter id.
1481  * @param[in] stats_mask
1482  *   To be updated stats_mask.
1483  * @param[out] error
1484  *   Pointer to rte meter error structure.
1485  *
1486  * @return
1487  *   0 on success, a negative errno value otherwise and rte_errno is set.
1488  */
1489 static int
1490 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1491                              uint32_t meter_id,
1492                              uint64_t stats_mask,
1493                              struct rte_mtr_error *error)
1494 {
1495         struct mlx5_priv *priv = dev->data->dev_private;
1496         struct mlx5_flow_meter_info *fm;
1497
1498         if (!priv->mtr_en)
1499                 return -rte_mtr_error_set(error, ENOTSUP,
1500                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1501                                           "Meter is not supported");
1502         /* Meter object must exist. */
1503         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1504         if (fm == NULL)
1505                 return -rte_mtr_error_set(error, ENOENT,
1506                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1507                                           NULL, "Meter object id not valid.");
1508         if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1509                 return -rte_mtr_error_set(error, ENOENT,
1510                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1511                                           NULL, "Fail to allocate "
1512                                           "counter for meter.");
1513         return 0;
1514 }
1515
1516 /**
1517  * Callback to read meter statistics.
1518  *
1519  * @param[in] dev
1520  *   Pointer to Ethernet device.
1521  * @param[in] meter_id
1522  *   Meter id.
1523  * @param[out] stats
1524  *   Pointer to store the statistics.
1525  * @param[out] stats_mask
1526  *   Pointer to store the stats_mask.
1527  * @param[in] clear
1528  *   Statistic to be cleared after read or not.
1529  * @param[out] error
1530  *   Pointer to rte meter error structure.
1531  *
1532  * @return
1533  *   0 on success, a negative errno value otherwise and rte_errno is set.
1534  */
1535 static int
1536 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1537                            uint32_t meter_id,
1538                            struct rte_mtr_stats *stats,
1539                            uint64_t *stats_mask,
1540                            int clear,
1541                            struct rte_mtr_error *error)
1542 {
1543         struct mlx5_priv *priv = dev->data->dev_private;
1544         struct mlx5_flow_meter_info *fm;
1545         uint64_t pkts;
1546         uint64_t bytes;
1547         int ret = 0;
1548
1549         if (!priv->mtr_en)
1550                 return -rte_mtr_error_set(error, ENOTSUP,
1551                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1552                                           "Meter is not supported");
1553         /* Meter object must exist. */
1554         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1555         if (fm == NULL)
1556                 return -rte_mtr_error_set(error, ENOENT,
1557                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1558                                           NULL, "Meter object id not valid.");
1559         *stats_mask = 0;
1560         if (fm->bytes_dropped)
1561                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1562         if (fm->pkts_dropped)
1563                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1564         memset(stats, 0, sizeof(*stats));
1565         if (fm->drop_cnt) {
1566                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1567                                                  &bytes);
1568                 if (ret)
1569                         goto error;
1570                 /* If need to read the packets, set it. */
1571                 if (fm->pkts_dropped)
1572                         stats->n_pkts_dropped = pkts;
1573                 /* If need to read the bytes, set it. */
1574                 if (fm->bytes_dropped)
1575                         stats->n_bytes_dropped = bytes;
1576         }
1577         return 0;
1578 error:
1579         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1580                                  "Failed to read meter drop counters.");
1581 }
1582
1583 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1584         .capabilities_get = mlx5_flow_mtr_cap_get,
1585         .meter_profile_add = mlx5_flow_meter_profile_add,
1586         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1587         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1588         .meter_policy_add = mlx5_flow_meter_policy_add,
1589         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1590         .create = mlx5_flow_meter_create,
1591         .destroy = mlx5_flow_meter_destroy,
1592         .meter_enable = mlx5_flow_meter_enable,
1593         .meter_disable = mlx5_flow_meter_disable,
1594         .meter_profile_update = mlx5_flow_meter_profile_update,
1595         .meter_dscp_table_update = NULL,
1596         .stats_update = mlx5_flow_meter_stats_update,
1597         .stats_read = mlx5_flow_meter_stats_read,
1598 };
1599
1600 /**
1601  * Get meter operations.
1602  *
1603  * @param dev
1604  *   Pointer to Ethernet device structure.
1605  * @param arg
1606  *   Pointer to set the mtr operations.
1607  *
1608  * @return
1609  *   Always 0.
1610  */
1611 int
1612 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1613 {
1614         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1615         return 0;
1616 }
1617
1618 /**
1619  * Find meter by id.
1620  *
1621  * @param priv
1622  *   Pointer to mlx5_priv.
1623  * @param meter_id
1624  *   Meter id.
1625  * @param mtr_idx
1626  *   Pointer to Meter index.
1627  *
1628  * @return
1629  *   Pointer to the meter info found on success, NULL otherwise.
1630  */
1631 struct mlx5_flow_meter_info *
1632 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1633                 uint32_t *mtr_idx)
1634 {
1635         struct mlx5_legacy_flow_meter *legacy_fm;
1636         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1637         struct mlx5_aso_mtr *aso_mtr;
1638         struct mlx5_aso_mtr_pools_mng *pools_mng =
1639                                 &priv->sh->mtrmng->pools_mng;
1640         union mlx5_l3t_data data;
1641
1642         if (priv->sh->meter_aso_en) {
1643                 rte_spinlock_lock(&pools_mng->mtrsl);
1644                 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1645                         rte_spinlock_unlock(&pools_mng->mtrsl);
1646                         return NULL;
1647                 }
1648                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1649                         !data.dword) {
1650                         rte_spinlock_unlock(&pools_mng->mtrsl);
1651                         return NULL;
1652                 }
1653                 if (mtr_idx)
1654                         *mtr_idx = data.dword;
1655                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1656                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1657                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1658                 rte_spinlock_unlock(&pools_mng->mtrsl);
1659                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1660                         return NULL;
1661                 return &aso_mtr->fm;
1662         }
1663         TAILQ_FOREACH(legacy_fm, fms, next)
1664                 if (meter_id == legacy_fm->fm.meter_id) {
1665                         if (mtr_idx)
1666                                 *mtr_idx = legacy_fm->idx;
1667                         return &legacy_fm->fm;
1668                 }
1669         return NULL;
1670 }
1671
1672 /**
1673  * Find meter by index.
1674  *
1675  * @param priv
1676  *   Pointer to mlx5_priv.
1677  * @param idx
1678  *   Meter index.
1679  *
1680  * @return
1681  *   Pointer to the meter info found on success, NULL otherwise.
1682  */
1683 struct mlx5_flow_meter_info *
1684 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1685 {
1686         struct mlx5_aso_mtr *aso_mtr;
1687
1688         if (priv->sh->meter_aso_en) {
1689                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1690                 if (!aso_mtr)
1691                         return NULL;
1692                 return &aso_mtr->fm;
1693         } else {
1694                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1695         }
1696 }
1697
1698 /**
1699  * Attach meter to flow.
1700  * Unidirectional Meter creation can only be done
1701  * when flow direction is known, i.e. when calling meter_attach.
1702  *
1703  * @param [in] priv
1704  *  Pointer to mlx5 private data.
1705  * @param[in] fm
1706  *   Pointer to flow meter.
1707  * @param [in] attr
1708  *  Pointer to flow attributes.
1709  * @param [out] error
1710  *  Pointer to error structure.
1711  *
1712  * @return
1713  *   0 on success, a negative errno value otherwise and rte_errno is set.
1714  */
1715 int
1716 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1717                        struct mlx5_flow_meter_info *fm,
1718                        const struct rte_flow_attr *attr,
1719                        struct rte_flow_error *error)
1720 {
1721         int ret = 0;
1722
1723         if (priv->sh->meter_aso_en) {
1724                 struct mlx5_aso_mtr *aso_mtr;
1725
1726                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1727                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1728                         return rte_flow_error_set(error, ENOENT,
1729                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1730                                         NULL,
1731                                         "Timeout in meter configuration");
1732                 }
1733                 rte_spinlock_lock(&fm->sl);
1734                 if (fm->shared || !fm->ref_cnt) {
1735                         fm->ref_cnt++;
1736                 } else {
1737                         rte_flow_error_set(error, EINVAL,
1738                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1739                                    "Meter cannot be shared");
1740                         ret = -1;
1741                 }
1742                 rte_spinlock_unlock(&fm->sl);
1743         } else {
1744                 rte_spinlock_lock(&fm->sl);
1745                 if (fm->meter_action) {
1746                         if (fm->shared &&
1747                             attr->transfer == fm->transfer &&
1748                             attr->ingress == fm->ingress &&
1749                             attr->egress == fm->egress) {
1750                                 fm->ref_cnt++;
1751                         } else {
1752                                 rte_flow_error_set(error, EINVAL,
1753                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1754                                         fm->shared ?
1755                                         "Meter attr not match." :
1756                                         "Meter cannot be shared.");
1757                                 ret = -1;
1758                         }
1759                 } else {
1760                         fm->ingress = attr->ingress;
1761                         fm->egress = attr->egress;
1762                         fm->transfer = attr->transfer;
1763                         fm->ref_cnt = 1;
1764                         /* This also creates the meter object. */
1765                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1766                                                                          fm);
1767                         if (!fm->meter_action) {
1768                                 fm->ref_cnt = 0;
1769                                 fm->ingress = 0;
1770                                 fm->egress = 0;
1771                                 fm->transfer = 0;
1772                                 rte_flow_error_set(error, EINVAL,
1773                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1774                                         "Meter action create failed.");
1775                                 ret = -1;
1776                         }
1777                 }
1778                 rte_spinlock_unlock(&fm->sl);
1779         }
1780         return ret ? -rte_errno : 0;
1781 }
1782
1783 /**
1784  * Detach meter from flow.
1785  *
1786  * @param [in] priv
1787  *  Pointer to mlx5 private data.
1788  * @param [in] fm
1789  *  Pointer to flow meter.
1790  */
1791 void
1792 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1793                        struct mlx5_flow_meter_info *fm)
1794 {
1795 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1796         rte_spinlock_lock(&fm->sl);
1797         MLX5_ASSERT(fm->ref_cnt);
1798         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1799                 mlx5_glue->destroy_flow_action(fm->meter_action);
1800                 fm->meter_action = NULL;
1801                 fm->ingress = 0;
1802                 fm->egress = 0;
1803                 fm->transfer = 0;
1804         }
1805         rte_spinlock_unlock(&fm->sl);
1806 #else
1807         (void)priv;
1808         (void)fm;
1809 #endif
1810 }
1811
1812 /**
1813  * Flush meter with Rx queue configuration.
1814  *
1815  * @param[in] dev
1816  *   Pointer to Ethernet device.
1817  */
1818 void
1819 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1820 {
1821         struct mlx5_priv *priv = dev->data->dev_private;
1822         struct mlx5_flow_meter_sub_policy *sub_policy;
1823         struct mlx5_flow_meter_policy *mtr_policy;
1824         void *entry;
1825         uint32_t i, policy_idx;
1826
1827         if (!priv->mtr_en)
1828                 return;
1829         if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1830                 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1831                                         i, entry) {
1832                         policy_idx = *(uint32_t *)entry;
1833                         sub_policy = mlx5_ipool_get
1834                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1835                                 policy_idx);
1836                         if (!sub_policy || !sub_policy->main_policy)
1837                                 continue;
1838                         mtr_policy = sub_policy->main_policy;
1839                         if (mtr_policy->is_queue || mtr_policy->is_rss)
1840                                 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1841                                         mtr_policy);
1842                 }
1843         }
1844 }
1845
1846 /**
1847  * Flush meter configuration.
1848  *
1849  * @param[in] dev
1850  *   Pointer to Ethernet device.
1851  * @param[out] error
1852  *   Pointer to rte meter error structure.
1853  *
1854  * @return
1855  *   0 on success, a negative errno value otherwise and rte_errno is set.
1856  */
1857 int
1858 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1859 {
1860         struct mlx5_priv *priv = dev->data->dev_private;
1861         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1862         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
1863         struct mlx5_flow_meter_profile *fmp;
1864         struct mlx5_legacy_flow_meter *legacy_fm;
1865         struct mlx5_flow_meter_info *fm;
1866         struct mlx5_flow_meter_sub_policy *sub_policy;
1867         void *tmp;
1868         uint32_t i, mtr_idx, policy_idx;
1869         void *entry;
1870         struct mlx5_aso_mtr *aso_mtr;
1871
1872         if (!priv->mtr_en)
1873                 return 0;
1874         if (priv->sh->meter_aso_en) {
1875                 if (priv->mtr_idx_tbl) {
1876                         MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
1877                                 mtr_idx = *(uint32_t *)entry;
1878                                 if (mtr_idx) {
1879                                         aso_mtr =
1880                                         mlx5_aso_meter_by_idx(priv, mtr_idx);
1881                                         fm = &aso_mtr->fm;
1882                                         (void)mlx5_flow_meter_params_flush(dev,
1883                                                 fm, mtr_idx);
1884                                 }
1885                         }
1886                         mlx5_l3t_destroy(priv->mtr_idx_tbl);
1887                         priv->mtr_idx_tbl = NULL;
1888                 }
1889         } else {
1890                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1891                         fm = &legacy_fm->fm;
1892                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
1893                                 return -rte_mtr_error_set(error, EINVAL,
1894                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1895                                 NULL, "MTR object meter profile invalid.");
1896                 }
1897         }
1898         if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1899                 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1900                                         i, entry) {
1901                         policy_idx = *(uint32_t *)entry;
1902                         sub_policy = mlx5_ipool_get
1903                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1904                                 policy_idx);
1905                         if (!sub_policy)
1906                                 return -rte_mtr_error_set(error,
1907                                                 EINVAL,
1908                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1909                                                 NULL, "MTR object "
1910                                                 "meter policy invalid.");
1911                         if (__mlx5_flow_meter_policy_delete(dev, i,
1912                                                 sub_policy->main_policy,
1913                                                 error, true))
1914                                 return -rte_mtr_error_set(error,
1915                                                 EINVAL,
1916                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1917                                                 NULL, "MTR object "
1918                                                 "meter policy invalid.");
1919                         mlx5_free(sub_policy->main_policy);
1920                 }
1921                 mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl);
1922                 priv->sh->mtrmng->policy_idx_tbl = NULL;
1923         }
1924         TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) {
1925                 /* Check unused. */
1926                 MLX5_ASSERT(!fmp->ref_cnt);
1927                 /* Remove from list. */
1928                 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
1929                 mlx5_free(fmp);
1930         }
1931         /* Delete default policy table. */
1932         mlx5_flow_destroy_def_policy(dev);
1933         if (priv->sh->refcnt == 1)
1934                 mlx5_flow_destroy_mtr_drop_tbls(dev);
1935         return 0;
1936 }