drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41         enum mlx5_meter_domain domain =
  42                 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
  43                         fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
  44                                 MLX5_MTR_DOMAIN_INGRESS;
  45         struct mlx5_flow_meter_def_policy *def_policy =
  46                 priv->sh->mtrmng->def_policy[domain];
  47
  48         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  49         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  50         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  51         MLX5_SET(flow_meter_parameters, fmp,
  52                 start_color, MLX5_FLOW_COLOR_GREEN);
  53         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  54         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  55         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  56         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  58         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  60         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  61         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  62         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  63         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  64         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  65         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  66         mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
  67         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  68         mtr_init.flow_meter_parameter = fmp;
  69         mtr_init.flow_meter_parameter_sz =
  70                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  71         mtr_init.active = fm->active_state;
  72         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  73 #else
  74         (void)priv;
  75         (void)fm;
  76         return NULL;
  77 #endif
  78 }
  79
  80 /**
  81  * Find meter profile by id.
  82  *
  83  * @param priv
  84  *   Pointer to mlx5_priv.
  85  * @param meter_profile_id
  86  *   Meter profile id.
  87  *
  88  * @return
  89  *   Pointer to the profile found on success, NULL otherwise.
  90  */
  91 static struct mlx5_flow_meter_profile *
  92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  93 {
  94         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
  95         struct mlx5_flow_meter_profile *fmp;
  96
  97         TAILQ_FOREACH(fmp, fmps, next)
  98                 if (meter_profile_id == fmp->id)
  99                         return fmp;
 100         return NULL;
 101 }
 102
 103 /**
 104  * Validate the MTR profile.
 105  *
 106  * @param[in] dev
 107  *   Pointer to Ethernet device.
 108  * @param[in] meter_profile_id
 109  *   Meter profile id.
 110  * @param[in] profile
 111  *   Pointer to meter profile detail.
 112  * @param[out] error
 113  *   Pointer to the error structure.
 114  *
 115  * @return
 116  *   0 on success, a negative errno value otherwise and rte_errno is set.
 117  */
 118 static int
 119 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 120                                  uint32_t meter_profile_id,
 121                                  struct rte_mtr_meter_profile *profile,
 122                                  struct rte_mtr_error *error)
 123 {
 124         struct mlx5_priv *priv = dev->data->dev_private;
 125         struct mlx5_flow_meter_profile *fmp;
 126
 127         /* Profile must not be NULL. */
 128         if (profile == NULL)
 129                 return -rte_mtr_error_set(error, EINVAL,
 130                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 131                                           NULL, "Meter profile is null.");
 132         /* Meter profile ID must be valid. */
 133         if (meter_profile_id == UINT32_MAX)
 134                 return -rte_mtr_error_set(error, EINVAL,
 135                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 136                                           NULL, "Meter profile id not valid.");
 137         /* Meter profile must not exist. */
 138         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 139         if (fmp)
 140                 return -rte_mtr_error_set(error, EEXIST,
 141                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 142                                           NULL,
 143                                           "Meter profile already exists.");
 144         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 145                 if (priv->config.hca_attr.qos.flow_meter_old) {
 146                         /* Verify support for flow meter parameters. */
 147                         if (profile->srtcm_rfc2697.cir > 0 &&
 148                             profile->srtcm_rfc2697.cir <= MLX5_SRTCM_CIR_MAX &&
 149                             profile->srtcm_rfc2697.cbs > 0 &&
 150                             profile->srtcm_rfc2697.cbs <= MLX5_SRTCM_CBS_MAX &&
 151                             profile->srtcm_rfc2697.ebs <= MLX5_SRTCM_EBS_MAX)
 152                                 return 0;
 153                         else
 154                                 return -rte_mtr_error_set
 155                                              (error, ENOTSUP,
 156                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 157                                               NULL,
 158                                               profile->srtcm_rfc2697.ebs ?
 159                                               "Metering value ebs must be 0." :
 160                                               "Invalid metering parameters.");
 161                 }
 162         }
 163         return -rte_mtr_error_set(error, ENOTSUP,
 164                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 165                                   NULL, "Metering algorithm not supported.");
 166 }
 167
 168 /**
 169  * Calculate mantissa and exponent for cir.
 170  *
 171  * @param[in] cir
 172  *   Value to be calculated.
 173  * @param[out] man
 174  *   Pointer to the mantissa.
 175  * @param[out] exp
 176  *   Pointer to the exp.
 177  */
 178 static void
 179 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 180 {
 181         int64_t _cir;
 182         int64_t delta = INT64_MAX;
 183         uint8_t _man = 0;
 184         uint8_t _exp = 0;
 185         uint64_t m, e;
 186
 187         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 188                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 189                         _cir = (1000000000ULL * m) >> e;
 190                         if (llabs(cir - _cir) <= delta) {
 191                                 delta = llabs(cir - _cir);
 192                                 _man = m;
 193                                 _exp = e;
 194                         }
 195                 }
 196         }
 197         *man = _man;
 198         *exp = _exp;
 199 }
 200
 201 /**
 202  * Calculate mantissa and exponent for xbs.
 203  *
 204  * @param[in] xbs
 205  *   Value to be calculated.
 206  * @param[out] man
 207  *   Pointer to the mantissa.
 208  * @param[out] exp
 209  *   Pointer to the exp.
 210  */
 211 static void
 212 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 213 {
 214         int _exp;
 215         double _man;
 216
 217         /* Special case xbs == 0 ? both exp and matissa are 0. */
 218         if (xbs == 0) {
 219                 *man = 0;
 220                 *exp = 0;
 221                 return;
 222         }
 223         /* xbs = xbs_mantissa * 2^xbs_exponent */
 224         _man = frexp(xbs, &_exp);
 225         _man = _man * pow(2, MLX5_MAN_WIDTH);
 226         _exp = _exp - MLX5_MAN_WIDTH;
 227         *man = (uint8_t)ceil(_man);
 228         *exp = _exp;
 229 }
 230
 231 /**
 232  * Fill the prm meter parameter.
 233  *
 234  * @param[in,out] fmp
 235  *   Pointer to meter profie to be converted.
 236  * @param[out] error
 237  *   Pointer to the error structure.
 238  *
 239  * @return
 240  *   0 on success, a negative errno value otherwise and rte_errno is set.
 241  */
 242 static int
 243 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 244                           struct rte_mtr_error *error)
 245 {
 246         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 247         uint8_t man, exp;
 248         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 249         uint32_t ebs_exp, ebs_man;
 250
 251         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 252                 return -rte_mtr_error_set(error, ENOTSUP,
 253                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 254                                 NULL, "Metering algorithm not supported.");
 255         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 256         mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir,
 257                                     &man, &exp);
 258         /* Check if cir mantissa is too large. */
 259         if (exp > ASO_DSEG_CIR_EXP_MASK)
 260                 return -rte_mtr_error_set(error, ENOTSUP,
 261                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 262                                           "meter profile parameter cir is"
 263                                           " not supported.");
 264         cir_man = man;
 265         cir_exp = exp;
 266          /* cbs = cbs_mantissa * 2^cbs_exponent */
 267         mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.cbs,
 268                                     &man, &exp);
 269         /* Check if cbs mantissa is too large. */
 270         if (exp > ASO_DSEG_EXP_MASK)
 271                 return -rte_mtr_error_set(error, ENOTSUP,
 272                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 273                                           "meter profile parameter cbs is"
 274                                           " not supported.");
 275         cbs_man = man;
 276         cbs_exp = exp;
 277         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 278                                 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 279                                 cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 280                                 cir_man);
 281         mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.ebs,
 282                                     &man, &exp);
 283         /* Check if ebs mantissa is too large. */
 284         if (exp > ASO_DSEG_EXP_MASK)
 285                 return -rte_mtr_error_set(error, ENOTSUP,
 286                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 287                                           "meter profile parameter ebs is"
 288                                           " not supported.");
 289         ebs_man = man;
 290         ebs_exp = exp;
 291         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 292                                         ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 293         return 0;
 294 }
 295
 296 /**
 297  * Callback to get MTR capabilities.
 298  *
 299  * @param[in] dev
 300  *   Pointer to Ethernet device.
 301  * @param[out] cap
 302  *   Pointer to save MTR capabilities.
 303  * @param[out] error
 304  *   Pointer to the error structure.
 305  *
 306  * @return
 307  *   0 on success, a negative errno value otherwise and rte_errno is set.
 308  */
 309 static int
 310 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 311                  struct rte_mtr_capabilities *cap,
 312                  struct rte_mtr_error *error __rte_unused)
 313 {
 314         struct mlx5_priv *priv = dev->data->dev_private;
 315         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 316
 317         if (!priv->mtr_en)
 318                 return -rte_mtr_error_set(error, ENOTSUP,
 319                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 320                                           "Meter is not supported");
 321         memset(cap, 0, sizeof(*cap));
 322         if (priv->sh->meter_aso_en)
 323                 /* 2 meters per one ASO cache line. */
 324                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 325         else
 326                 cap->n_max = 1 << qattr->log_max_flow_meter;
 327         cap->n_shared_max = cap->n_max;
 328         cap->identical = 1;
 329         cap->shared_identical = 1;
 330         cap->shared_n_flows_per_mtr_max = 4 << 20;
 331         /* 2M flows can share the same meter. */
 332         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 333         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 334         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 335         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 336                           RTE_MTR_STATS_N_PKTS_DROPPED;
 337         return 0;
 338 }
 339
 340 /**
 341  * Callback to add MTR profile.
 342  *
 343  * @param[in] dev
 344  *   Pointer to Ethernet device.
 345  * @param[in] meter_profile_id
 346  *   Meter profile id.
 347  * @param[in] profile
 348  *   Pointer to meter profile detail.
 349  * @param[out] error
 350  *   Pointer to the error structure.
 351  *
 352  * @return
 353  *   0 on success, a negative errno value otherwise and rte_errno is set.
 354  */
 355 static int
 356 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 357                        uint32_t meter_profile_id,
 358                        struct rte_mtr_meter_profile *profile,
 359                        struct rte_mtr_error *error)
 360 {
 361         struct mlx5_priv *priv = dev->data->dev_private;
 362         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
 363         struct mlx5_flow_meter_profile *fmp;
 364         int ret;
 365
 366         if (!priv->mtr_en)
 367                 return -rte_mtr_error_set(error, ENOTSUP,
 368                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 369                                           "Meter is not supported");
 370         /* Check input params. */
 371         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 372                                                profile, error);
 373         if (ret)
 374                 return ret;
 375         /* Meter profile memory allocation. */
 376         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 377                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 378         if (fmp == NULL)
 379                 return -rte_mtr_error_set(error, ENOMEM,
 380                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 381                                           NULL, "Meter profile memory "
 382                                           "alloc failed.");
 383         /* Fill profile info. */
 384         fmp->id = meter_profile_id;
 385         fmp->profile = *profile;
 386         /* Fill the flow meter parameters for the PRM. */
 387         ret = mlx5_flow_meter_param_fill(fmp, error);
 388         if (ret)
 389                 goto error;
 390         /* Add to list. */
 391         TAILQ_INSERT_TAIL(fmps, fmp, next);
 392         return 0;
 393 error:
 394         mlx5_free(fmp);
 395         return ret;
 396 }
 397
 398 /**
 399  * Callback to delete MTR profile.
 400  *
 401  * @param[in] dev
 402  *   Pointer to Ethernet device.
 403  * @param[in] meter_profile_id
 404  *   Meter profile id.
 405  * @param[out] error
 406  *   Pointer to the error structure.
 407  *
 408  * @return
 409  *   0 on success, a negative errno value otherwise and rte_errno is set.
 410  */
 411 static int
 412 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 413                           uint32_t meter_profile_id,
 414                           struct rte_mtr_error *error)
 415 {
 416         struct mlx5_priv *priv = dev->data->dev_private;
 417         struct mlx5_flow_meter_profile *fmp;
 418
 419         if (!priv->mtr_en)
 420                 return -rte_mtr_error_set(error, ENOTSUP,
 421                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 422                                           "Meter is not supported");
 423         /* Meter profile must exist. */
 424         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 425         if (fmp == NULL)
 426                 return -rte_mtr_error_set(error, ENOENT,
 427                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 428                                           &meter_profile_id,
 429                                           "Meter profile id is invalid.");
 430         /* Check profile is unused. */
 431         if (fmp->ref_cnt)
 432                 return -rte_mtr_error_set(error, EBUSY,
 433                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 434                                           NULL, "Meter profile is in use.");
 435         /* Remove from list. */
 436         TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
 437         mlx5_free(fmp);
 438         return 0;
 439 }
 440
 441 /**
 442  * Find policy by id.
 443  *
 444  * @param[in] dev
 445  *   Pointer to Ethernet device.
 446  * @param policy_id
 447  *   Policy id.
 448  *
 449  * @return
 450  *   Pointer to the policy found on success, NULL otherwise.
 451  */
 452 struct mlx5_flow_meter_policy *
 453 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
 454                             uint32_t policy_id,
 455                             uint32_t *policy_idx)
 456 {
 457         struct mlx5_priv *priv = dev->data->dev_private;
 458         struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
 459         union mlx5_l3t_data data;
 460
 461         if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM ||
 462                 !priv->sh->mtrmng->policy_idx_tbl)
 463                 return NULL;
 464         if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl,
 465                                 policy_id, &data) ||
 466                                 !data.dword)
 467                 return NULL;
 468         if (policy_idx)
 469                 *policy_idx = data.dword;
 470         sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 471                                         data.dword);
 472         /* Remove reference taken by the mlx5_l3t_get_entry. */
 473         mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 474                                 policy_id);
 475         if (sub_policy)
 476                 if (sub_policy->main_policy_id)
 477                         return sub_policy->main_policy;
 478         return NULL;
 479 }
 480
 481 /**
 482  * Callback to check MTR policy action validate
 483  *
 484  * @param[in] dev
 485  *   Pointer to Ethernet device.
 486  * @param[in] actions
 487  *   Pointer to meter policy action detail.
 488  * @param[out] error
 489  *   Pointer to the error structure.
 490  *
 491  * @return
 492  *   0 on success, a negative errno value otherwise and rte_errno is set.
 493  */
 494 static int
 495 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
 496         struct rte_mtr_meter_policy_params *policy,
 497         struct rte_mtr_error *error)
 498 {
 499         struct mlx5_priv *priv = dev->data->dev_private;
 500         struct rte_flow_attr attr = { .transfer =
 501                         priv->config.dv_esw_en ? 1 : 0};
 502         bool is_rss = false;
 503         bool is_def_policy = false;
 504         uint8_t domain_bitmap;
 505         int ret;
 506
 507         if (!priv->mtr_en || !priv->sh->meter_aso_en)
 508                 return -rte_mtr_error_set(error, ENOTSUP,
 509                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 510                                 NULL, "meter policy unsupported.");
 511         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 512                         &is_rss, &domain_bitmap, &is_def_policy, error);
 513         if (ret)
 514                 return ret;
 515         return 0;
 516 }
 517
 518 static int
 519 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 520                         uint32_t policy_id,
 521                         struct mlx5_flow_meter_policy *mtr_policy,
 522                         struct rte_mtr_error *error)
 523 {
 524         struct mlx5_priv *priv = dev->data->dev_private;
 525         struct mlx5_flow_meter_sub_policy *sub_policy;
 526         uint32_t i, j;
 527         uint16_t sub_policy_num;
 528
 529         rte_spinlock_lock(&mtr_policy->sl);
 530         if (mtr_policy->ref_cnt) {
 531                 rte_spinlock_unlock(&mtr_policy->sl);
 532                 return -rte_mtr_error_set(error, EBUSY,
 533                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 534                                  NULL,
 535                                 "Meter policy object is being used.");
 536         }
 537         mlx5_flow_destroy_policy_rules(dev, mtr_policy);
 538         mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
 539         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 540                 sub_policy_num = (mtr_policy->sub_policy_num >>
 541                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 542                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 543                 if (sub_policy_num) {
 544                         for (j = 0; j < sub_policy_num; j++) {
 545                                 sub_policy = mtr_policy->sub_policys[i][j];
 546                                 if (sub_policy)
 547                                         mlx5_ipool_free
 548                                         (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 549                                         sub_policy->idx);
 550                         }
 551                 }
 552         }
 553         if (priv->sh->mtrmng->policy_idx_tbl) {
 554                 if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
 555                                         policy_id)) {
 556                         rte_spinlock_unlock(&mtr_policy->sl);
 557                         return -rte_mtr_error_set(error, ENOTSUP,
 558                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 559                                 "Fail to delete policy in index table.");
 560                 }
 561         }
 562         rte_spinlock_unlock(&mtr_policy->sl);
 563         return 0;
 564 }
 565
 566 /**
 567  * Callback to add MTR policy.
 568  *
 569  * @param[in] dev
 570  *   Pointer to Ethernet device.
 571  * @param[out] policy_id
 572  *   Pointer to policy id
 573  * @param[in] actions
 574  *   Pointer to meter policy action detail.
 575  * @param[out] error
 576  *   Pointer to the error structure.
 577  *
 578  * @return
 579  *   0 on success, a negative errno value otherwise and rte_errno is set.
 580  */
 581 static int
 582 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
 583                         uint32_t policy_id,
 584                         struct rte_mtr_meter_policy_params *policy,
 585                         struct rte_mtr_error *error)
 586 {
 587         struct mlx5_priv *priv = dev->data->dev_private;
 588         struct rte_flow_attr attr = { .transfer =
 589                         priv->config.dv_esw_en ? 1 : 0};
 590         uint32_t sub_policy_idx = 0;
 591         uint32_t policy_idx = 0;
 592         struct mlx5_flow_meter_policy *mtr_policy = NULL;
 593         struct mlx5_flow_meter_sub_policy *sub_policy;
 594         bool is_rss = false;
 595         bool is_def_policy = false;
 596         uint32_t i;
 597         int ret;
 598         uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
 599         uint16_t sub_policy_num;
 600         uint8_t domain_bitmap = 0;
 601         union mlx5_l3t_data data;
 602
 603         if (!priv->mtr_en)
 604                 return -rte_mtr_error_set(error, ENOTSUP,
 605                                           RTE_MTR_ERROR_TYPE_METER_POLICY,
 606                                           NULL, "meter policy unsupported.");
 607         if (policy_id == MLX5_INVALID_POLICY_ID)
 608                 return -rte_mtr_error_set(error, ENOTSUP,
 609                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 610                         "policy ID is invalid. ");
 611         if (policy_id == priv->sh->mtrmng->def_policy_id)
 612                 return -rte_mtr_error_set(error, EEXIST,
 613                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 614                         "policy ID exists. ");
 615         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id,
 616                                 &policy_idx);
 617         if (mtr_policy)
 618                 return -rte_mtr_error_set(error, EEXIST,
 619                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 620                         "policy ID exists. ");
 621         ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
 622                         &is_rss, &domain_bitmap, &is_def_policy, error);
 623         if (ret)
 624                 return ret;
 625         if (!domain_bitmap)
 626                 return -rte_mtr_error_set(error, ENOTSUP,
 627                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 628                                 NULL, "fail to find policy domain.");
 629         if (is_def_policy) {
 630                 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
 631                         return -rte_mtr_error_set(error, EEXIST,
 632                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
 633                                 NULL, "a policy with similar actions "
 634                                 "is already configured");
 635                 if (mlx5_flow_create_def_policy(dev))
 636                         return -rte_mtr_error_set(error, ENOTSUP,
 637                                 RTE_MTR_ERROR_TYPE_METER_POLICY,
 638                                 NULL,
 639                                 "fail to create non-terminated policy.");
 640                 priv->sh->mtrmng->def_policy_id = policy_id;
 641                 return 0;
 642         }
 643         if (!priv->sh->meter_aso_en)
 644                 return -rte_mtr_error_set(error, ENOTSUP,
 645                         RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 646                         "no ASO capability to support the policy ");
 647         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 648                 if (!(domain_bitmap & (1 << i)))
 649                         continue;
 650                 if (is_rss) {
 651                         policy_size +=
 652                         sizeof(struct mlx5_flow_meter_sub_policy *) *
 653                         MLX5_MTR_RSS_MAX_SUB_POLICY;
 654                         break;
 655                 }
 656                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
 657         }
 658         mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
 659                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 660         if (!mtr_policy)
 661                 return -rte_mtr_error_set(error, ENOMEM,
 662                                 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
 663                                 "Memory alloc failed for meter policy.");
 664         policy_size = sizeof(struct mlx5_flow_meter_policy);
 665         for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
 666                 if (!(domain_bitmap & (1 << i)))
 667                         continue;
 668                 mtr_policy->ingress = (i == MLX5_MTR_DOMAIN_INGRESS) ? 1 : 0;
 669                 mtr_policy->egress = (i == MLX5_MTR_DOMAIN_EGRESS) ? 1 : 0;
 670                 mtr_policy->transfer = (i == MLX5_MTR_DOMAIN_TRANSFER) ? 1 : 0;
 671                 sub_policy = mlx5_ipool_zmalloc
 672                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
 673                                 &sub_policy_idx);
 674                 if (!sub_policy)
 675                         goto policy_add_err;
 676                 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
 677                         goto policy_add_err;
 678                 sub_policy->idx = sub_policy_idx;
 679                 sub_policy->main_policy = mtr_policy;
 680                 if (!policy_idx) {
 681                         policy_idx = sub_policy_idx;
 682                         sub_policy->main_policy_id = 1;
 683                 }
 684                 mtr_policy->sub_policys[i] =
 685                 (struct mlx5_flow_meter_sub_policy **)
 686                         ((uint8_t *)mtr_policy + policy_size);
 687                 mtr_policy->sub_policys[i][0] = sub_policy;
 688                 sub_policy_num = (mtr_policy->sub_policy_num >>
 689                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
 690                         MLX5_MTR_SUB_POLICY_NUM_MASK;
 691                 sub_policy_num++;
 692                 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
 693                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
 694                 mtr_policy->sub_policy_num |=
 695                         (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
 696                         (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
 697                 if (is_rss) {
 698                         mtr_policy->is_rss = 1;
 699                         break;
 700                 }
 701                 policy_size += sizeof(struct mlx5_flow_meter_sub_policy  *);
 702         }
 703         rte_spinlock_init(&mtr_policy->sl);
 704         ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
 705                                         policy->actions, error);
 706         if (ret)
 707                 goto policy_add_err;
 708         if (!is_rss) {
 709                 /* Create policy rules in HW. */
 710                 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
 711                 if (ret)
 712                         goto policy_add_err;
 713         }
 714         data.dword = policy_idx;
 715         if (!priv->sh->mtrmng->policy_idx_tbl) {
 716                 priv->sh->mtrmng->policy_idx_tbl =
 717                                 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
 718                 if (!priv->sh->mtrmng->policy_idx_tbl)
 719                         goto policy_add_err;
 720         }
 721         if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl,
 722                                 policy_id, &data))
 723                 goto policy_add_err;
 724         return 0;
 725 policy_add_err:
 726         if (mtr_policy) {
 727                 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
 728                         mtr_policy, error);
 729                 mlx5_free(mtr_policy);
 730                 if (ret)
 731                         return ret;
 732         }
 733         return -rte_mtr_error_set(error, ENOTSUP,
 734                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 735                                   NULL, "Failed to create devx policy.");
 736 }
 737
 738 /**
 739  * Callback to delete MTR policy.
 740  *
 741  * @param[in] dev
 742  *   Pointer to Ethernet device.
 743  * @param[in] policy_id
 744  *   Meter policy id.
 745  * @param[out] error
 746  *   Pointer to the error structure.
 747  *
 748  * @return
 749  *   0 on success, a negative errno value otherwise and rte_errno is set.
 750  */
 751 static int
 752 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
 753                           uint32_t policy_id,
 754                           struct rte_mtr_error *error)
 755 {
 756         struct mlx5_priv *priv = dev->data->dev_private;
 757         struct mlx5_flow_meter_policy *mtr_policy;
 758         uint32_t policy_idx;
 759         int ret;
 760
 761         if (policy_id == priv->sh->mtrmng->def_policy_id) {
 762                 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
 763                         return -rte_mtr_error_set(error, ENOTSUP,
 764                                 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 765                                 "Meter policy object is being used.");
 766                 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
 767                 return 0;
 768         }
 769         mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
 770         if (!mtr_policy)
 771                 return -rte_mtr_error_set(error, ENOTSUP,
 772                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
 773                         "Meter policy id is invalid. ");
 774         ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
 775                                                 error);
 776         if (ret)
 777                 return ret;
 778         mlx5_free(mtr_policy);
 779         return 0;
 780 }
 781
 782 /**
 783  * Modify the flow meter action.
 784  *
 785  * @param[in] priv
 786  *   Pointer to mlx5 private data structure.
 787  * @param[in] fm
 788  *   Pointer to flow meter to be modified.
 789  * @param[in] srtcm
 790  *   Pointer to meter srtcm description parameter.
 791  * @param[in] modify_bits
 792  *   The bit in srtcm to be updated.
 793  * @param[in] active_state
 794  *   The state to be updated.
 795  * @return
 796  *   0 on success, o negative value otherwise.
 797  */
 798 static int
 799 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 800                 struct mlx5_flow_meter_info *fm,
 801                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 802                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 803 {
 804 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 805         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 806         uint32_t *attr;
 807         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 808         int ret;
 809         struct mlx5_aso_mtr *aso_mtr = NULL;
 810         uint32_t cbs_cir, ebs_eir, val;
 811
 812         if (priv->sh->meter_aso_en) {
 813                 fm->is_enable = !!is_enable;
 814                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 815                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 816                 if (ret)
 817                         return ret;
 818                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 819                 if (ret)
 820                         return ret;
 821         } else {
 822                 /* Fill command parameters. */
 823                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 824                 mod_attr.flow_meter_parameter = in;
 825                 mod_attr.flow_meter_parameter_sz =
 826                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 827                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 828                         mod_attr.active = !!active_state;
 829                 else
 830                         mod_attr.active = 0;
 831                 attr = in;
 832                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 833                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 834                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 835                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 836                                 ASO_DSEG_EXP_MASK;
 837                         MLX5_SET(flow_meter_parameters, attr,
 838                                 cbs_exponent, val);
 839                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 840                                 ASO_DSEG_MAN_MASK;
 841                         MLX5_SET(flow_meter_parameters, attr,
 842                                 cbs_mantissa, val);
 843                 }
 844                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
 845                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
 846                                 ASO_DSEG_EXP_MASK;
 847                         MLX5_SET(flow_meter_parameters, attr,
 848                                 cir_exponent, val);
 849                         val = cbs_cir & ASO_DSEG_MAN_MASK;
 850                         MLX5_SET(flow_meter_parameters, attr,
 851                                 cir_mantissa, val);
 852                 }
 853                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
 854                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
 855                                 ASO_DSEG_EXP_MASK;
 856                         MLX5_SET(flow_meter_parameters, attr,
 857                                 ebs_exponent, val);
 858                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
 859                                 ASO_DSEG_MAN_MASK;
 860                         MLX5_SET(flow_meter_parameters, attr,
 861                                 ebs_mantissa, val);
 862                 }
 863                 /* Apply modifications to meter only if it was created. */
 864                 if (fm->meter_action) {
 865                         ret = mlx5_glue->dv_modify_flow_action_meter
 866                                         (fm->meter_action, &mod_attr,
 867                                         rte_cpu_to_be_64(modify_bits));
 868                         if (ret)
 869                                 return ret;
 870                 }
 871                 /* Update succeedded modify meter parameters. */
 872                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 873                         fm->active_state = !!active_state;
 874         }
 875         return 0;
 876 #else
 877         (void)priv;
 878         (void)fm;
 879         (void)srtcm;
 880         (void)modify_bits;
 881         (void)active_state;
 882         (void)is_enable;
 883         return -ENOTSUP;
 884 #endif
 885 }
 886
 887 static void
 888 mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter_info *fm,
 889                                 uint64_t stats_mask)
 890 {
 891         fm->bytes_dropped =
 892                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
 893         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
 894 }
 895
 896 static int
 897 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
 898                         struct mlx5_flow_meter_info *fm,
 899                         uint32_t mtr_idx)
 900 {
 901         struct mlx5_priv *priv = dev->data->dev_private;
 902         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
 903         struct mlx5_flow_meter_profile *fmp;
 904         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
 905
 906         /* Meter object must not have any owner. */
 907         MLX5_ASSERT(!fm->ref_cnt);
 908         /* Get meter profile. */
 909         fmp = fm->profile;
 910         if (fmp == NULL)
 911                 return -1;
 912         /* Update dependencies. */
 913         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
 914         /* Remove from list. */
 915         if (!priv->sh->meter_aso_en) {
 916                 legacy_fm = container_of(fm, struct mlx5_legacy_flow_meter, fm);
 917                 TAILQ_REMOVE(fms, legacy_fm, next);
 918         }
 919         /* Free drop counters. */
 920         if (fm->drop_cnt)
 921                 mlx5_counter_free(dev, fm->drop_cnt);
 922         /* Free meter flow table. */
 923         if (fm->flow_ipool)
 924                 mlx5_ipool_destroy(fm->flow_ipool);
 925         mlx5_flow_destroy_mtr_tbls(dev, fm->mfts);
 926         if (priv->sh->meter_aso_en)
 927                 mlx5_flow_mtr_free(dev, mtr_idx);
 928         else
 929                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
 930                                         legacy_fm->idx);
 931         return 0;
 932 }
 933
 934 /**
 935  * Destroy meter rules.
 936  *
 937  * @param[in] dev
 938  *   Pointer to Ethernet device.
 939  * @param[in] meter_id
 940  *   Meter id.
 941  * @param[out] error
 942  *   Pointer to rte meter error structure.
 943  *
 944  * @return
 945  *   0 on success, a negative errno value otherwise and rte_errno is set.
 946  */
 947 static int
 948 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
 949                         struct rte_mtr_error *error)
 950 {
 951         struct mlx5_priv *priv = dev->data->dev_private;
 952         struct mlx5_flow_meter_info *fm;
 953         uint32_t mtr_idx = 0;
 954
 955         if (!priv->mtr_en)
 956                 return -rte_mtr_error_set(error, ENOTSUP,
 957                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 958                                           "Meter is not supported");
 959         /* Meter object must exist. */
 960         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
 961         if (fm == NULL)
 962                 return -rte_mtr_error_set(error, ENOENT,
 963                                           RTE_MTR_ERROR_TYPE_MTR_ID,
 964                                           NULL, "Meter object id not valid.");
 965         /* Meter object must not have any owner. */
 966         if (fm->ref_cnt > 0)
 967                 return -rte_mtr_error_set(error, EBUSY,
 968                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 969                                           NULL, "Meter object is being used.");
 970         if (priv->sh->meter_aso_en) {
 971                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id))
 972                         return -rte_mtr_error_set(error, EBUSY,
 973                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 974                                 "Fail to delete ASO Meter in index table.");
 975         }
 976         /* Destroy the meter profile. */
 977         if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
 978                 return -rte_mtr_error_set(error, EINVAL,
 979                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 980                                         NULL, "MTR object meter profile invalid.");
 981         return 0;
 982 }
 983
 984 /**
 985  * Modify meter state.
 986  *
 987  * @param[in] priv
 988  *   Pointer to mlx5 private data structure.
 989  * @param[in] fm
 990  *   Pointer to flow meter.
 991  * @param[in] new_state
 992  *   New state to update.
 993  * @param[out] error
 994  *   Pointer to rte meter error structure.
 995  *
 996  * @return
 997  *   0 on success, a negative errno value otherwise and rte_errno is set.
 998  */
 999 static int
1000 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1001                              struct mlx5_flow_meter_info *fm,
1002                              uint32_t new_state,
1003                              struct rte_mtr_error *error)
1004 {
1005         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1006                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1007                 .ebs_eir = 0,
1008         };
1009         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1010                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1011         int ret;
1012
1013         if (new_state == MLX5_FLOW_METER_DISABLE)
1014                 ret = mlx5_flow_meter_action_modify(priv, fm,
1015                                 &srtcm, modify_bits, 0, 0);
1016         else
1017                 ret = mlx5_flow_meter_action_modify(priv, fm,
1018                                                    &fm->profile->srtcm_prm,
1019                                                     modify_bits, 0, 1);
1020         if (ret)
1021                 return -rte_mtr_error_set(error, -ret,
1022                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1023                                           NULL,
1024                                           new_state ?
1025                                           "Failed to enable meter." :
1026                                           "Failed to disable meter.");
1027         return 0;
1028 }
1029
1030 /**
1031  * Callback to enable flow meter.
1032  *
1033  * @param[in] dev
1034  *   Pointer to Ethernet device.
1035  * @param[in] meter_id
1036  *   Meter id.
1037  * @param[out] error
1038  *   Pointer to rte meter error structure.
1039  *
1040  * @return
1041  *   0 on success, a negative errno value otherwise and rte_errno is set.
1042  */
1043 static int
1044 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1045                        uint32_t meter_id,
1046                        struct rte_mtr_error *error)
1047 {
1048         struct mlx5_priv *priv = dev->data->dev_private;
1049         struct mlx5_flow_meter_info *fm;
1050         int ret;
1051
1052         if (!priv->mtr_en)
1053                 return -rte_mtr_error_set(error, ENOTSUP,
1054                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1055                                           "Meter is not supported");
1056         /* Meter object must exist. */
1057         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1058         if (fm == NULL)
1059                 return -rte_mtr_error_set(error, ENOENT,
1060                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1061                                           NULL, "Meter not found.");
1062         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1063                 return 0;
1064         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1065                                            error);
1066         if (!ret)
1067                 fm->active_state = MLX5_FLOW_METER_ENABLE;
1068         return ret;
1069 }
1070
1071 /**
1072  * Callback to disable flow meter.
1073  *
1074  * @param[in] dev
1075  *   Pointer to Ethernet device.
1076  * @param[in] meter_id
1077  *   Meter id.
1078  * @param[out] error
1079  *   Pointer to rte meter error structure.
1080  *
1081  * @return
1082  *   0 on success, a negative errno value otherwise and rte_errno is set.
1083  */
1084 static int
1085 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1086                         uint32_t meter_id,
1087                         struct rte_mtr_error *error)
1088 {
1089         struct mlx5_priv *priv = dev->data->dev_private;
1090         struct mlx5_flow_meter_info *fm;
1091         int ret;
1092
1093         if (!priv->mtr_en)
1094                 return -rte_mtr_error_set(error, ENOTSUP,
1095                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1096                                           "Meter is not supported");
1097         /* Meter object must exist. */
1098         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1099         if (fm == NULL)
1100                 return -rte_mtr_error_set(error, ENOENT,
1101                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1102                                           NULL, "Meter not found.");
1103         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1104                 return 0;
1105         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1106                                            error);
1107         if (!ret)
1108                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1109         return ret;
1110 }
1111
1112 /**
1113  * Callback to update meter profile.
1114  *
1115  * @param[in] dev
1116  *   Pointer to Ethernet device.
1117  * @param[in] meter_id
1118  *   Meter id.
1119  * @param[in] meter_profile_id
1120  *   To be updated meter profile id.
1121  * @param[out] error
1122  *   Pointer to rte meter error structure.
1123  *
1124  * @return
1125  *   0 on success, a negative errno value otherwise and rte_errno is set.
1126  */
1127 static int
1128 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1129                                uint32_t meter_id,
1130                                uint32_t meter_profile_id,
1131                                struct rte_mtr_error *error)
1132 {
1133         struct mlx5_priv *priv = dev->data->dev_private;
1134         struct mlx5_flow_meter_profile *fmp;
1135         struct mlx5_flow_meter_profile *old_fmp;
1136         struct mlx5_flow_meter_info *fm;
1137         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1138                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1139         int ret;
1140
1141         if (!priv->mtr_en)
1142                 return -rte_mtr_error_set(error, ENOTSUP,
1143                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1144                                           "Meter is not supported");
1145         /* Meter profile must exist. */
1146         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1147         if (fmp == NULL)
1148                 return -rte_mtr_error_set(error, ENOENT,
1149                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1150                                           NULL, "Meter profile not found.");
1151         /* Meter object must exist. */
1152         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1153         if (fm == NULL)
1154                 return -rte_mtr_error_set(error, ENOENT,
1155                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1156                                           NULL, "Meter not found.");
1157         /* MTR object already set to meter profile id. */
1158         old_fmp = fm->profile;
1159         if (fmp == old_fmp)
1160                 return 0;
1161         /* Update the profile. */
1162         fm->profile = fmp;
1163         /* Update meter params in HW (if not disabled). */
1164         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1165                 return 0;
1166         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1167                                               modify_bits, fm->active_state, 1);
1168         if (ret) {
1169                 fm->profile = old_fmp;
1170                 return -rte_mtr_error_set(error, -ret,
1171                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1172                                           NULL, "Failed to update meter"
1173                                           " parmeters in hardware.");
1174         }
1175         old_fmp->ref_cnt--;
1176         fmp->ref_cnt++;
1177         return 0;
1178 }
1179
1180 /**
1181  * Callback to update meter stats mask.
1182  *
1183  * @param[in] dev
1184  *   Pointer to Ethernet device.
1185  * @param[in] meter_id
1186  *   Meter id.
1187  * @param[in] stats_mask
1188  *   To be updated stats_mask.
1189  * @param[out] error
1190  *   Pointer to rte meter error structure.
1191  *
1192  * @return
1193  *   0 on success, a negative errno value otherwise and rte_errno is set.
1194  */
1195 static int
1196 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1197                              uint32_t meter_id,
1198                              uint64_t stats_mask,
1199                              struct rte_mtr_error *error)
1200 {
1201         struct mlx5_priv *priv = dev->data->dev_private;
1202         struct mlx5_flow_meter_info *fm;
1203
1204         if (!priv->mtr_en)
1205                 return -rte_mtr_error_set(error, ENOTSUP,
1206                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1207                                           "Meter is not supported");
1208         /* Meter object must exist. */
1209         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1210         if (fm == NULL)
1211                 return -rte_mtr_error_set(error, ENOENT,
1212                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1213                                           NULL, "Meter object id not valid.");
1214         mlx5_flow_meter_stats_enable_update(fm, stats_mask);
1215         return 0;
1216 }
1217
1218 /**
1219  * Callback to read meter statistics.
1220  *
1221  * @param[in] dev
1222  *   Pointer to Ethernet device.
1223  * @param[in] meter_id
1224  *   Meter id.
1225  * @param[out] stats
1226  *   Pointer to store the statistics.
1227  * @param[out] stats_mask
1228  *   Pointer to store the stats_mask.
1229  * @param[in] clear
1230  *   Statistic to be cleared after read or not.
1231  * @param[out] error
1232  *   Pointer to rte meter error structure.
1233  *
1234  * @return
1235  *   0 on success, a negative errno value otherwise and rte_errno is set.
1236  */
1237 static int
1238 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1239                            uint32_t meter_id,
1240                            struct rte_mtr_stats *stats,
1241                            uint64_t *stats_mask,
1242                            int clear,
1243                            struct rte_mtr_error *error)
1244 {
1245         struct mlx5_priv *priv = dev->data->dev_private;
1246         struct mlx5_flow_meter_info *fm;
1247         uint64_t pkts;
1248         uint64_t bytes;
1249         int ret = 0;
1250
1251         if (!priv->mtr_en)
1252                 return -rte_mtr_error_set(error, ENOTSUP,
1253                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1254                                           "Meter is not supported");
1255         /* Meter object must exist. */
1256         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1257         if (fm == NULL)
1258                 return -rte_mtr_error_set(error, ENOENT,
1259                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1260                                           NULL, "Meter object id not valid.");
1261         *stats_mask = 0;
1262         if (fm->bytes_dropped)
1263                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1264         if (fm->pkts_dropped)
1265                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1266         memset(stats, 0, sizeof(*stats));
1267         if (fm->drop_cnt) {
1268                 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1269                                                  &bytes);
1270                 if (ret)
1271                         goto error;
1272                 /* If need to read the packets, set it. */
1273                 if (fm->pkts_dropped)
1274                         stats->n_pkts_dropped = pkts;
1275                 /* If need to read the bytes, set it. */
1276                 if (fm->bytes_dropped)
1277                         stats->n_bytes_dropped = bytes;
1278         }
1279         return 0;
1280 error:
1281         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1282                                  "Failed to read meter drop counters.");
1283 }
1284
1285 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1286         .capabilities_get = mlx5_flow_mtr_cap_get,
1287         .meter_profile_add = mlx5_flow_meter_profile_add,
1288         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1289         .meter_policy_validate = mlx5_flow_meter_policy_validate,
1290         .meter_policy_add = mlx5_flow_meter_policy_add,
1291         .meter_policy_delete = mlx5_flow_meter_policy_delete,
1292         .destroy = mlx5_flow_meter_destroy,
1293         .meter_enable = mlx5_flow_meter_enable,
1294         .meter_disable = mlx5_flow_meter_disable,
1295         .meter_profile_update = mlx5_flow_meter_profile_update,
1296         .meter_dscp_table_update = NULL,
1297         .stats_update = mlx5_flow_meter_stats_update,
1298         .stats_read = mlx5_flow_meter_stats_read,
1299 };
1300
1301 /**
1302  * Get meter operations.
1303  *
1304  * @param dev
1305  *   Pointer to Ethernet device structure.
1306  * @param arg
1307  *   Pointer to set the mtr operations.
1308  *
1309  * @return
1310  *   Always 0.
1311  */
1312 int
1313 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1314 {
1315         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1316         return 0;
1317 }
1318
1319 /**
1320  * Find meter by id.
1321  *
1322  * @param priv
1323  *   Pointer to mlx5_priv.
1324  * @param meter_id
1325  *   Meter id.
1326  * @param mtr_idx
1327  *   Pointer to Meter index.
1328  *
1329  * @return
1330  *   Pointer to the profile found on success, NULL otherwise.
1331  */
1332 struct mlx5_flow_meter_info *
1333 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1334                 uint32_t *mtr_idx)
1335 {
1336         struct mlx5_legacy_flow_meter *legacy_fm;
1337         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1338         struct mlx5_aso_mtr *aso_mtr;
1339         struct mlx5_aso_mtr_pools_mng *pools_mng =
1340                                 &priv->sh->mtrmng->pools_mng;
1341         union mlx5_l3t_data data;
1342
1343         if (priv->sh->meter_aso_en) {
1344                 rte_spinlock_lock(&pools_mng->mtrsl);
1345                 if (priv->mtr_idx_tbl) {
1346                         if (mlx5_l3t_get_entry(priv->mtr_idx_tbl,
1347                                 meter_id, &data) ||
1348                                 !data.dword) {
1349                                 rte_spinlock_unlock(&pools_mng->mtrsl);
1350                                 return NULL;
1351                         }
1352                         if (mtr_idx)
1353                                 *mtr_idx = data.dword;
1354                         aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1355                         /* Remove reference taken by the mlx5_l3t_get_entry. */
1356                         mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1357                 } else {
1358                         if (mtr_idx)
1359                                 *mtr_idx = meter_id;
1360                         aso_mtr = mlx5_aso_meter_by_idx(priv, meter_id);
1361                 }
1362                 rte_spinlock_unlock(&pools_mng->mtrsl);
1363                 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1364                         return NULL;
1365                 return &aso_mtr->fm;
1366         }
1367         TAILQ_FOREACH(legacy_fm, fms, next)
1368                 if (meter_id == legacy_fm->meter_id) {
1369                         if (mtr_idx)
1370                                 *mtr_idx = legacy_fm->idx;
1371                         return &legacy_fm->fm;
1372                 }
1373         return NULL;
1374 }
1375
1376 /**
1377  * Find meter by index.
1378  *
1379  * @param priv
1380  *   Pointer to mlx5_priv.
1381  * @param idx
1382  *   Meter index.
1383  *
1384  * @return
1385  *   Pointer to the profile found on success, NULL otherwise.
1386  */
1387 struct mlx5_flow_meter_info *
1388 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1389 {
1390         struct mlx5_aso_mtr *aso_mtr;
1391
1392         if (priv->sh->meter_aso_en) {
1393                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1394                 return &aso_mtr->fm;
1395         } else {
1396                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1397         }
1398 }
1399
1400 /**
1401  * Attach meter to flow.
1402  * Unidirectional Meter creation can only be done
1403  * when flow direction is known, i.e. when calling meter_attach.
1404  *
1405  * @param [in] priv
1406  *  Pointer to mlx5 private data.
1407  * @param[in] fm
1408  *   Pointer to flow meter.
1409  * @param [in] attr
1410  *  Pointer to flow attributes.
1411  * @param [out] error
1412  *  Pointer to error structure.
1413  *
1414  * @return
1415  *   0 on success, a negative errno value otherwise and rte_errno is set.
1416  */
1417 int
1418 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1419                        struct mlx5_flow_meter_info *fm,
1420                        const struct rte_flow_attr *attr,
1421                        struct rte_flow_error *error)
1422 {
1423         int ret = 0;
1424
1425         if (priv->sh->meter_aso_en) {
1426                 struct mlx5_aso_mtr *aso_mtr;
1427
1428                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1429                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1430                         return rte_flow_error_set(error, ENOENT,
1431                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1432                                         NULL,
1433                                         "Timeout in meter configuration");
1434                 }
1435                 rte_spinlock_lock(&fm->sl);
1436                 if (fm->shared || !fm->ref_cnt) {
1437                         fm->ref_cnt++;
1438                 } else {
1439                         rte_flow_error_set(error, EINVAL,
1440                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1441                                    "Meter cannot be shared");
1442                         ret = -1;
1443                 }
1444                 rte_spinlock_unlock(&fm->sl);
1445         } else {
1446                 rte_spinlock_lock(&fm->sl);
1447                 if (fm->meter_action) {
1448                         if (fm->shared &&
1449                             attr->transfer == fm->transfer &&
1450                             attr->ingress == fm->ingress &&
1451                             attr->egress == fm->egress) {
1452                                 fm->ref_cnt++;
1453                         } else {
1454                                 rte_flow_error_set(error, EINVAL,
1455                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1456                                         fm->shared ?
1457                                         "Meter attr not match." :
1458                                         "Meter cannot be shared.");
1459                                 ret = -1;
1460                         }
1461                 } else {
1462                         fm->ingress = attr->ingress;
1463                         fm->egress = attr->egress;
1464                         fm->transfer = attr->transfer;
1465                         fm->ref_cnt = 1;
1466                         /* This also creates the meter object. */
1467                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1468                                                                          fm);
1469                         if (!fm->meter_action) {
1470                                 fm->ref_cnt = 0;
1471                                 fm->ingress = 0;
1472                                 fm->egress = 0;
1473                                 fm->transfer = 0;
1474                                 rte_flow_error_set(error, EINVAL,
1475                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1476                                         "Meter action create failed.");
1477                                 ret = -1;
1478                         }
1479                 }
1480                 rte_spinlock_unlock(&fm->sl);
1481         }
1482         return ret ? -rte_errno : 0;
1483 }
1484
1485 /**
1486  * Detach meter from flow.
1487  *
1488  * @param [in] priv
1489  *  Pointer to mlx5 private data.
1490  * @param [in] fm
1491  *  Pointer to flow meter.
1492  */
1493 void
1494 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1495                        struct mlx5_flow_meter_info *fm)
1496 {
1497 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1498         rte_spinlock_lock(&fm->sl);
1499         MLX5_ASSERT(fm->ref_cnt);
1500         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1501                 mlx5_glue->destroy_flow_action(fm->meter_action);
1502                 fm->meter_action = NULL;
1503                 fm->ingress = 0;
1504                 fm->egress = 0;
1505                 fm->transfer = 0;
1506         }
1507         rte_spinlock_unlock(&fm->sl);
1508 #else
1509         (void)priv;
1510         (void)fm;
1511 #endif
1512 }
1513
1514 /**
1515  * Flush meter configuration.
1516  *
1517  * @param[in] dev
1518  *   Pointer to Ethernet device.
1519  * @param[out] error
1520  *   Pointer to rte meter error structure.
1521  *
1522  * @return
1523  *   0 on success, a negative errno value otherwise and rte_errno is set.
1524  */
1525 int
1526 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1527 {
1528         struct mlx5_priv *priv = dev->data->dev_private;
1529         struct mlx5_aso_mtr_pools_mng *pools_mng =
1530                                 &priv->sh->mtrmng->pools_mng;
1531         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1532         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
1533         struct mlx5_flow_meter_profile *fmp;
1534         struct mlx5_legacy_flow_meter *legacy_fm;
1535         struct mlx5_flow_meter_info *fm;
1536         struct mlx5_aso_mtr_pool *mtr_pool;
1537         struct mlx5_flow_meter_sub_policy *sub_policy;
1538         void *tmp;
1539         uint32_t i, offset, mtr_idx, policy_idx;
1540         void *entry;
1541
1542         if (!priv->mtr_en)
1543                 return 0;
1544         if (priv->sh->meter_aso_en) {
1545                 i = pools_mng->n_valid;
1546                 while (i--) {
1547                         mtr_pool = pools_mng->pools[i];
1548                         for (offset = 0; offset < MLX5_ASO_MTRS_PER_POOL;
1549                                 offset++) {
1550                                 fm = &mtr_pool->mtrs[offset].fm;
1551                                 mtr_idx = MLX5_MAKE_MTR_IDX(i, offset);
1552                                 (void)mlx5_flow_meter_params_flush(dev,
1553                                                 fm, mtr_idx);
1554                         }
1555                 }
1556         } else {
1557                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1558                         fm = &legacy_fm->fm;
1559                         if (mlx5_flow_meter_params_flush(dev, fm, 0))
1560                                 return -rte_mtr_error_set(error, EINVAL,
1561                                 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1562                                 NULL, "MTR object meter profile invalid.");
1563                 }
1564         }
1565         if (priv->sh->mtrmng->policy_idx_tbl) {
1566                 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1567                                         i, entry) {
1568                         policy_idx = *(uint32_t *)entry;
1569                         sub_policy = mlx5_ipool_get
1570                                 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1571                                 policy_idx);
1572                         if (!sub_policy)
1573                                 return -rte_mtr_error_set(error,
1574                                                 EINVAL,
1575                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1576                                                 NULL, "MTR object "
1577                                                 "meter policy invalid.");
1578                         if (__mlx5_flow_meter_policy_delete(dev, i,
1579                                                 sub_policy->main_policy,
1580                                                 error))
1581                                 return -rte_mtr_error_set(error,
1582                                                 EINVAL,
1583                                         RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1584                                                 NULL, "MTR object "
1585                                                 "meter policy invalid.");
1586                         mlx5_free(sub_policy->main_policy);
1587                 }
1588                 mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl);
1589                 priv->sh->mtrmng->policy_idx_tbl = NULL;
1590         }
1591         TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) {
1592                 /* Check unused. */
1593                 MLX5_ASSERT(!fmp->ref_cnt);
1594                 /* Remove from list. */
1595                 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
1596                 mlx5_free(fmp);
1597         }
1598         /* Delete default policy table. */
1599         mlx5_flow_destroy_def_policy(dev);
1600         mlx5_flow_destroy_mtr_drop_tbls(dev);
1601         return 0;
1602 }