drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41
  42         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  43         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  44         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  45         MLX5_SET(flow_meter_parameters, fmp,
  46                 start_color, MLX5_FLOW_COLOR_GREEN);
  47         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  48         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  49         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  50         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  51         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  52         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  53         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  54         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  55         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  56         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  58         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  60         mtr_init.next_table =
  61                 fm->transfer ? fm->mfts->transfer.tbl->obj :
  62                         fm->egress ? fm->mfts->egress.tbl->obj :
  63                                 fm->mfts->ingress.tbl->obj;
  64         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  65         mtr_init.flow_meter_parameter = fmp;
  66         mtr_init.flow_meter_parameter_sz =
  67                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  68         mtr_init.active = fm->active_state;
  69         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  70 #else
  71         (void)priv;
  72         (void)fm;
  73         return NULL;
  74 #endif
  75 }
  76
  77 /**
  78  * Find meter profile by id.
  79  *
  80  * @param priv
  81  *   Pointer to mlx5_priv.
  82  * @param meter_profile_id
  83  *   Meter profile id.
  84  *
  85  * @return
  86  *   Pointer to the profile found on success, NULL otherwise.
  87  */
  88 static struct mlx5_flow_meter_profile *
  89 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  90 {
  91         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
  92         struct mlx5_flow_meter_profile *fmp;
  93
  94         TAILQ_FOREACH(fmp, fmps, next)
  95                 if (meter_profile_id == fmp->id)
  96                         return fmp;
  97         return NULL;
  98 }
  99
 100 /**
 101  * Validate the MTR profile.
 102  *
 103  * @param[in] dev
 104  *   Pointer to Ethernet device.
 105  * @param[in] meter_profile_id
 106  *   Meter profile id.
 107  * @param[in] profile
 108  *   Pointer to meter profile detail.
 109  * @param[out] error
 110  *   Pointer to the error structure.
 111  *
 112  * @return
 113  *   0 on success, a negative errno value otherwise and rte_errno is set.
 114  */
 115 static int
 116 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 117                                  uint32_t meter_profile_id,
 118                                  struct rte_mtr_meter_profile *profile,
 119                                  struct rte_mtr_error *error)
 120 {
 121         struct mlx5_priv *priv = dev->data->dev_private;
 122         struct mlx5_flow_meter_profile *fmp;
 123
 124         /* Profile must not be NULL. */
 125         if (profile == NULL)
 126                 return -rte_mtr_error_set(error, EINVAL,
 127                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 128                                           NULL, "Meter profile is null.");
 129         /* Meter profile ID must be valid. */
 130         if (meter_profile_id == UINT32_MAX)
 131                 return -rte_mtr_error_set(error, EINVAL,
 132                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 133                                           NULL, "Meter profile id not valid.");
 134         /* Meter profile must not exist. */
 135         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 136         if (fmp)
 137                 return -rte_mtr_error_set(error, EEXIST,
 138                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 139                                           NULL,
 140                                           "Meter profile already exists.");
 141         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 142                 if (priv->config.hca_attr.qos.flow_meter_old) {
 143                         /* Verify support for flow meter parameters. */
 144                         if (profile->srtcm_rfc2697.cir > 0 &&
 145                             profile->srtcm_rfc2697.cir <= MLX5_SRTCM_CIR_MAX &&
 146                             profile->srtcm_rfc2697.cbs > 0 &&
 147                             profile->srtcm_rfc2697.cbs <= MLX5_SRTCM_CBS_MAX &&
 148                             profile->srtcm_rfc2697.ebs <= MLX5_SRTCM_EBS_MAX)
 149                                 return 0;
 150                         else
 151                                 return -rte_mtr_error_set
 152                                              (error, ENOTSUP,
 153                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 154                                               NULL,
 155                                               profile->srtcm_rfc2697.ebs ?
 156                                               "Metering value ebs must be 0." :
 157                                               "Invalid metering parameters.");
 158                 }
 159         }
 160         return -rte_mtr_error_set(error, ENOTSUP,
 161                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 162                                   NULL, "Metering algorithm not supported.");
 163 }
 164
 165 /**
 166  * Calculate mantissa and exponent for cir.
 167  *
 168  * @param[in] cir
 169  *   Value to be calculated.
 170  * @param[out] man
 171  *   Pointer to the mantissa.
 172  * @param[out] exp
 173  *   Pointer to the exp.
 174  */
 175 static void
 176 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 177 {
 178         int64_t _cir;
 179         int64_t delta = INT64_MAX;
 180         uint8_t _man = 0;
 181         uint8_t _exp = 0;
 182         uint64_t m, e;
 183
 184         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 185                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 186                         _cir = (1000000000ULL * m) >> e;
 187                         if (llabs(cir - _cir) <= delta) {
 188                                 delta = llabs(cir - _cir);
 189                                 _man = m;
 190                                 _exp = e;
 191                         }
 192                 }
 193         }
 194         *man = _man;
 195         *exp = _exp;
 196 }
 197
 198 /**
 199  * Calculate mantissa and exponent for xbs.
 200  *
 201  * @param[in] xbs
 202  *   Value to be calculated.
 203  * @param[out] man
 204  *   Pointer to the mantissa.
 205  * @param[out] exp
 206  *   Pointer to the exp.
 207  */
 208 static void
 209 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 210 {
 211         int _exp;
 212         double _man;
 213
 214         /* Special case xbs == 0 ? both exp and matissa are 0. */
 215         if (xbs == 0) {
 216                 *man = 0;
 217                 *exp = 0;
 218                 return;
 219         }
 220         /* xbs = xbs_mantissa * 2^xbs_exponent */
 221         _man = frexp(xbs, &_exp);
 222         _man = _man * pow(2, MLX5_MAN_WIDTH);
 223         _exp = _exp - MLX5_MAN_WIDTH;
 224         *man = (uint8_t)ceil(_man);
 225         *exp = _exp;
 226 }
 227
 228 /**
 229  * Fill the prm meter parameter.
 230  *
 231  * @param[in,out] fmp
 232  *   Pointer to meter profie to be converted.
 233  * @param[out] error
 234  *   Pointer to the error structure.
 235  *
 236  * @return
 237  *   0 on success, a negative errno value otherwise and rte_errno is set.
 238  */
 239 static int
 240 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 241                           struct rte_mtr_error *error)
 242 {
 243         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 244         uint8_t man, exp;
 245         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 246         uint32_t ebs_exp, ebs_man;
 247
 248         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 249                 return -rte_mtr_error_set(error, ENOTSUP,
 250                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 251                                 NULL, "Metering algorithm not supported.");
 252         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 253         mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir,
 254                                     &man, &exp);
 255         /* Check if cir mantissa is too large. */
 256         if (exp > ASO_DSEG_CIR_EXP_MASK)
 257                 return -rte_mtr_error_set(error, ENOTSUP,
 258                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 259                                           "meter profile parameter cir is"
 260                                           " not supported.");
 261         cir_man = man;
 262         cir_exp = exp;
 263          /* cbs = cbs_mantissa * 2^cbs_exponent */
 264         mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.cbs,
 265                                     &man, &exp);
 266         /* Check if cbs mantissa is too large. */
 267         if (exp > ASO_DSEG_EXP_MASK)
 268                 return -rte_mtr_error_set(error, ENOTSUP,
 269                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 270                                           "meter profile parameter cbs is"
 271                                           " not supported.");
 272         cbs_man = man;
 273         cbs_exp = exp;
 274         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 275                                 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 276                                 cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 277                                 cir_man);
 278         mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.ebs,
 279                                     &man, &exp);
 280         /* Check if ebs mantissa is too large. */
 281         if (exp > ASO_DSEG_EXP_MASK)
 282                 return -rte_mtr_error_set(error, ENOTSUP,
 283                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 284                                           "meter profile parameter ebs is"
 285                                           " not supported.");
 286         ebs_man = man;
 287         ebs_exp = exp;
 288         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 289                                         ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 290         return 0;
 291 }
 292
 293 /**
 294  * Callback to get MTR capabilities.
 295  *
 296  * @param[in] dev
 297  *   Pointer to Ethernet device.
 298  * @param[out] cap
 299  *   Pointer to save MTR capabilities.
 300  * @param[out] error
 301  *   Pointer to the error structure.
 302  *
 303  * @return
 304  *   0 on success, a negative errno value otherwise and rte_errno is set.
 305  */
 306 static int
 307 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 308                  struct rte_mtr_capabilities *cap,
 309                  struct rte_mtr_error *error __rte_unused)
 310 {
 311         struct mlx5_priv *priv = dev->data->dev_private;
 312         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 313
 314         if (!priv->mtr_en)
 315                 return -rte_mtr_error_set(error, ENOTSUP,
 316                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 317                                           "Meter is not supported");
 318         memset(cap, 0, sizeof(*cap));
 319         if (priv->sh->meter_aso_en)
 320             /* 2 meters per one ASO cache line. */
 321                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 322         else
 323                 cap->n_max = 1 << qattr->log_max_flow_meter;
 324         cap->n_shared_max = cap->n_max;
 325         cap->identical = 1;
 326         cap->shared_identical = 1;
 327         cap->shared_n_flows_per_mtr_max = 4 << 20;
 328         /* 2M flows can share the same meter. */
 329         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 330         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 331         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 332         cap->policer_action_drop_supported = 1;
 333         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 334                           RTE_MTR_STATS_N_PKTS_DROPPED;
 335         return 0;
 336 }
 337
 338 /**
 339  * Callback to add MTR profile.
 340  *
 341  * @param[in] dev
 342  *   Pointer to Ethernet device.
 343  * @param[in] meter_profile_id
 344  *   Meter profile id.
 345  * @param[in] profile
 346  *   Pointer to meter profile detail.
 347  * @param[out] error
 348  *   Pointer to the error structure.
 349  *
 350  * @return
 351  *   0 on success, a negative errno value otherwise and rte_errno is set.
 352  */
 353 static int
 354 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 355                        uint32_t meter_profile_id,
 356                        struct rte_mtr_meter_profile *profile,
 357                        struct rte_mtr_error *error)
 358 {
 359         struct mlx5_priv *priv = dev->data->dev_private;
 360         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
 361         struct mlx5_flow_meter_profile *fmp;
 362         int ret;
 363
 364         if (!priv->mtr_en)
 365                 return -rte_mtr_error_set(error, ENOTSUP,
 366                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 367                                           "Meter is not supported");
 368         /* Check input params. */
 369         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 370                                                profile, error);
 371         if (ret)
 372                 return ret;
 373         /* Meter profile memory allocation. */
 374         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 375                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 376         if (fmp == NULL)
 377                 return -rte_mtr_error_set(error, ENOMEM,
 378                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 379                                           NULL, "Meter profile memory "
 380                                           "alloc failed.");
 381         /* Fill profile info. */
 382         fmp->id = meter_profile_id;
 383         fmp->profile = *profile;
 384         /* Fill the flow meter parameters for the PRM. */
 385         ret = mlx5_flow_meter_param_fill(fmp, error);
 386         if (ret)
 387                 goto error;
 388         /* Add to list. */
 389         TAILQ_INSERT_TAIL(fmps, fmp, next);
 390         return 0;
 391 error:
 392         mlx5_free(fmp);
 393         return ret;
 394 }
 395
 396 /**
 397  * Callback to delete MTR profile.
 398  *
 399  * @param[in] dev
 400  *   Pointer to Ethernet device.
 401  * @param[in] meter_profile_id
 402  *   Meter profile id.
 403  * @param[out] error
 404  *   Pointer to the error structure.
 405  *
 406  * @return
 407  *   0 on success, a negative errno value otherwise and rte_errno is set.
 408  */
 409 static int
 410 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 411                           uint32_t meter_profile_id,
 412                           struct rte_mtr_error *error)
 413 {
 414         struct mlx5_priv *priv = dev->data->dev_private;
 415         struct mlx5_flow_meter_profile *fmp;
 416
 417         if (!priv->mtr_en)
 418                 return -rte_mtr_error_set(error, ENOTSUP,
 419                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 420                                           "Meter is not supported");
 421         /* Meter profile must exist. */
 422         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 423         if (fmp == NULL)
 424                 return -rte_mtr_error_set(error, ENOENT,
 425                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 426                                           &meter_profile_id,
 427                                           "Meter profile id is invalid.");
 428         /* Check profile is unused. */
 429         if (fmp->ref_cnt)
 430                 return -rte_mtr_error_set(error, EBUSY,
 431                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 432                                           NULL, "Meter profile is in use.");
 433         /* Remove from list. */
 434         TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
 435         mlx5_free(fmp);
 436         return 0;
 437 }
 438
 439 /**
 440  * Convert wrong color setting action to verbose error.
 441  *
 442  * @param[in] action
 443  *   Policy color action.
 444  *
 445  * @return
 446  *   Verbose meter color error type.
 447  */
 448 static inline enum rte_mtr_error_type
 449 action2error(enum rte_mtr_policer_action action)
 450 {
 451         switch (action) {
 452         case MTR_POLICER_ACTION_COLOR_GREEN:
 453                 return RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN;
 454         case MTR_POLICER_ACTION_COLOR_YELLOW:
 455                 return RTE_MTR_ERROR_TYPE_POLICER_ACTION_YELLOW;
 456         case MTR_POLICER_ACTION_COLOR_RED:
 457                 return RTE_MTR_ERROR_TYPE_POLICER_ACTION_RED;
 458         default:
 459                 break;
 460         }
 461         return RTE_MTR_ERROR_TYPE_UNSPECIFIED;
 462 }
 463
 464 /**
 465  * Check meter validation.
 466  *
 467  * @param[in] priv
 468  *   Pointer to mlx5 private data structure.
 469  * @param[in] meter_id
 470  *   Meter id.
 471  * @param[in] params
 472  *   Pointer to rte meter parameters.
 473  * @param[out] error
 474  *   Pointer to rte meter error structure.
 475  *
 476  * @return
 477  *   0 on success, a negative errno value otherwise and rte_errno is set.
 478  */
 479 static int
 480 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 481                          struct rte_mtr_params *params,
 482                          struct rte_mtr_error *error)
 483 {
 484         /* Meter must use global drop action. */
 485         if (!priv->sh->dr_drop_action)
 486                 return -rte_mtr_error_set(error, ENOTSUP,
 487                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 488                                           NULL,
 489                                           "No drop action ready for meter.");
 490         /* Meter params must not be NULL. */
 491         if (params == NULL)
 492                 return -rte_mtr_error_set(error, EINVAL,
 493                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 494                                           NULL, "Meter object params null.");
 495         /* Previous meter color is not supported. */
 496         if (params->use_prev_mtr_color)
 497                 return -rte_mtr_error_set(error, ENOTSUP,
 498                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 499                                           NULL,
 500                                           "Previous meter color "
 501                                           "not supported.");
 502         /* Validate policer settings. */
 503         if (params->action[RTE_COLOR_RED] != MTR_POLICER_ACTION_DROP)
 504                 return -rte_mtr_error_set
 505                                 (error, ENOTSUP,
 506                                  action2error(params->action[RTE_COLOR_RED]),
 507                                  NULL,
 508                                  "Red color only supports drop action.");
 509         if (params->action[RTE_COLOR_GREEN] != MTR_POLICER_ACTION_COLOR_GREEN)
 510                 return -rte_mtr_error_set
 511                                 (error, ENOTSUP,
 512                                  action2error(params->action[RTE_COLOR_GREEN]),
 513                                  NULL,
 514                                  "Green color only supports recolor green action.");
 515         /* Validate meter id. */
 516         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 517                 return -rte_mtr_error_set(error, EEXIST,
 518                                           RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 519                                           "Meter object already exists.");
 520         return 0;
 521 }
 522
 523 /**
 524  * Modify the flow meter action.
 525  *
 526  * @param[in] priv
 527  *   Pointer to mlx5 private data structure.
 528  * @param[in] fm
 529  *   Pointer to flow meter to be modified.
 530  * @param[in] srtcm
 531  *   Pointer to meter srtcm description parameter.
 532  * @param[in] modify_bits
 533  *   The bit in srtcm to be updated.
 534  * @param[in] active_state
 535  *   The state to be updated.
 536  * @return
 537  *   0 on success, o negative value otherwise.
 538  */
 539 static int
 540 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 541                 struct mlx5_flow_meter_info *fm,
 542                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 543                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 544 {
 545 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 546         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 547         uint32_t *attr;
 548         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 549         int ret;
 550         struct mlx5_aso_mtr *aso_mtr = NULL;
 551         uint32_t cbs_cir, ebs_eir, val;
 552
 553         if (priv->sh->meter_aso_en) {
 554                 fm->is_enable = !!is_enable;
 555                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 556                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 557                 if (ret)
 558                         return ret;
 559                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 560                 if (ret)
 561                         return ret;
 562         } else {
 563                 /* Fill command parameters. */
 564                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 565                 mod_attr.flow_meter_parameter = in;
 566                 mod_attr.flow_meter_parameter_sz =
 567                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 568                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 569                         mod_attr.active = !!active_state;
 570                 else
 571                         mod_attr.active = 0;
 572                 attr = in;
 573                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 574                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 575                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 576                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 577                                 ASO_DSEG_EXP_MASK;
 578                         MLX5_SET(flow_meter_parameters, attr,
 579                                 cbs_exponent, val);
 580                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 581                                 ASO_DSEG_MAN_MASK;
 582                         MLX5_SET(flow_meter_parameters, attr,
 583                                 cbs_mantissa, val);
 584                 }
 585                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
 586                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
 587                                 ASO_DSEG_EXP_MASK;
 588                         MLX5_SET(flow_meter_parameters, attr,
 589                                 cir_exponent, val);
 590                         val = cbs_cir & ASO_DSEG_MAN_MASK;
 591                         MLX5_SET(flow_meter_parameters, attr,
 592                                 cir_mantissa, val);
 593                 }
 594                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
 595                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
 596                                 ASO_DSEG_EXP_MASK;
 597                         MLX5_SET(flow_meter_parameters, attr,
 598                                 ebs_exponent, val);
 599                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
 600                                 ASO_DSEG_MAN_MASK;
 601                         MLX5_SET(flow_meter_parameters, attr,
 602                                 ebs_mantissa, val);
 603                 }
 604                 /* Apply modifications to meter only if it was created. */
 605                 if (fm->mfts->meter_action) {
 606                         ret = mlx5_glue->dv_modify_flow_action_meter
 607                                         (fm->mfts->meter_action, &mod_attr,
 608                                         rte_cpu_to_be_64(modify_bits));
 609                         if (ret)
 610                                 return ret;
 611                 }
 612                 /* Update succeedded modify meter parameters. */
 613                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 614                         fm->active_state = !!active_state;
 615         }
 616         return 0;
 617 #else
 618         (void)priv;
 619         (void)fm;
 620         (void)srtcm;
 621         (void)modify_bits;
 622         (void)active_state;
 623         return -ENOTSUP;
 624 #endif
 625 }
 626
 627 static void
 628 mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter_info *fm,
 629                                 uint64_t stats_mask)
 630 {
 631         fm->green_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_GREEN) ? 1 : 0;
 632         fm->green_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_GREEN) ? 1 : 0;
 633         fm->red_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_RED) ? 1 : 0;
 634         fm->red_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_RED) ? 1 : 0;
 635         fm->bytes_dropped =
 636                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
 637         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
 638 }
 639
 640 /**
 641  * Create meter rules.
 642  *
 643  * @param[in] dev
 644  *   Pointer to Ethernet device.
 645  * @param[in] meter_id
 646  *   Meter id.
 647  * @param[in] params
 648  *   Pointer to rte meter parameters.
 649  * @param[in] shared
 650  *   Meter shared with other flow or not.
 651  * @param[out] error
 652  *   Pointer to rte meter error structure.
 653  *
 654  * @return
 655  *   0 on success, a negative errno value otherwise and rte_errno is set.
 656  */
 657 static int
 658 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
 659                        struct rte_mtr_params *params, int shared,
 660                        struct rte_mtr_error *error)
 661 {
 662         struct mlx5_priv *priv = dev->data->dev_private;
 663         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
 664         struct mlx5_flow_meter_profile *fmp;
 665         struct mlx5_legacy_flow_meter *legacy_fm;
 666         struct mlx5_flow_meter_info *fm;
 667         const struct rte_flow_attr attr = {
 668                                 .ingress = 1,
 669                                 .egress = 1,
 670                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
 671                         };
 672         struct mlx5_indexed_pool_config flow_ipool_cfg = {
 673                 .size = 0,
 674                 .trunk_size = 64,
 675                 .need_lock = 1,
 676                 .type = "mlx5_flow_mtr_flow_id_pool",
 677         };
 678         struct mlx5_aso_mtr *aso_mtr;
 679         union mlx5_l3t_data data;
 680         uint32_t mtr_idx;
 681         int ret;
 682         uint8_t mtr_id_bits;
 683         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
 684                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
 685
 686         if (!priv->mtr_en)
 687                 return -rte_mtr_error_set(error, ENOTSUP,
 688                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 689                                           "Meter is not supported");
 690         /* Validate the parameters. */
 691         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
 692         if (ret)
 693                 return ret;
 694         /* Meter profile must exist. */
 695         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
 696         if (fmp == NULL)
 697                 return -rte_mtr_error_set(error, ENOENT,
 698                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 699                                           NULL, "Meter profile id not valid.");
 700         /* Allocate the flow meter memory. */
 701         if (priv->sh->meter_aso_en) {
 702                 mtr_idx = mlx5_flow_mtr_alloc(dev);
 703                 if (!mtr_idx)
 704                         return -rte_mtr_error_set(error, ENOMEM,
 705                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 706                                 "Memory alloc failed for meter.");
 707                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
 708                 fm = &aso_mtr->fm;
 709         } else {
 710                 legacy_fm = mlx5_ipool_zmalloc
 711                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
 712                 if (legacy_fm == NULL)
 713                         return -rte_mtr_error_set(error, ENOMEM,
 714                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 715                                 "Memory alloc failed for meter.");
 716                 legacy_fm->idx = mtr_idx;
 717                 fm = &legacy_fm->fm;
 718         }
 719         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
 720         if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) {
 721                 DRV_LOG(ERR, "Meter number exceeds max limit.");
 722                 goto error;
 723         }
 724         if (mtr_id_bits > priv->max_mtr_bits)
 725                 priv->max_mtr_bits = mtr_id_bits;
 726         /* Fill the flow meter parameters. */
 727         fm->meter_id = meter_id;
 728         fm->profile = fmp;
 729         memcpy(fm->action, params->action, sizeof(params->action));
 730         mlx5_flow_meter_stats_enable_update(fm, params->stats_mask);
 731         /* Alloc policer counters. */
 732         if (fm->green_bytes || fm->green_pkts) {
 733                 fm->policer_stats.pass_cnt = mlx5_counter_alloc(dev);
 734                 if (!fm->policer_stats.pass_cnt)
 735                         goto error;
 736         }
 737         if (fm->red_bytes || fm->red_pkts ||
 738             fm->bytes_dropped || fm->pkts_dropped) {
 739                 fm->policer_stats.drop_cnt = mlx5_counter_alloc(dev);
 740                 if (!fm->policer_stats.drop_cnt)
 741                         goto error;
 742         }
 743         fm->mfts = mlx5_flow_create_mtr_tbls(dev);
 744         if (!fm->mfts)
 745                 goto error;
 746         ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr);
 747         if (ret)
 748                 goto error;
 749         /* Add to the flow meter list. */
 750         if (!priv->sh->meter_aso_en)
 751                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
 752         fm->active_state = 1; /* Config meter starts as active. */
 753         fm->is_enable = 1;
 754         fm->shared = !!shared;
 755         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
 756         fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
 757         if (!fm->flow_ipool)
 758                 goto error;
 759         rte_spinlock_init(&fm->sl);
 760         /* If ASO meter supported, allocate ASO flow meter. */
 761         if (priv->sh->meter_aso_en) {
 762                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 763                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 764                 if (ret)
 765                         goto error;
 766                 data.dword = mtr_idx;
 767                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
 768                         goto error;
 769         }
 770         return 0;
 771 error:
 772         mlx5_flow_destroy_policer_rules(dev, fm, &attr);
 773         mlx5_flow_destroy_mtr_tbls(dev, fm->mfts);
 774         /* Free policer counters. */
 775         if (fm->policer_stats.pass_cnt)
 776                 mlx5_counter_free(dev, fm->policer_stats.pass_cnt);
 777         if (fm->policer_stats.drop_cnt)
 778                 mlx5_counter_free(dev, fm->policer_stats.drop_cnt);
 779         if (priv->sh->meter_aso_en)
 780                 mlx5_flow_mtr_free(dev, mtr_idx);
 781         else
 782                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
 783         return -rte_mtr_error_set(error, -ret,
 784                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 785                                   NULL, "Failed to create devx meter.");
 786 }
 787
 788 static int
 789 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
 790                         struct mlx5_flow_meter_info *fm,
 791                         const struct rte_flow_attr *attr,
 792                         uint32_t mtr_idx)
 793 {
 794         struct mlx5_priv *priv = dev->data->dev_private;
 795         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
 796         struct mlx5_flow_meter_profile *fmp;
 797         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
 798
 799         /* Meter object must not have any owner. */
 800         MLX5_ASSERT(!fm->ref_cnt);
 801         /* Get meter profile. */
 802         fmp = fm->profile;
 803         if (fmp == NULL)
 804                 return -1;
 805         /* Update dependencies. */
 806         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
 807         /* Remove from list. */
 808         if (!priv->sh->meter_aso_en) {
 809                 legacy_fm = container_of(fm, struct mlx5_legacy_flow_meter, fm);
 810                 TAILQ_REMOVE(fms, legacy_fm, next);
 811         }
 812         /* Free policer counters. */
 813         if (fm->policer_stats.pass_cnt)
 814                 mlx5_counter_free(dev, fm->policer_stats.pass_cnt);
 815         if (fm->policer_stats.drop_cnt)
 816                 mlx5_counter_free(dev, fm->policer_stats.drop_cnt);
 817         /* Free meter flow table. */
 818         if (fm->flow_ipool)
 819                 mlx5_ipool_destroy(fm->flow_ipool);
 820         mlx5_flow_destroy_policer_rules(dev, fm, attr);
 821         mlx5_flow_destroy_mtr_tbls(dev, fm->mfts);
 822         if (priv->sh->meter_aso_en)
 823                 mlx5_flow_mtr_free(dev, mtr_idx);
 824         else
 825                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
 826                                         legacy_fm->idx);
 827         return 0;
 828 }
 829
 830 /**
 831  * Destroy meter rules.
 832  *
 833  * @param[in] dev
 834  *   Pointer to Ethernet device.
 835  * @param[in] meter_id
 836  *   Meter id.
 837  * @param[out] error
 838  *   Pointer to rte meter error structure.
 839  *
 840  * @return
 841  *   0 on success, a negative errno value otherwise and rte_errno is set.
 842  */
 843 static int
 844 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
 845                         struct rte_mtr_error *error)
 846 {
 847         struct mlx5_priv *priv = dev->data->dev_private;
 848         struct mlx5_flow_meter_info *fm;
 849         const struct rte_flow_attr attr = {
 850                                 .ingress = 1,
 851                                 .egress = 1,
 852                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
 853                         };
 854         uint32_t mtr_idx = 0;
 855
 856         if (!priv->mtr_en)
 857                 return -rte_mtr_error_set(error, ENOTSUP,
 858                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 859                                           "Meter is not supported");
 860         /* Meter object must exist. */
 861         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
 862         if (fm == NULL)
 863                 return -rte_mtr_error_set(error, ENOENT,
 864                                           RTE_MTR_ERROR_TYPE_MTR_ID,
 865                                           NULL, "Meter object id not valid.");
 866         /* Meter object must not have any owner. */
 867         if (fm->ref_cnt > 0)
 868                 return -rte_mtr_error_set(error, EBUSY,
 869                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 870                                           NULL, "Meter object is being used.");
 871         if (priv->sh->meter_aso_en) {
 872                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id))
 873                         return -rte_mtr_error_set(error, EBUSY,
 874                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 875                                 "Fail to delete ASO Meter in index table.");
 876         }
 877         /* Destroy the meter profile. */
 878         if (mlx5_flow_meter_params_flush(dev, fm, &attr, mtr_idx))
 879                 return -rte_mtr_error_set(error, EINVAL,
 880                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 881                                         NULL, "MTR object meter profile invalid.");
 882         return 0;
 883 }
 884
 885 /**
 886  * Modify meter state.
 887  *
 888  * @param[in] priv
 889  *   Pointer to mlx5 private data structure.
 890  * @param[in] fm
 891  *   Pointer to flow meter.
 892  * @param[in] new_state
 893  *   New state to update.
 894  * @param[out] error
 895  *   Pointer to rte meter error structure.
 896  *
 897  * @return
 898  *   0 on success, a negative errno value otherwise and rte_errno is set.
 899  */
 900 static int
 901 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
 902                              struct mlx5_flow_meter_info *fm,
 903                              uint32_t new_state,
 904                              struct rte_mtr_error *error)
 905 {
 906         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
 907                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
 908                 .ebs_eir = 0,
 909         };
 910         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
 911                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
 912         int ret;
 913
 914         if (new_state == MLX5_FLOW_METER_DISABLE)
 915                 ret = mlx5_flow_meter_action_modify(priv, fm,
 916                                 &srtcm, modify_bits, 0, 0);
 917         else
 918                 ret = mlx5_flow_meter_action_modify(priv, fm,
 919                                                    &fm->profile->srtcm_prm,
 920                                                     modify_bits, 0, 1);
 921         if (ret)
 922                 return -rte_mtr_error_set(error, -ret,
 923                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 924                                           NULL,
 925                                           new_state ?
 926                                           "Failed to enable meter." :
 927                                           "Failed to disable meter.");
 928         return 0;
 929 }
 930
 931 /**
 932  * Callback to enable flow meter.
 933  *
 934  * @param[in] dev
 935  *   Pointer to Ethernet device.
 936  * @param[in] meter_id
 937  *   Meter id.
 938  * @param[out] error
 939  *   Pointer to rte meter error structure.
 940  *
 941  * @return
 942  *   0 on success, a negative errno value otherwise and rte_errno is set.
 943  */
 944 static int
 945 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
 946                        uint32_t meter_id,
 947                        struct rte_mtr_error *error)
 948 {
 949         struct mlx5_priv *priv = dev->data->dev_private;
 950         struct mlx5_flow_meter_info *fm;
 951         int ret;
 952
 953         if (!priv->mtr_en)
 954                 return -rte_mtr_error_set(error, ENOTSUP,
 955                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 956                                           "Meter is not supported");
 957         /* Meter object must exist. */
 958         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
 959         if (fm == NULL)
 960                 return -rte_mtr_error_set(error, ENOENT,
 961                                           RTE_MTR_ERROR_TYPE_MTR_ID,
 962                                           NULL, "Meter not found.");
 963         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
 964                 return 0;
 965         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
 966                                            error);
 967         if (!ret)
 968                 fm->active_state = MLX5_FLOW_METER_ENABLE;
 969         return ret;
 970 }
 971
 972 /**
 973  * Callback to disable flow meter.
 974  *
 975  * @param[in] dev
 976  *   Pointer to Ethernet device.
 977  * @param[in] meter_id
 978  *   Meter id.
 979  * @param[out] error
 980  *   Pointer to rte meter error structure.
 981  *
 982  * @return
 983  *   0 on success, a negative errno value otherwise and rte_errno is set.
 984  */
 985 static int
 986 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
 987                         uint32_t meter_id,
 988                         struct rte_mtr_error *error)
 989 {
 990         struct mlx5_priv *priv = dev->data->dev_private;
 991         struct mlx5_flow_meter_info *fm;
 992         int ret;
 993
 994         if (!priv->mtr_en)
 995                 return -rte_mtr_error_set(error, ENOTSUP,
 996                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 997                                           "Meter is not supported");
 998         /* Meter object must exist. */
 999         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1000         if (fm == NULL)
1001                 return -rte_mtr_error_set(error, ENOENT,
1002                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1003                                           NULL, "Meter not found.");
1004         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1005                 return 0;
1006         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1007                                            error);
1008         if (!ret)
1009                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1010         return ret;
1011 }
1012
1013 /**
1014  * Callback to update meter profile.
1015  *
1016  * @param[in] dev
1017  *   Pointer to Ethernet device.
1018  * @param[in] meter_id
1019  *   Meter id.
1020  * @param[in] meter_profile_id
1021  *   To be updated meter profile id.
1022  * @param[out] error
1023  *   Pointer to rte meter error structure.
1024  *
1025  * @return
1026  *   0 on success, a negative errno value otherwise and rte_errno is set.
1027  */
1028 static int
1029 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1030                                uint32_t meter_id,
1031                                uint32_t meter_profile_id,
1032                                struct rte_mtr_error *error)
1033 {
1034         struct mlx5_priv *priv = dev->data->dev_private;
1035         struct mlx5_flow_meter_profile *fmp;
1036         struct mlx5_flow_meter_profile *old_fmp;
1037         struct mlx5_flow_meter_info *fm;
1038         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1039                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1040         int ret;
1041
1042         if (!priv->mtr_en)
1043                 return -rte_mtr_error_set(error, ENOTSUP,
1044                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1045                                           "Meter is not supported");
1046         /* Meter profile must exist. */
1047         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1048         if (fmp == NULL)
1049                 return -rte_mtr_error_set(error, ENOENT,
1050                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1051                                           NULL, "Meter profile not found.");
1052         /* Meter object must exist. */
1053         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1054         if (fm == NULL)
1055                 return -rte_mtr_error_set(error, ENOENT,
1056                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1057                                           NULL, "Meter not found.");
1058         /* MTR object already set to meter profile id. */
1059         old_fmp = fm->profile;
1060         if (fmp == old_fmp)
1061                 return 0;
1062         /* Update the profile. */
1063         fm->profile = fmp;
1064         /* Update meter params in HW (if not disabled). */
1065         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1066                 return 0;
1067         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1068                                               modify_bits, fm->active_state, 1);
1069         if (ret) {
1070                 fm->profile = old_fmp;
1071                 return -rte_mtr_error_set(error, -ret,
1072                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1073                                           NULL, "Failed to update meter"
1074                                           " parmeters in hardware.");
1075         }
1076         old_fmp->ref_cnt--;
1077         fmp->ref_cnt++;
1078         return 0;
1079 }
1080
1081 /**
1082  * Callback to update meter stats mask.
1083  *
1084  * @param[in] dev
1085  *   Pointer to Ethernet device.
1086  * @param[in] meter_id
1087  *   Meter id.
1088  * @param[in] stats_mask
1089  *   To be updated stats_mask.
1090  * @param[out] error
1091  *   Pointer to rte meter error structure.
1092  *
1093  * @return
1094  *   0 on success, a negative errno value otherwise and rte_errno is set.
1095  */
1096 static int
1097 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1098                              uint32_t meter_id,
1099                              uint64_t stats_mask,
1100                              struct rte_mtr_error *error)
1101 {
1102         struct mlx5_priv *priv = dev->data->dev_private;
1103         struct mlx5_flow_meter_info *fm;
1104         const struct rte_flow_attr attr = {
1105                                 .ingress = 1,
1106                                 .egress = 1,
1107                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
1108                         };
1109         bool need_updated = false;
1110         struct mlx5_flow_policer_stats old_policer_stats;
1111
1112         if (!priv->mtr_en)
1113                 return -rte_mtr_error_set(error, ENOTSUP,
1114                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1115                                           "Meter is not supported");
1116         /* Meter object must exist. */
1117         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1118         if (fm == NULL)
1119                 return -rte_mtr_error_set(error, ENOENT,
1120                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1121                                           NULL, "Meter object id not valid.");
1122         old_policer_stats.pass_cnt = 0;
1123         old_policer_stats.drop_cnt = 0;
1124         if (!!((RTE_MTR_STATS_N_PKTS_GREEN |
1125                                 RTE_MTR_STATS_N_BYTES_GREEN) & stats_mask) !=
1126                 !!fm->policer_stats.pass_cnt) {
1127                 need_updated = true;
1128                 if (fm->policer_stats.pass_cnt) {
1129                         old_policer_stats.pass_cnt = fm->policer_stats.pass_cnt;
1130                         fm->policer_stats.pass_cnt = 0;
1131                 } else {
1132                         fm->policer_stats.pass_cnt =
1133                                 mlx5_counter_alloc(dev);
1134                         if (!fm->policer_stats.pass_cnt)
1135                                 return -rte_mtr_error_set(error, ENOMEM,
1136                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1137                                           "Counter alloc failed for meter.");
1138                 }
1139         }
1140         if (!!((RTE_MTR_STATS_N_PKTS_RED | RTE_MTR_STATS_N_BYTES_RED |
1141                 RTE_MTR_STATS_N_PKTS_DROPPED | RTE_MTR_STATS_N_BYTES_DROPPED) &
1142                 stats_mask) !=
1143                 !!fm->policer_stats.drop_cnt) {
1144                 need_updated = true;
1145                 if (fm->policer_stats.drop_cnt) {
1146                         old_policer_stats.drop_cnt = fm->policer_stats.drop_cnt;
1147                         fm->policer_stats.drop_cnt = 0;
1148                 } else {
1149                         fm->policer_stats.drop_cnt =
1150                                 mlx5_counter_alloc(dev);
1151                         if (!fm->policer_stats.drop_cnt)
1152                                 return -rte_mtr_error_set(error, ENOMEM,
1153                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1154                                           "Counter alloc failed for meter.");
1155                 }
1156         }
1157         if (need_updated) {
1158                 if (mlx5_flow_prepare_policer_rules(dev, fm, &attr)) {
1159                         if (fm->policer_stats.pass_cnt &&
1160                                 fm->policer_stats.pass_cnt !=
1161                                 old_policer_stats.pass_cnt)
1162                                 mlx5_counter_free(dev,
1163                                         fm->policer_stats.pass_cnt);
1164                         fm->policer_stats.pass_cnt =
1165                                         old_policer_stats.pass_cnt;
1166                         if (fm->policer_stats.drop_cnt &&
1167                                 fm->policer_stats.drop_cnt !=
1168                                 old_policer_stats.drop_cnt)
1169                                 mlx5_counter_free(dev,
1170                                         fm->policer_stats.drop_cnt);
1171                         fm->policer_stats.pass_cnt =
1172                                         old_policer_stats.pass_cnt;
1173                         return -rte_mtr_error_set(error, ENOTSUP,
1174                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1175                                 NULL, "Failed to create meter policer rules.");
1176                 }
1177                 /* Free old policer counters. */
1178                 if (old_policer_stats.pass_cnt)
1179                         mlx5_counter_free(dev,
1180                                 old_policer_stats.pass_cnt);
1181                 if (old_policer_stats.drop_cnt)
1182                         mlx5_counter_free(dev,
1183                                 old_policer_stats.drop_cnt);
1184         }
1185         mlx5_flow_meter_stats_enable_update(fm, stats_mask);
1186         return 0;
1187 }
1188
1189 /**
1190  * Callback to read meter statistics.
1191  *
1192  * @param[in] dev
1193  *   Pointer to Ethernet device.
1194  * @param[in] meter_id
1195  *   Meter id.
1196  * @param[out] stats
1197  *   Pointer to store the statistics.
1198  * @param[out] stats_mask
1199  *   Pointer to store the stats_mask.
1200  * @param[in] clear
1201  *   Statistic to be cleared after read or not.
1202  * @param[out] error
1203  *   Pointer to rte meter error structure.
1204  *
1205  * @return
1206  *   0 on success, a negative errno value otherwise and rte_errno is set.
1207  */
1208 static int
1209 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1210                            uint32_t meter_id,
1211                            struct rte_mtr_stats *stats,
1212                            uint64_t *stats_mask,
1213                            int clear,
1214                            struct rte_mtr_error *error)
1215 {
1216         struct mlx5_priv *priv = dev->data->dev_private;
1217         struct mlx5_flow_meter_info *fm;
1218         struct mlx5_flow_policer_stats *ps;
1219         uint64_t pkts;
1220         uint64_t bytes;
1221         int ret = 0;
1222
1223         if (!priv->mtr_en)
1224                 return -rte_mtr_error_set(error, ENOTSUP,
1225                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1226                                           "Meter is not supported");
1227         /* Meter object must exist. */
1228         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1229         if (fm == NULL)
1230                 return -rte_mtr_error_set(error, ENOENT,
1231                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1232                                           NULL, "Meter object id not valid.");
1233         ps = &fm->policer_stats;
1234         *stats_mask = 0;
1235         if (fm->green_bytes)
1236                 *stats_mask |= RTE_MTR_STATS_N_BYTES_GREEN;
1237         if (fm->green_pkts)
1238                 *stats_mask |= RTE_MTR_STATS_N_PKTS_GREEN;
1239         if (fm->red_bytes)
1240                 *stats_mask |= RTE_MTR_STATS_N_BYTES_RED;
1241         if (fm->red_pkts)
1242                 *stats_mask |= RTE_MTR_STATS_N_PKTS_RED;
1243         if (fm->bytes_dropped)
1244                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1245         if (fm->pkts_dropped)
1246                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1247         memset(stats, 0, sizeof(*stats));
1248         if (ps->pass_cnt) {
1249                 ret = mlx5_counter_query(dev, ps->pass_cnt, clear, &pkts,
1250                                                  &bytes);
1251                 if (ret)
1252                         goto error;
1253                 /* If need to read the packets, set it. */
1254                 if (fm->green_pkts)
1255                         stats->n_pkts[RTE_COLOR_GREEN] = pkts;
1256                 /* If need to read the bytes, set it. */
1257                 if (fm->green_bytes)
1258                         stats->n_bytes[RTE_COLOR_GREEN] = bytes;
1259         }
1260         if (ps->drop_cnt) {
1261                 ret = mlx5_counter_query(dev, ps->drop_cnt, clear, &pkts,
1262                                                  &bytes);
1263                 if (ret)
1264                         goto error;
1265                 /* If need to read the packets, set it. */
1266                 if (fm->pkts_dropped)
1267                         stats->n_pkts_dropped = pkts;
1268                 /* If need to read the bytes, set it. */
1269                 if (fm->bytes_dropped)
1270                         stats->n_bytes_dropped = bytes;
1271         }
1272         return 0;
1273 error:
1274         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1275                                  "Failed to read policer counters.");
1276 }
1277
1278 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1279         .capabilities_get = mlx5_flow_mtr_cap_get,
1280         .meter_profile_add = mlx5_flow_meter_profile_add,
1281         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1282         .create = mlx5_flow_meter_create,
1283         .destroy = mlx5_flow_meter_destroy,
1284         .meter_enable = mlx5_flow_meter_enable,
1285         .meter_disable = mlx5_flow_meter_disable,
1286         .meter_profile_update = mlx5_flow_meter_profile_update,
1287         .meter_dscp_table_update = NULL,
1288         .policer_actions_update = NULL,
1289         .stats_update = mlx5_flow_meter_stats_update,
1290         .stats_read = mlx5_flow_meter_stats_read,
1291 };
1292
1293 /**
1294  * Get meter operations.
1295  *
1296  * @param dev
1297  *   Pointer to Ethernet device structure.
1298  * @param arg
1299  *   Pointer to set the mtr operations.
1300  *
1301  * @return
1302  *   Always 0.
1303  */
1304 int
1305 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1306 {
1307         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1308         return 0;
1309 }
1310
1311 /**
1312  * Find meter by id.
1313  *
1314  * @param priv
1315  *   Pointer to mlx5_priv.
1316  * @param meter_id
1317  *   Meter id.
1318  * @param mtr_idx
1319  *   Pointer to Meter index.
1320  *
1321  * @return
1322  *   Pointer to the profile found on success, NULL otherwise.
1323  */
1324 struct mlx5_flow_meter_info *
1325 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1326                 uint32_t *mtr_idx)
1327 {
1328         struct mlx5_legacy_flow_meter *legacy_fm;
1329         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1330         struct mlx5_aso_mtr *aso_mtr;
1331         struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng;
1332         union mlx5_l3t_data data;
1333
1334         if (priv->sh->meter_aso_en) {
1335                 rte_spinlock_lock(&mtrmng->mtrsl);
1336                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1337                         !data.dword) {
1338                         rte_spinlock_unlock(&mtrmng->mtrsl);
1339                         return NULL;
1340                 }
1341                 if (mtr_idx)
1342                         *mtr_idx = data.dword;
1343                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1344                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1345                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1346                 MLX5_ASSERT(meter_id == aso_mtr->fm.meter_id);
1347                 rte_spinlock_unlock(&mtrmng->mtrsl);
1348                 return &aso_mtr->fm;
1349         }
1350         TAILQ_FOREACH(legacy_fm, fms, next)
1351                 if (meter_id == legacy_fm->fm.meter_id) {
1352                         if (mtr_idx)
1353                                 *mtr_idx = legacy_fm->idx;
1354                         return &legacy_fm->fm;
1355                 }
1356         return NULL;
1357 }
1358
1359 /**
1360  * Find meter by index.
1361  *
1362  * @param priv
1363  *   Pointer to mlx5_priv.
1364  * @param idx
1365  *   Meter index.
1366  *
1367  * @return
1368  *   Pointer to the profile found on success, NULL otherwise.
1369  */
1370 struct mlx5_flow_meter_info *
1371 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1372 {
1373         struct mlx5_aso_mtr *aso_mtr;
1374
1375         if (priv->sh->meter_aso_en) {
1376                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1377                 return &aso_mtr->fm;
1378         } else {
1379                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1380         }
1381 }
1382
1383 /**
1384  * Attach meter to flow.
1385  * Unidirectional Meter creation can only be done
1386  * when flow direction is known, i.e. when calling meter_attach.
1387  *
1388  * @param [in] priv
1389  *  Pointer to mlx5 private data.
1390  * @param[in] fm
1391  *   Pointer to flow meter.
1392  * @param [in] attr
1393  *  Pointer to flow attributes.
1394  * @param [out] error
1395  *  Pointer to error structure.
1396  *
1397  * @return
1398  *   0 on success, a negative errno value otherwise and rte_errno is set.
1399  */
1400 int
1401 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1402                        struct mlx5_flow_meter_info *fm,
1403                        const struct rte_flow_attr *attr,
1404                        struct rte_flow_error *error)
1405 {
1406         int ret = 0;
1407
1408         rte_spinlock_lock(&fm->sl);
1409         if (fm->mfts->meter_action) {
1410                 if (fm->shared &&
1411                     attr->transfer == fm->transfer &&
1412                     attr->ingress == fm->ingress &&
1413                     attr->egress == fm->egress)
1414                         fm->ref_cnt++;
1415                 else
1416                         ret = -1;
1417         } else {
1418                 fm->ingress = attr->ingress;
1419                 fm->egress = attr->egress;
1420                 fm->transfer = attr->transfer;
1421                  fm->ref_cnt = 1;
1422                 /* This also creates the meter object. */
1423                 fm->mfts->meter_action = mlx5_flow_meter_action_create(priv,
1424                                                                        fm);
1425                 if (!fm->mfts->meter_action) {
1426                         fm->ref_cnt = 0;
1427                         fm->ingress = 0;
1428                         fm->egress = 0;
1429                         fm->transfer = 0;
1430                         ret = -1;
1431                         DRV_LOG(ERR, "Meter action create failed.");
1432                 }
1433         }
1434         rte_spinlock_unlock(&fm->sl);
1435         if (ret)
1436                 rte_flow_error_set(error, EINVAL,
1437                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1438                                    fm->mfts->meter_action ?
1439                                    "Meter attr not match" :
1440                                    "Meter action create failed");
1441         return ret ? -rte_errno : 0;
1442 }
1443
1444 /**
1445  * Detach meter from flow.
1446  *
1447  * @param [in] fm
1448  *  Pointer to flow meter.
1449  */
1450 void
1451 mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm)
1452 {
1453 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1454         rte_spinlock_lock(&fm->sl);
1455         MLX5_ASSERT(fm->ref_cnt);
1456         if (--fm->ref_cnt == 0) {
1457                 mlx5_glue->destroy_flow_action(fm->mfts->meter_action);
1458                 fm->mfts->meter_action = NULL;
1459                 fm->ingress = 0;
1460                 fm->egress = 0;
1461                 fm->transfer = 0;
1462         }
1463         rte_spinlock_unlock(&fm->sl);
1464 #else
1465         (void)fm;
1466 #endif
1467 }
1468
1469 /**
1470  * Flush meter configuration.
1471  *
1472  * @param[in] dev
1473  *   Pointer to Ethernet device.
1474  * @param[out] error
1475  *   Pointer to rte meter error structure.
1476  *
1477  * @return
1478  *   0 on success, a negative errno value otherwise and rte_errno is set.
1479  */
1480 int
1481 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1482 {
1483         struct mlx5_priv *priv = dev->data->dev_private;
1484         struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng;
1485         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1486         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
1487         struct mlx5_flow_meter_profile *fmp;
1488         struct mlx5_legacy_flow_meter *legacy_fm;
1489         struct mlx5_flow_meter_info *fm;
1490         struct mlx5_aso_mtr_pool *mtr_pool;
1491         const struct rte_flow_attr attr = {
1492                                 .ingress = 1,
1493                                 .egress = 1,
1494                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
1495                         };
1496         void *tmp;
1497         uint32_t i, offset, mtr_idx;
1498
1499         if (priv->sh->meter_aso_en) {
1500                 i = mtrmng->n_valid;
1501                 while (i--) {
1502                         mtr_pool = mtrmng->pools[i];
1503                         for (offset = 0; offset < MLX5_ASO_MTRS_PER_POOL;
1504                                 offset++) {
1505                                 fm = &mtr_pool->mtrs[offset].fm;
1506                                 mtr_idx = MLX5_MAKE_MTR_IDX(i, offset);
1507                                 if (fm->meter_id != UINT32_MAX &&
1508                                         mlx5_flow_meter_params_flush(dev,
1509                                                 fm, &attr, mtr_idx))
1510                                         return -rte_mtr_error_set
1511                                         (error, EINVAL,
1512                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1513                                         NULL, "MTR object meter profile invalid.");
1514                         }
1515                 }
1516         } else {
1517                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1518                         fm = &legacy_fm->fm;
1519                         if (mlx5_flow_meter_params_flush(dev, fm, &attr, 0))
1520                                 return -rte_mtr_error_set(error, EINVAL,
1521                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1522                                         NULL, "MTR object meter profile invalid.");
1523                 }
1524         }
1525         TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) {
1526                 /* Check unused. */
1527                 MLX5_ASSERT(!fmp->ref_cnt);
1528                 /* Remove from list. */
1529                 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
1530                 mlx5_free(fmp);
1531         }
1532         return 0;
1533 }