drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41
  42         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  43         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  44         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  45         MLX5_SET(flow_meter_parameters, fmp,
  46                 start_color, MLX5_FLOW_COLOR_GREEN);
  47         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  48         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  49         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  50         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  51         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  52         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  53         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  54         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  55         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  56         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  58         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  60         mtr_init.next_table =
  61                 fm->transfer ? fm->mfts->transfer.tbl->obj :
  62                         fm->egress ? fm->mfts->egress.tbl->obj :
  63                                 fm->mfts->ingress.tbl->obj;
  64         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  65         mtr_init.flow_meter_parameter = fmp;
  66         mtr_init.flow_meter_parameter_sz =
  67                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  68         mtr_init.active = fm->active_state;
  69         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  70 #else
  71         (void)priv;
  72         (void)fm;
  73         return NULL;
  74 #endif
  75 }
  76
  77 /**
  78  * Find meter profile by id.
  79  *
  80  * @param priv
  81  *   Pointer to mlx5_priv.
  82  * @param meter_profile_id
  83  *   Meter profile id.
  84  *
  85  * @return
  86  *   Pointer to the profile found on success, NULL otherwise.
  87  */
  88 static struct mlx5_flow_meter_profile *
  89 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  90 {
  91         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
  92         struct mlx5_flow_meter_profile *fmp;
  93
  94         TAILQ_FOREACH(fmp, fmps, next)
  95                 if (meter_profile_id == fmp->id)
  96                         return fmp;
  97         return NULL;
  98 }
  99
 100 /**
 101  * Validate the MTR profile.
 102  *
 103  * @param[in] dev
 104  *   Pointer to Ethernet device.
 105  * @param[in] meter_profile_id
 106  *   Meter profile id.
 107  * @param[in] profile
 108  *   Pointer to meter profile detail.
 109  * @param[out] error
 110  *   Pointer to the error structure.
 111  *
 112  * @return
 113  *   0 on success, a negative errno value otherwise and rte_errno is set.
 114  */
 115 static int
 116 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 117                                  uint32_t meter_profile_id,
 118                                  struct rte_mtr_meter_profile *profile,
 119                                  struct rte_mtr_error *error)
 120 {
 121         struct mlx5_priv *priv = dev->data->dev_private;
 122         struct mlx5_flow_meter_profile *fmp;
 123
 124         /* Profile must not be NULL. */
 125         if (profile == NULL)
 126                 return -rte_mtr_error_set(error, EINVAL,
 127                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 128                                           NULL, "Meter profile is null.");
 129         /* Meter profile ID must be valid. */
 130         if (meter_profile_id == UINT32_MAX)
 131                 return -rte_mtr_error_set(error, EINVAL,
 132                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 133                                           NULL, "Meter profile id not valid.");
 134         /* Meter profile must not exist. */
 135         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 136         if (fmp)
 137                 return -rte_mtr_error_set(error, EEXIST,
 138                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 139                                           NULL,
 140                                           "Meter profile already exists.");
 141         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 142                 if (priv->config.hca_attr.qos.flow_meter_old) {
 143                         /* Verify support for flow meter parameters. */
 144                         if (profile->srtcm_rfc2697.cir > 0 &&
 145                             profile->srtcm_rfc2697.cir <= MLX5_SRTCM_CIR_MAX &&
 146                             profile->srtcm_rfc2697.cbs > 0 &&
 147                             profile->srtcm_rfc2697.cbs <= MLX5_SRTCM_CBS_MAX &&
 148                             profile->srtcm_rfc2697.ebs <= MLX5_SRTCM_EBS_MAX)
 149                                 return 0;
 150                         else
 151                                 return -rte_mtr_error_set
 152                                              (error, ENOTSUP,
 153                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 154                                               NULL,
 155                                               profile->srtcm_rfc2697.ebs ?
 156                                               "Metering value ebs must be 0." :
 157                                               "Invalid metering parameters.");
 158                 }
 159         }
 160         return -rte_mtr_error_set(error, ENOTSUP,
 161                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 162                                   NULL, "Metering algorithm not supported.");
 163 }
 164
 165 /**
 166  * Calculate mantissa and exponent for cir.
 167  *
 168  * @param[in] cir
 169  *   Value to be calculated.
 170  * @param[out] man
 171  *   Pointer to the mantissa.
 172  * @param[out] exp
 173  *   Pointer to the exp.
 174  */
 175 static void
 176 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 177 {
 178         int64_t _cir;
 179         int64_t delta = INT64_MAX;
 180         uint8_t _man = 0;
 181         uint8_t _exp = 0;
 182         uint64_t m, e;
 183
 184         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 185                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 186                         _cir = (1000000000ULL * m) >> e;
 187                         if (llabs(cir - _cir) <= delta) {
 188                                 delta = llabs(cir - _cir);
 189                                 _man = m;
 190                                 _exp = e;
 191                         }
 192                 }
 193         }
 194         *man = _man;
 195         *exp = _exp;
 196 }
 197
 198 /**
 199  * Calculate mantissa and exponent for xbs.
 200  *
 201  * @param[in] xbs
 202  *   Value to be calculated.
 203  * @param[out] man
 204  *   Pointer to the mantissa.
 205  * @param[out] exp
 206  *   Pointer to the exp.
 207  */
 208 static void
 209 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 210 {
 211         int _exp;
 212         double _man;
 213
 214         /* Special case xbs == 0 ? both exp and matissa are 0. */
 215         if (xbs == 0) {
 216                 *man = 0;
 217                 *exp = 0;
 218                 return;
 219         }
 220         /* xbs = xbs_mantissa * 2^xbs_exponent */
 221         _man = frexp(xbs, &_exp);
 222         _man = _man * pow(2, MLX5_MAN_WIDTH);
 223         _exp = _exp - MLX5_MAN_WIDTH;
 224         *man = (uint8_t)ceil(_man);
 225         *exp = _exp;
 226 }
 227
 228 /**
 229  * Fill the prm meter parameter.
 230  *
 231  * @param[in,out] fmp
 232  *   Pointer to meter profie to be converted.
 233  * @param[out] error
 234  *   Pointer to the error structure.
 235  *
 236  * @return
 237  *   0 on success, a negative errno value otherwise and rte_errno is set.
 238  */
 239 static int
 240 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 241                           struct rte_mtr_error *error)
 242 {
 243         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 244         uint8_t man, exp;
 245         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 246         uint32_t ebs_exp, ebs_man;
 247
 248         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 249                 return -rte_mtr_error_set(error, ENOTSUP,
 250                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 251                                 NULL, "Metering algorithm not supported.");
 252         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 253         mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir,
 254                                     &man, &exp);
 255         /* Check if cir mantissa is too large. */
 256         if (exp > ASO_DSEG_CIR_EXP_MASK)
 257                 return -rte_mtr_error_set(error, ENOTSUP,
 258                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 259                                           "meter profile parameter cir is"
 260                                           " not supported.");
 261         cir_man = man;
 262         cir_exp = exp;
 263          /* cbs = cbs_mantissa * 2^cbs_exponent */
 264         mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.cbs,
 265                                     &man, &exp);
 266         /* Check if cbs mantissa is too large. */
 267         if (exp > ASO_DSEG_EXP_MASK)
 268                 return -rte_mtr_error_set(error, ENOTSUP,
 269                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 270                                           "meter profile parameter cbs is"
 271                                           " not supported.");
 272         cbs_man = man;
 273         cbs_exp = exp;
 274         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 275                                 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 276                                 cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 277                                 cir_man);
 278         mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.ebs,
 279                                     &man, &exp);
 280         /* Check if ebs mantissa is too large. */
 281         if (exp > ASO_DSEG_EXP_MASK)
 282                 return -rte_mtr_error_set(error, ENOTSUP,
 283                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 284                                           "meter profile parameter ebs is"
 285                                           " not supported.");
 286         ebs_man = man;
 287         ebs_exp = exp;
 288         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 289                                         ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 290         return 0;
 291 }
 292
 293 /**
 294  * Callback to get MTR capabilities.
 295  *
 296  * @param[in] dev
 297  *   Pointer to Ethernet device.
 298  * @param[out] cap
 299  *   Pointer to save MTR capabilities.
 300  * @param[out] error
 301  *   Pointer to the error structure.
 302  *
 303  * @return
 304  *   0 on success, a negative errno value otherwise and rte_errno is set.
 305  */
 306 static int
 307 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 308                  struct rte_mtr_capabilities *cap,
 309                  struct rte_mtr_error *error __rte_unused)
 310 {
 311         struct mlx5_priv *priv = dev->data->dev_private;
 312         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 313
 314         if (!priv->mtr_en)
 315                 return -rte_mtr_error_set(error, ENOTSUP,
 316                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 317                                           "Meter is not supported");
 318         memset(cap, 0, sizeof(*cap));
 319         if (priv->sh->meter_aso_en)
 320             /* 2 meters per one ASO cache line. */
 321                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 322         else
 323                 cap->n_max = 1 << qattr->log_max_flow_meter;
 324         cap->n_shared_max = cap->n_max;
 325         cap->identical = 1;
 326         cap->shared_identical = 1;
 327         cap->shared_n_flows_per_mtr_max = 4 << 20;
 328         /* 2M flows can share the same meter. */
 329         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 330         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 331         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 332         cap->policer_action_drop_supported = 1;
 333         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 334                           RTE_MTR_STATS_N_PKTS_DROPPED;
 335         return 0;
 336 }
 337
 338 /**
 339  * Callback to add MTR profile.
 340  *
 341  * @param[in] dev
 342  *   Pointer to Ethernet device.
 343  * @param[in] meter_profile_id
 344  *   Meter profile id.
 345  * @param[in] profile
 346  *   Pointer to meter profile detail.
 347  * @param[out] error
 348  *   Pointer to the error structure.
 349  *
 350  * @return
 351  *   0 on success, a negative errno value otherwise and rte_errno is set.
 352  */
 353 static int
 354 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 355                        uint32_t meter_profile_id,
 356                        struct rte_mtr_meter_profile *profile,
 357                        struct rte_mtr_error *error)
 358 {
 359         struct mlx5_priv *priv = dev->data->dev_private;
 360         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
 361         struct mlx5_flow_meter_profile *fmp;
 362         int ret;
 363
 364         if (!priv->mtr_en)
 365                 return -rte_mtr_error_set(error, ENOTSUP,
 366                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 367                                           "Meter is not supported");
 368         /* Check input params. */
 369         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 370                                                profile, error);
 371         if (ret)
 372                 return ret;
 373         /* Meter profile memory allocation. */
 374         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 375                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 376         if (fmp == NULL)
 377                 return -rte_mtr_error_set(error, ENOMEM,
 378                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 379                                           NULL, "Meter profile memory "
 380                                           "alloc failed.");
 381         /* Fill profile info. */
 382         fmp->id = meter_profile_id;
 383         fmp->profile = *profile;
 384         /* Fill the flow meter parameters for the PRM. */
 385         ret = mlx5_flow_meter_param_fill(fmp, error);
 386         if (ret)
 387                 goto error;
 388         /* Add to list. */
 389         TAILQ_INSERT_TAIL(fmps, fmp, next);
 390         return 0;
 391 error:
 392         mlx5_free(fmp);
 393         return ret;
 394 }
 395
 396 /**
 397  * Callback to delete MTR profile.
 398  *
 399  * @param[in] dev
 400  *   Pointer to Ethernet device.
 401  * @param[in] meter_profile_id
 402  *   Meter profile id.
 403  * @param[out] error
 404  *   Pointer to the error structure.
 405  *
 406  * @return
 407  *   0 on success, a negative errno value otherwise and rte_errno is set.
 408  */
 409 static int
 410 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 411                           uint32_t meter_profile_id,
 412                           struct rte_mtr_error *error)
 413 {
 414         struct mlx5_priv *priv = dev->data->dev_private;
 415         struct mlx5_flow_meter_profile *fmp;
 416
 417         if (!priv->mtr_en)
 418                 return -rte_mtr_error_set(error, ENOTSUP,
 419                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 420                                           "Meter is not supported");
 421         /* Meter profile must exist. */
 422         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 423         if (fmp == NULL)
 424                 return -rte_mtr_error_set(error, ENOENT,
 425                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 426                                           &meter_profile_id,
 427                                           "Meter profile id is invalid.");
 428         /* Check profile is unused. */
 429         if (fmp->ref_cnt)
 430                 return -rte_mtr_error_set(error, EBUSY,
 431                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 432                                           NULL, "Meter profile is in use.");
 433         /* Remove from list. */
 434         TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
 435         mlx5_free(fmp);
 436         return 0;
 437 }
 438
 439 /**
 440  * Convert wrong color setting action to verbose error.
 441  *
 442  * @param[in] action
 443  *   Policy color action.
 444  *
 445  * @return
 446  *   Verbose meter color error type.
 447  */
 448 static inline enum rte_mtr_error_type
 449 action2error(enum rte_mtr_policer_action action)
 450 {
 451         switch (action) {
 452         case MTR_POLICER_ACTION_COLOR_GREEN:
 453                 return RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN;
 454         case MTR_POLICER_ACTION_COLOR_YELLOW:
 455                 return RTE_MTR_ERROR_TYPE_POLICER_ACTION_YELLOW;
 456         case MTR_POLICER_ACTION_COLOR_RED:
 457                 return RTE_MTR_ERROR_TYPE_POLICER_ACTION_RED;
 458         default:
 459                 break;
 460         }
 461         return RTE_MTR_ERROR_TYPE_UNSPECIFIED;
 462 }
 463
 464 /**
 465  * Check meter validation.
 466  *
 467  * @param[in] priv
 468  *   Pointer to mlx5 private data structure.
 469  * @param[in] meter_id
 470  *   Meter id.
 471  * @param[in] params
 472  *   Pointer to rte meter parameters.
 473  * @param[out] error
 474  *   Pointer to rte meter error structure.
 475  *
 476  * @return
 477  *   0 on success, a negative errno value otherwise and rte_errno is set.
 478  */
 479 static int
 480 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 481                          struct rte_mtr_params *params,
 482                          struct rte_mtr_error *error)
 483 {
 484         /* Meter must use global drop action. */
 485         if (!priv->sh->dr_drop_action)
 486                 return -rte_mtr_error_set(error, ENOTSUP,
 487                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 488                                           NULL,
 489                                           "No drop action ready for meter.");
 490         /* Meter params must not be NULL. */
 491         if (params == NULL)
 492                 return -rte_mtr_error_set(error, EINVAL,
 493                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 494                                           NULL, "Meter object params null.");
 495         /* Previous meter color is not supported. */
 496         if (params->use_prev_mtr_color)
 497                 return -rte_mtr_error_set(error, ENOTSUP,
 498                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 499                                           NULL,
 500                                           "Previous meter color "
 501                                           "not supported.");
 502         /* Validate policer settings. */
 503         if (params->action[RTE_COLOR_RED] != MTR_POLICER_ACTION_DROP)
 504                 return -rte_mtr_error_set
 505                                 (error, ENOTSUP,
 506                                  action2error(params->action[RTE_COLOR_RED]),
 507                                  NULL,
 508                                  "Red color only supports drop action.");
 509         if (params->action[RTE_COLOR_GREEN] != MTR_POLICER_ACTION_COLOR_GREEN)
 510                 return -rte_mtr_error_set
 511                                 (error, ENOTSUP,
 512                                  action2error(params->action[RTE_COLOR_GREEN]),
 513                                  NULL,
 514                                  "Green color only supports recolor green action.");
 515         /* Validate meter id. */
 516         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 517                 return -rte_mtr_error_set(error, EEXIST,
 518                                           RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 519                                           "Meter object already exists.");
 520         return 0;
 521 }
 522
 523 /**
 524  * Modify the flow meter action.
 525  *
 526  * @param[in] priv
 527  *   Pointer to mlx5 private data structure.
 528  * @param[in] fm
 529  *   Pointer to flow meter to be modified.
 530  * @param[in] srtcm
 531  *   Pointer to meter srtcm description parameter.
 532  * @param[in] modify_bits
 533  *   The bit in srtcm to be updated.
 534  * @param[in] active_state
 535  *   The state to be updated.
 536  * @return
 537  *   0 on success, o negative value otherwise.
 538  */
 539 static int
 540 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 541                 struct mlx5_flow_meter_info *fm,
 542                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 543                 uint64_t modify_bits, uint32_t active_state)
 544 {
 545 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 546         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 547         uint32_t *attr;
 548         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 549         int ret;
 550         uint32_t cbs_cir, ebs_eir, val;
 551
 552         /* Fill command parameters. */
 553         mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 554         mod_attr.flow_meter_parameter = in;
 555         mod_attr.flow_meter_parameter_sz =
 556                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 557         if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 558                 mod_attr.active = !!active_state;
 559         else
 560                 mod_attr.active = 0;
 561         attr = in;
 562         cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 563         ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 564         if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 565                 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
 566                 MLX5_SET(flow_meter_parameters, attr, cbs_exponent, val);
 567                 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
 568                 MLX5_SET(flow_meter_parameters, attr, cbs_mantissa, val);
 569         }
 570         if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
 571                 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
 572                 MLX5_SET(flow_meter_parameters, attr, cir_exponent, val);
 573                 val = cbs_cir & ASO_DSEG_MAN_MASK;
 574                 MLX5_SET(flow_meter_parameters, attr, cir_mantissa, val);
 575         }
 576         if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
 577                 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
 578                 MLX5_SET(flow_meter_parameters, attr, ebs_exponent, val);
 579                 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
 580                 MLX5_SET(flow_meter_parameters, attr, ebs_mantissa, val);
 581         }
 582         /* Apply modifications to meter only if it was created. */
 583         if (fm->mfts->meter_action) {
 584                 ret = mlx5_glue->dv_modify_flow_action_meter
 585                                         (fm->mfts->meter_action, &mod_attr,
 586                                         rte_cpu_to_be_64(modify_bits));
 587                 if (ret)
 588                         return ret;
 589         }
 590         /* Update succeedded modify meter parameters. */
 591         if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 592                 fm->active_state = !!active_state;
 593         return 0;
 594 #else
 595         (void)priv;
 596         (void)fm;
 597         (void)srtcm;
 598         (void)modify_bits;
 599         (void)active_state;
 600         return -ENOTSUP;
 601 #endif
 602 }
 603
 604 static void
 605 mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter_info *fm,
 606                                 uint64_t stats_mask)
 607 {
 608         fm->green_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_GREEN) ? 1 : 0;
 609         fm->green_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_GREEN) ? 1 : 0;
 610         fm->red_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_RED) ? 1 : 0;
 611         fm->red_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_RED) ? 1 : 0;
 612         fm->bytes_dropped =
 613                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
 614         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
 615 }
 616
 617 /**
 618  * Create meter rules.
 619  *
 620  * @param[in] dev
 621  *   Pointer to Ethernet device.
 622  * @param[in] meter_id
 623  *   Meter id.
 624  * @param[in] params
 625  *   Pointer to rte meter parameters.
 626  * @param[in] shared
 627  *   Meter shared with other flow or not.
 628  * @param[out] error
 629  *   Pointer to rte meter error structure.
 630  *
 631  * @return
 632  *   0 on success, a negative errno value otherwise and rte_errno is set.
 633  */
 634 static int
 635 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
 636                        struct rte_mtr_params *params, int shared,
 637                        struct rte_mtr_error *error)
 638 {
 639         struct mlx5_priv *priv = dev->data->dev_private;
 640         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
 641         struct mlx5_flow_meter_profile *fmp;
 642         struct mlx5_legacy_flow_meter *legacy_fm;
 643         struct mlx5_flow_meter_info *fm;
 644         const struct rte_flow_attr attr = {
 645                                 .ingress = 1,
 646                                 .egress = 1,
 647                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
 648                         };
 649         struct mlx5_indexed_pool_config flow_ipool_cfg = {
 650                 .size = 0,
 651                 .trunk_size = 64,
 652                 .need_lock = 1,
 653                 .type = "mlx5_flow_mtr_flow_id_pool",
 654         };
 655         struct mlx5_aso_mtr *aso_mtr;
 656         uint32_t mtr_idx;
 657         int ret;
 658         uint8_t mtr_id_bits;
 659         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
 660                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
 661
 662         if (!priv->mtr_en)
 663                 return -rte_mtr_error_set(error, ENOTSUP,
 664                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 665                                           "Meter is not supported");
 666         /* Validate the parameters. */
 667         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
 668         if (ret)
 669                 return ret;
 670         /* Meter profile must exist. */
 671         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
 672         if (fmp == NULL)
 673                 return -rte_mtr_error_set(error, ENOENT,
 674                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 675                                           NULL, "Meter profile id not valid.");
 676         /* Allocate the flow meter memory. */
 677         if (priv->sh->meter_aso_en) {
 678                 mtr_idx = mlx5_flow_mtr_alloc(dev);
 679                 if (!mtr_idx)
 680                         return -rte_mtr_error_set(error, ENOMEM,
 681                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 682                                 "Memory alloc failed for meter.");
 683                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
 684                 fm = &aso_mtr->fm;
 685         } else {
 686                 legacy_fm = mlx5_ipool_zmalloc
 687                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
 688                 if (legacy_fm == NULL)
 689                         return -rte_mtr_error_set(error, ENOMEM,
 690                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 691                                 "Memory alloc failed for meter.");
 692                 legacy_fm->idx = mtr_idx;
 693                 fm = &legacy_fm->fm;
 694         }
 695         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
 696         if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) {
 697                 DRV_LOG(ERR, "Meter number exceeds max limit.");
 698                 goto error;
 699         }
 700         if (mtr_id_bits > priv->max_mtr_bits)
 701                 priv->max_mtr_bits = mtr_id_bits;
 702         /* Fill the flow meter parameters. */
 703         fm->meter_id = meter_id;
 704         fm->profile = fmp;
 705         memcpy(fm->action, params->action, sizeof(params->action));
 706         mlx5_flow_meter_stats_enable_update(fm, params->stats_mask);
 707
 708         /* Alloc policer counters. */
 709         if (fm->green_bytes || fm->green_pkts) {
 710                 fm->policer_stats.pass_cnt = mlx5_counter_alloc(dev);
 711                 if (!fm->policer_stats.pass_cnt)
 712                         goto error;
 713         }
 714         if (fm->red_bytes || fm->red_pkts ||
 715             fm->bytes_dropped || fm->pkts_dropped) {
 716                 fm->policer_stats.drop_cnt = mlx5_counter_alloc(dev);
 717                 if (!fm->policer_stats.drop_cnt)
 718                         goto error;
 719         }
 720         fm->mfts = mlx5_flow_create_mtr_tbls(dev);
 721         if (!fm->mfts)
 722                 goto error;
 723         ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr);
 724         if (ret)
 725                 goto error;
 726         /* Add to the flow meter list. */
 727         if (!priv->sh->meter_aso_en)
 728                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
 729         fm->active_state = 1; /* Config meter starts as active. */
 730         fm->shared = !!shared;
 731         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
 732         fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
 733         if (!fm->flow_ipool)
 734                 goto error;
 735         rte_spinlock_init(&fm->sl);
 736         return 0;
 737 error:
 738         mlx5_flow_destroy_policer_rules(dev, fm, &attr);
 739         mlx5_flow_destroy_mtr_tbls(dev, fm->mfts);
 740         /* Free policer counters. */
 741         if (fm->policer_stats.pass_cnt)
 742                 mlx5_counter_free(dev, fm->policer_stats.pass_cnt);
 743         if (fm->policer_stats.drop_cnt)
 744                 mlx5_counter_free(dev, fm->policer_stats.drop_cnt);
 745         if (priv->sh->meter_aso_en)
 746                 mlx5_flow_mtr_free(dev, mtr_idx);
 747         else
 748                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
 749         return -rte_mtr_error_set(error, -ret,
 750                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 751                                   NULL, "Failed to create devx meter.");
 752 }
 753
 754 static int
 755 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
 756                         struct mlx5_flow_meter_info *fm,
 757                         const struct rte_flow_attr *attr,
 758                         uint32_t mtr_idx)
 759 {
 760         struct mlx5_priv *priv = dev->data->dev_private;
 761         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
 762         struct mlx5_flow_meter_profile *fmp;
 763         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
 764
 765         /* Meter object must not have any owner. */
 766         MLX5_ASSERT(!fm->ref_cnt);
 767         /* Get meter profile. */
 768         fmp = fm->profile;
 769         if (fmp == NULL)
 770                 return -1;
 771         /* Update dependencies. */
 772         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
 773         /* Remove from list. */
 774         if (!priv->sh->meter_aso_en) {
 775                 legacy_fm = container_of(fm, struct mlx5_legacy_flow_meter, fm);
 776                 TAILQ_REMOVE(fms, legacy_fm, next);
 777         }
 778         /* Free policer counters. */
 779         if (fm->policer_stats.pass_cnt)
 780                 mlx5_counter_free(dev, fm->policer_stats.pass_cnt);
 781         if (fm->policer_stats.drop_cnt)
 782                 mlx5_counter_free(dev, fm->policer_stats.drop_cnt);
 783         /* Free meter flow table. */
 784         if (fm->flow_ipool)
 785                 mlx5_ipool_destroy(fm->flow_ipool);
 786         mlx5_flow_destroy_policer_rules(dev, fm, attr);
 787         mlx5_flow_destroy_mtr_tbls(dev, fm->mfts);
 788         if (priv->sh->meter_aso_en)
 789                 mlx5_flow_mtr_free(dev, mtr_idx);
 790         else
 791                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
 792                                         legacy_fm->idx);
 793         return 0;
 794 }
 795
 796 /**
 797  * Destroy meter rules.
 798  *
 799  * @param[in] dev
 800  *   Pointer to Ethernet device.
 801  * @param[in] meter_id
 802  *   Meter id.
 803  * @param[out] error
 804  *   Pointer to rte meter error structure.
 805  *
 806  * @return
 807  *   0 on success, a negative errno value otherwise and rte_errno is set.
 808  */
 809 static int
 810 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
 811                         struct rte_mtr_error *error)
 812 {
 813         struct mlx5_priv *priv = dev->data->dev_private;
 814         struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng;
 815         struct mlx5_flow_meter_info *fm;
 816         const struct rte_flow_attr attr = {
 817                                 .ingress = 1,
 818                                 .egress = 1,
 819                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
 820                         };
 821         uint32_t mtr_idx = 0;
 822
 823         if (!priv->mtr_en)
 824                 return -rte_mtr_error_set(error, ENOTSUP,
 825                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 826                                           "Meter is not supported");
 827         /* Meter object must exist. */
 828         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
 829         if (fm == NULL)
 830                 return -rte_mtr_error_set(error, ENOENT,
 831                                           RTE_MTR_ERROR_TYPE_MTR_ID,
 832                                           NULL, "Meter object id not valid.");
 833         /* Meter object must not have any owner. */
 834         if (fm->ref_cnt > 0)
 835                 return -rte_mtr_error_set(error, EBUSY,
 836                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 837                                           NULL, "Meter object is being used.");
 838         if (priv->sh->meter_aso_en) {
 839                 if (mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id))
 840                         return -rte_mtr_error_set(error, EBUSY,
 841                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 842                                 "Fail to delete ASO Meter in index table.");
 843         }
 844         /* Destroy the meter profile. */
 845         if (mlx5_flow_meter_params_flush(dev, fm, &attr, mtr_idx))
 846                 return -rte_mtr_error_set(error, EINVAL,
 847                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 848                                         NULL, "MTR object meter profile invalid.");
 849         return 0;
 850 }
 851
 852 /**
 853  * Modify meter state.
 854  *
 855  * @param[in] priv
 856  *   Pointer to mlx5 private data structure.
 857  * @param[in] fm
 858  *   Pointer to flow meter.
 859  * @param[in] new_state
 860  *   New state to update.
 861  * @param[out] error
 862  *   Pointer to rte meter error structure.
 863  *
 864  * @return
 865  *   0 on success, a negative errno value otherwise and rte_errno is set.
 866  */
 867 static int
 868 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
 869                              struct mlx5_flow_meter_info *fm,
 870                              uint32_t new_state,
 871                              struct rte_mtr_error *error)
 872 {
 873         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
 874                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
 875                 .ebs_eir = 0,
 876         };
 877         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
 878                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
 879         int ret;
 880
 881         if (new_state == MLX5_FLOW_METER_DISABLE)
 882                 ret = mlx5_flow_meter_action_modify(priv, fm, &srtcm,
 883                                                     modify_bits, 0);
 884         else
 885                 ret = mlx5_flow_meter_action_modify(priv, fm,
 886                                                    &fm->profile->srtcm_prm,
 887                                                     modify_bits, 0);
 888         if (ret)
 889                 return -rte_mtr_error_set(error, -ret,
 890                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 891                                           NULL,
 892                                           new_state ?
 893                                           "Failed to enable meter." :
 894                                           "Failed to disable meter.");
 895         return 0;
 896 }
 897
 898 /**
 899  * Callback to enable flow meter.
 900  *
 901  * @param[in] dev
 902  *   Pointer to Ethernet device.
 903  * @param[in] meter_id
 904  *   Meter id.
 905  * @param[out] error
 906  *   Pointer to rte meter error structure.
 907  *
 908  * @return
 909  *   0 on success, a negative errno value otherwise and rte_errno is set.
 910  */
 911 static int
 912 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
 913                        uint32_t meter_id,
 914                        struct rte_mtr_error *error)
 915 {
 916         struct mlx5_priv *priv = dev->data->dev_private;
 917         struct mlx5_flow_meter_info *fm;
 918         int ret;
 919
 920         if (!priv->mtr_en)
 921                 return -rte_mtr_error_set(error, ENOTSUP,
 922                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 923                                           "Meter is not supported");
 924         /* Meter object must exist. */
 925         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
 926         if (fm == NULL)
 927                 return -rte_mtr_error_set(error, ENOENT,
 928                                           RTE_MTR_ERROR_TYPE_MTR_ID,
 929                                           NULL, "Meter not found.");
 930         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
 931                 return 0;
 932         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
 933                                            error);
 934         if (!ret)
 935                 fm->active_state = MLX5_FLOW_METER_ENABLE;
 936         return ret;
 937 }
 938
 939 /**
 940  * Callback to disable flow meter.
 941  *
 942  * @param[in] dev
 943  *   Pointer to Ethernet device.
 944  * @param[in] meter_id
 945  *   Meter id.
 946  * @param[out] error
 947  *   Pointer to rte meter error structure.
 948  *
 949  * @return
 950  *   0 on success, a negative errno value otherwise and rte_errno is set.
 951  */
 952 static int
 953 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
 954                         uint32_t meter_id,
 955                         struct rte_mtr_error *error)
 956 {
 957         struct mlx5_priv *priv = dev->data->dev_private;
 958         struct mlx5_flow_meter_info *fm;
 959         int ret;
 960
 961         if (!priv->mtr_en)
 962                 return -rte_mtr_error_set(error, ENOTSUP,
 963                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 964                                           "Meter is not supported");
 965         /* Meter object must exist. */
 966         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
 967         if (fm == NULL)
 968                 return -rte_mtr_error_set(error, ENOENT,
 969                                           RTE_MTR_ERROR_TYPE_MTR_ID,
 970                                           NULL, "Meter not found.");
 971         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
 972                 return 0;
 973         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
 974                                            error);
 975         if (!ret)
 976                 fm->active_state = MLX5_FLOW_METER_DISABLE;
 977         return ret;
 978 }
 979
 980 /**
 981  * Callback to update meter profile.
 982  *
 983  * @param[in] dev
 984  *   Pointer to Ethernet device.
 985  * @param[in] meter_id
 986  *   Meter id.
 987  * @param[in] meter_profile_id
 988  *   To be updated meter profile id.
 989  * @param[out] error
 990  *   Pointer to rte meter error structure.
 991  *
 992  * @return
 993  *   0 on success, a negative errno value otherwise and rte_errno is set.
 994  */
 995 static int
 996 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
 997                                uint32_t meter_id,
 998                                uint32_t meter_profile_id,
 999                                struct rte_mtr_error *error)
1000 {
1001         struct mlx5_priv *priv = dev->data->dev_private;
1002         struct mlx5_flow_meter_profile *fmp;
1003         struct mlx5_flow_meter_profile *old_fmp;
1004         struct mlx5_flow_meter_info *fm;
1005         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1006                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1007         int ret;
1008
1009         if (!priv->mtr_en)
1010                 return -rte_mtr_error_set(error, ENOTSUP,
1011                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1012                                           "Meter is not supported");
1013         /* Meter profile must exist. */
1014         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1015         if (fmp == NULL)
1016                 return -rte_mtr_error_set(error, ENOENT,
1017                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1018                                           NULL, "Meter profile not found.");
1019         /* Meter object must exist. */
1020         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1021         if (fm == NULL)
1022                 return -rte_mtr_error_set(error, ENOENT,
1023                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1024                                           NULL, "Meter not found.");
1025         /* MTR object already set to meter profile id. */
1026         old_fmp = fm->profile;
1027         if (fmp == old_fmp)
1028                 return 0;
1029         /* Update the profile. */
1030         fm->profile = fmp;
1031         /* Update meter params in HW (if not disabled). */
1032         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1033                 return 0;
1034         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1035                                               modify_bits, fm->active_state);
1036         if (ret) {
1037                 fm->profile = old_fmp;
1038                 return -rte_mtr_error_set(error, -ret,
1039                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1040                                           NULL, "Failed to update meter"
1041                                           " parmeters in hardware.");
1042         }
1043         old_fmp->ref_cnt--;
1044         fmp->ref_cnt++;
1045         return 0;
1046 }
1047
1048 /**
1049  * Callback to update meter stats mask.
1050  *
1051  * @param[in] dev
1052  *   Pointer to Ethernet device.
1053  * @param[in] meter_id
1054  *   Meter id.
1055  * @param[in] stats_mask
1056  *   To be updated stats_mask.
1057  * @param[out] error
1058  *   Pointer to rte meter error structure.
1059  *
1060  * @return
1061  *   0 on success, a negative errno value otherwise and rte_errno is set.
1062  */
1063 static int
1064 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1065                              uint32_t meter_id,
1066                              uint64_t stats_mask,
1067                              struct rte_mtr_error *error)
1068 {
1069         struct mlx5_priv *priv = dev->data->dev_private;
1070         struct mlx5_flow_meter_info *fm;
1071         const struct rte_flow_attr attr = {
1072                                 .ingress = 1,
1073                                 .egress = 1,
1074                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
1075                         };
1076         bool need_updated = false;
1077         struct mlx5_flow_policer_stats old_policer_stats;
1078
1079         if (!priv->mtr_en)
1080                 return -rte_mtr_error_set(error, ENOTSUP,
1081                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1082                                           "Meter is not supported");
1083         /* Meter object must exist. */
1084         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1085         if (fm == NULL)
1086                 return -rte_mtr_error_set(error, ENOENT,
1087                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1088                                           NULL, "Meter object id not valid.");
1089         old_policer_stats.pass_cnt = 0;
1090         old_policer_stats.drop_cnt = 0;
1091         if (!!((RTE_MTR_STATS_N_PKTS_GREEN |
1092                                 RTE_MTR_STATS_N_BYTES_GREEN) & stats_mask) !=
1093                 !!fm->policer_stats.pass_cnt) {
1094                 need_updated = true;
1095                 if (fm->policer_stats.pass_cnt) {
1096                         old_policer_stats.pass_cnt = fm->policer_stats.pass_cnt;
1097                         fm->policer_stats.pass_cnt = 0;
1098                 } else {
1099                         fm->policer_stats.pass_cnt =
1100                                 mlx5_counter_alloc(dev);
1101                         if (!fm->policer_stats.pass_cnt)
1102                                 return -rte_mtr_error_set(error, ENOMEM,
1103                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1104                                           "Counter alloc failed for meter.");
1105                 }
1106         }
1107         if (!!((RTE_MTR_STATS_N_PKTS_RED | RTE_MTR_STATS_N_BYTES_RED |
1108                 RTE_MTR_STATS_N_PKTS_DROPPED | RTE_MTR_STATS_N_BYTES_DROPPED) &
1109                 stats_mask) !=
1110                 !!fm->policer_stats.drop_cnt) {
1111                 need_updated = true;
1112                 if (fm->policer_stats.drop_cnt) {
1113                         old_policer_stats.drop_cnt = fm->policer_stats.drop_cnt;
1114                         fm->policer_stats.drop_cnt = 0;
1115                 } else {
1116                         fm->policer_stats.drop_cnt =
1117                                 mlx5_counter_alloc(dev);
1118                         if (!fm->policer_stats.drop_cnt)
1119                                 return -rte_mtr_error_set(error, ENOMEM,
1120                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1121                                           "Counter alloc failed for meter.");
1122                 }
1123         }
1124         if (need_updated) {
1125                 if (mlx5_flow_prepare_policer_rules(dev, fm, &attr)) {
1126                         if (fm->policer_stats.pass_cnt &&
1127                                 fm->policer_stats.pass_cnt !=
1128                                 old_policer_stats.pass_cnt)
1129                                 mlx5_counter_free(dev,
1130                                         fm->policer_stats.pass_cnt);
1131                         fm->policer_stats.pass_cnt =
1132                                         old_policer_stats.pass_cnt;
1133                         if (fm->policer_stats.drop_cnt &&
1134                                 fm->policer_stats.drop_cnt !=
1135                                 old_policer_stats.drop_cnt)
1136                                 mlx5_counter_free(dev,
1137                                         fm->policer_stats.drop_cnt);
1138                         fm->policer_stats.pass_cnt =
1139                                         old_policer_stats.pass_cnt;
1140                         return -rte_mtr_error_set(error, ENOTSUP,
1141                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1142                                 NULL, "Failed to create meter policer rules.");
1143                 }
1144                 /* Free old policer counters. */
1145                 if (old_policer_stats.pass_cnt)
1146                         mlx5_counter_free(dev,
1147                                 old_policer_stats.pass_cnt);
1148                 if (old_policer_stats.drop_cnt)
1149                         mlx5_counter_free(dev,
1150                                 old_policer_stats.drop_cnt);
1151         }
1152         mlx5_flow_meter_stats_enable_update(fm, stats_mask);
1153         return 0;
1154 }
1155
1156 /**
1157  * Callback to read meter statistics.
1158  *
1159  * @param[in] dev
1160  *   Pointer to Ethernet device.
1161  * @param[in] meter_id
1162  *   Meter id.
1163  * @param[out] stats
1164  *   Pointer to store the statistics.
1165  * @param[out] stats_mask
1166  *   Pointer to store the stats_mask.
1167  * @param[in] clear
1168  *   Statistic to be cleared after read or not.
1169  * @param[out] error
1170  *   Pointer to rte meter error structure.
1171  *
1172  * @return
1173  *   0 on success, a negative errno value otherwise and rte_errno is set.
1174  */
1175 static int
1176 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1177                            uint32_t meter_id,
1178                            struct rte_mtr_stats *stats,
1179                            uint64_t *stats_mask,
1180                            int clear,
1181                            struct rte_mtr_error *error)
1182 {
1183         struct mlx5_priv *priv = dev->data->dev_private;
1184         struct mlx5_flow_meter_info *fm;
1185         struct mlx5_flow_policer_stats *ps;
1186         uint64_t pkts;
1187         uint64_t bytes;
1188         int ret = 0;
1189
1190         if (!priv->mtr_en)
1191                 return -rte_mtr_error_set(error, ENOTSUP,
1192                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1193                                           "Meter is not supported");
1194         /* Meter object must exist. */
1195         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1196         if (fm == NULL)
1197                 return -rte_mtr_error_set(error, ENOENT,
1198                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1199                                           NULL, "Meter object id not valid.");
1200         ps = &fm->policer_stats;
1201         *stats_mask = 0;
1202         if (fm->green_bytes)
1203                 *stats_mask |= RTE_MTR_STATS_N_BYTES_GREEN;
1204         if (fm->green_pkts)
1205                 *stats_mask |= RTE_MTR_STATS_N_PKTS_GREEN;
1206         if (fm->red_bytes)
1207                 *stats_mask |= RTE_MTR_STATS_N_BYTES_RED;
1208         if (fm->red_pkts)
1209                 *stats_mask |= RTE_MTR_STATS_N_PKTS_RED;
1210         if (fm->bytes_dropped)
1211                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1212         if (fm->pkts_dropped)
1213                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1214         memset(stats, 0, sizeof(*stats));
1215         if (ps->pass_cnt) {
1216                 ret = mlx5_counter_query(dev, ps->pass_cnt, clear, &pkts,
1217                                                  &bytes);
1218                 if (ret)
1219                         goto error;
1220                 /* If need to read the packets, set it. */
1221                 if (fm->green_pkts)
1222                         stats->n_pkts[RTE_COLOR_GREEN] = pkts;
1223                 /* If need to read the bytes, set it. */
1224                 if (fm->green_bytes)
1225                         stats->n_bytes[RTE_COLOR_GREEN] = bytes;
1226         }
1227         if (ps->drop_cnt) {
1228                 ret = mlx5_counter_query(dev, ps->drop_cnt, clear, &pkts,
1229                                                  &bytes);
1230                 if (ret)
1231                         goto error;
1232                 /* If need to read the packets, set it. */
1233                 if (fm->pkts_dropped)
1234                         stats->n_pkts_dropped = pkts;
1235                 /* If need to read the bytes, set it. */
1236                 if (fm->bytes_dropped)
1237                         stats->n_bytes_dropped = bytes;
1238         }
1239         return 0;
1240 error:
1241         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1242                                  "Failed to read policer counters.");
1243 }
1244
1245 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1246         .capabilities_get = mlx5_flow_mtr_cap_get,
1247         .meter_profile_add = mlx5_flow_meter_profile_add,
1248         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1249         .create = mlx5_flow_meter_create,
1250         .destroy = mlx5_flow_meter_destroy,
1251         .meter_enable = mlx5_flow_meter_enable,
1252         .meter_disable = mlx5_flow_meter_disable,
1253         .meter_profile_update = mlx5_flow_meter_profile_update,
1254         .meter_dscp_table_update = NULL,
1255         .policer_actions_update = NULL,
1256         .stats_update = mlx5_flow_meter_stats_update,
1257         .stats_read = mlx5_flow_meter_stats_read,
1258 };
1259
1260 /**
1261  * Get meter operations.
1262  *
1263  * @param dev
1264  *   Pointer to Ethernet device structure.
1265  * @param arg
1266  *   Pointer to set the mtr operations.
1267  *
1268  * @return
1269  *   Always 0.
1270  */
1271 int
1272 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1273 {
1274         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1275         return 0;
1276 }
1277
1278 /**
1279  * Find meter by id.
1280  *
1281  * @param priv
1282  *   Pointer to mlx5_priv.
1283  * @param meter_id
1284  *   Meter id.
1285  *
1286  * @return
1287  *   Pointer to the profile found on success, NULL otherwise.
1288  */
1289 struct mlx5_flow_meter_info *
1290 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1291                 uint32_t *mtr_idx)
1292 {
1293         struct mlx5_legacy_flow_meter *legacy_fm;
1294         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1295         struct mlx5_aso_mtr *aso_mtr;
1296         struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng;
1297         union mlx5_l3t_data data;
1298
1299         if (priv->sh->meter_aso_en) {
1300                 rte_spinlock_lock(&mtrmng->mtrsl);
1301                 if (!mtrmng->n_valid) {
1302                         rte_spinlock_unlock(&mtrmng->mtrsl);
1303                         return NULL;
1304                 }
1305                 if (mlx5_l3t_get_entry(mtrmng->mtr_idx_tbl, meter_id, &data) ||
1306                         !data.dword) {
1307                         rte_spinlock_unlock(&mtrmng->mtrsl);
1308                         return NULL;
1309                 }
1310                 if (mtr_idx)
1311                         *mtr_idx = data.dword;
1312                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1313                 mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id);
1314                 if (meter_id == aso_mtr->fm.meter_id) {
1315                         rte_spinlock_unlock(&mtrmng->mtrsl);
1316                         return &aso_mtr->fm;
1317                 }
1318                 rte_spinlock_unlock(&mtrmng->mtrsl);
1319         } else {
1320                 TAILQ_FOREACH(legacy_fm, fms, next)
1321                         if (meter_id == legacy_fm->fm.meter_id)
1322                                 return &legacy_fm->fm;
1323         }
1324         return NULL;
1325 }
1326
1327 /**
1328  * Find meter by index.
1329  *
1330  * @param priv
1331  *   Pointer to mlx5_priv.
1332  * @param idx
1333  *   Meter index.
1334  *
1335  * @return
1336  *   Pointer to the profile found on success, NULL otherwise.
1337  */
1338 struct mlx5_flow_meter_info *
1339 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1340 {
1341         struct mlx5_aso_mtr *aso_mtr;
1342
1343         if (priv->sh->meter_aso_en) {
1344                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1345                 return &aso_mtr->fm;
1346         } else {
1347                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1348         }
1349 }
1350
1351 /**
1352  * Attach meter to flow.
1353  * Unidirectional Meter creation can only be done
1354  * when flow direction is known, i.e. when calling meter_attach.
1355  *
1356  * @param [in] priv
1357  *  Pointer to mlx5 private data.
1358  * @param[in] fm
1359  *   Pointer to flow meter.
1360  * @param [in] attr
1361  *  Pointer to flow attributes.
1362  * @param [out] error
1363  *  Pointer to error structure.
1364  *
1365  * @return
1366  *   0 on success, a negative errno value otherwise and rte_errno is set.
1367  */
1368 int
1369 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1370                        struct mlx5_flow_meter_info *fm,
1371                        const struct rte_flow_attr *attr,
1372                        struct rte_flow_error *error)
1373 {
1374         int ret = 0;
1375
1376         rte_spinlock_lock(&fm->sl);
1377         if (fm->mfts->meter_action) {
1378                 if (fm->shared &&
1379                     attr->transfer == fm->transfer &&
1380                     attr->ingress == fm->ingress &&
1381                     attr->egress == fm->egress)
1382                         fm->ref_cnt++;
1383                 else
1384                         ret = -1;
1385         } else {
1386                 fm->ingress = attr->ingress;
1387                 fm->egress = attr->egress;
1388                 fm->transfer = attr->transfer;
1389                  fm->ref_cnt = 1;
1390                 /* This also creates the meter object. */
1391                 fm->mfts->meter_action = mlx5_flow_meter_action_create(priv,
1392                                                                        fm);
1393                 if (!fm->mfts->meter_action) {
1394                         fm->ref_cnt = 0;
1395                         fm->ingress = 0;
1396                         fm->egress = 0;
1397                         fm->transfer = 0;
1398                         ret = -1;
1399                         DRV_LOG(ERR, "Meter action create failed.");
1400                 }
1401         }
1402         rte_spinlock_unlock(&fm->sl);
1403         if (ret)
1404                 rte_flow_error_set(error, EINVAL,
1405                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1406                                    fm->mfts->meter_action ?
1407                                    "Meter attr not match" :
1408                                    "Meter action create failed");
1409         return ret ? -rte_errno : 0;
1410 }
1411
1412 /**
1413  * Detach meter from flow.
1414  *
1415  * @param [in] fm
1416  *  Pointer to flow meter.
1417  */
1418 void
1419 mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm)
1420 {
1421 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1422         rte_spinlock_lock(&fm->sl);
1423         MLX5_ASSERT(fm->ref_cnt);
1424         if (--fm->ref_cnt == 0) {
1425                 mlx5_glue->destroy_flow_action(fm->mfts->meter_action);
1426                 fm->mfts->meter_action = NULL;
1427                 fm->ingress = 0;
1428                 fm->egress = 0;
1429                 fm->transfer = 0;
1430         }
1431         rte_spinlock_unlock(&fm->sl);
1432 #else
1433         (void)fm;
1434 #endif
1435 }
1436
1437 /**
1438  * Flush meter configuration.
1439  *
1440  * @param[in] dev
1441  *   Pointer to Ethernet device.
1442  * @param[out] error
1443  *   Pointer to rte meter error structure.
1444  *
1445  * @return
1446  *   0 on success, a negative errno value otherwise and rte_errno is set.
1447  */
1448 int
1449 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1450 {
1451         struct mlx5_priv *priv = dev->data->dev_private;
1452         struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng;
1453         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1454         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
1455         struct mlx5_flow_meter_profile *fmp;
1456         struct mlx5_legacy_flow_meter *legacy_fm;
1457         struct mlx5_flow_meter_info *fm;
1458         struct mlx5_aso_mtr_pool *mtr_pool;
1459         const struct rte_flow_attr attr = {
1460                                 .ingress = 1,
1461                                 .egress = 1,
1462                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
1463                         };
1464         void *tmp;
1465         uint32_t i, offset, mtr_idx;
1466
1467         if (priv->sh->meter_aso_en) {
1468                 i = mtrmng->n_valid;
1469                 while (i--) {
1470                         mtr_pool = mtrmng->pools[i];
1471                         for (offset = 0; offset < MLX5_ASO_MTRS_PER_POOL;
1472                                 offset++) {
1473                                 fm = &mtr_pool->mtrs[offset].fm;
1474                                 mtr_idx = MLX5_MAKE_MTR_IDX(i, offset);
1475                                 if (fm->meter_id != UINT32_MAX &&
1476                                         mlx5_flow_meter_params_flush(dev,
1477                                                 fm, &attr, mtr_idx))
1478                                         return -rte_mtr_error_set
1479                                         (error, EINVAL,
1480                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1481                                         NULL, "MTR object meter profile invalid.");
1482                         }
1483                 }
1484         } else {
1485                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1486                         fm = &legacy_fm->fm;
1487                         if (mlx5_flow_meter_params_flush(dev, fm, &attr, 0))
1488                                 return -rte_mtr_error_set(error, EINVAL,
1489                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1490                                         NULL, "MTR object meter profile invalid.");
1491                 }
1492         }
1493         TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) {
1494                 /* Check unused. */
1495                 MLX5_ASSERT(!fmp->ref_cnt);
1496                 /* Remove from list. */
1497                 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
1498                 mlx5_free(fmp);
1499         }
1500         return 0;
1501 }