drivers/net/mlx5/mlx5_flow_meter.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright 2018 Mellanox Technologies, Ltd
   4  */
   5 #include <math.h>
   6
   7 #include <rte_tailq.h>
   8 #include <rte_malloc.h>
   9 #include <rte_mtr.h>
  10 #include <rte_mtr_driver.h>
  11
  12 #include <mlx5_devx_cmds.h>
  13 #include <mlx5_malloc.h>
  14
  15 #include "mlx5.h"
  16 #include "mlx5_flow.h"
  17
  18 /**
  19  * Create the meter action.
  20  *
  21  * @param priv
  22  *   Pointer to mlx5_priv.
  23  * @param[in] fm
  24  *   Pointer to flow meter to be converted.
  25  *
  26  * @return
  27  *   Pointer to the meter action on success, NULL otherwise.
  28  */
  29 static void *
  30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
  31                               struct mlx5_flow_meter_info *fm)
  32 {
  33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
  34         struct mlx5dv_dr_flow_meter_attr mtr_init;
  35         uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
  36         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
  37                                                      &fm->profile->srtcm_prm;
  38         uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
  39         uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
  40         uint32_t val;
  41
  42         memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
  43         MLX5_SET(flow_meter_parameters, fmp, valid, 1);
  44         MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
  45         MLX5_SET(flow_meter_parameters, fmp,
  46                 start_color, MLX5_FLOW_COLOR_GREEN);
  47         MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
  48         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  49         MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
  50         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  51         MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
  52         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  53         MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
  54         val = (cbs_cir & ASO_DSEG_MAN_MASK);
  55         MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
  56         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
  57         MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
  58         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
  59         MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
  60         mtr_init.next_table =
  61                 fm->transfer ? fm->mfts->transfer.tbl->obj :
  62                         fm->egress ? fm->mfts->egress.tbl->obj :
  63                                 fm->mfts->ingress.tbl->obj;
  64         mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
  65         mtr_init.flow_meter_parameter = fmp;
  66         mtr_init.flow_meter_parameter_sz =
  67                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
  68         mtr_init.active = fm->active_state;
  69         return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
  70 #else
  71         (void)priv;
  72         (void)fm;
  73         return NULL;
  74 #endif
  75 }
  76
  77 /**
  78  * Find meter profile by id.
  79  *
  80  * @param priv
  81  *   Pointer to mlx5_priv.
  82  * @param meter_profile_id
  83  *   Meter profile id.
  84  *
  85  * @return
  86  *   Pointer to the profile found on success, NULL otherwise.
  87  */
  88 static struct mlx5_flow_meter_profile *
  89 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
  90 {
  91         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
  92         struct mlx5_flow_meter_profile *fmp;
  93
  94         TAILQ_FOREACH(fmp, fmps, next)
  95                 if (meter_profile_id == fmp->id)
  96                         return fmp;
  97         return NULL;
  98 }
  99
 100 /**
 101  * Validate the MTR profile.
 102  *
 103  * @param[in] dev
 104  *   Pointer to Ethernet device.
 105  * @param[in] meter_profile_id
 106  *   Meter profile id.
 107  * @param[in] profile
 108  *   Pointer to meter profile detail.
 109  * @param[out] error
 110  *   Pointer to the error structure.
 111  *
 112  * @return
 113  *   0 on success, a negative errno value otherwise and rte_errno is set.
 114  */
 115 static int
 116 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
 117                                  uint32_t meter_profile_id,
 118                                  struct rte_mtr_meter_profile *profile,
 119                                  struct rte_mtr_error *error)
 120 {
 121         struct mlx5_priv *priv = dev->data->dev_private;
 122         struct mlx5_flow_meter_profile *fmp;
 123
 124         /* Profile must not be NULL. */
 125         if (profile == NULL)
 126                 return -rte_mtr_error_set(error, EINVAL,
 127                                           RTE_MTR_ERROR_TYPE_METER_PROFILE,
 128                                           NULL, "Meter profile is null.");
 129         /* Meter profile ID must be valid. */
 130         if (meter_profile_id == UINT32_MAX)
 131                 return -rte_mtr_error_set(error, EINVAL,
 132                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 133                                           NULL, "Meter profile id not valid.");
 134         /* Meter profile must not exist. */
 135         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 136         if (fmp)
 137                 return -rte_mtr_error_set(error, EEXIST,
 138                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 139                                           NULL,
 140                                           "Meter profile already exists.");
 141         if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
 142                 if (priv->config.hca_attr.qos.flow_meter_old) {
 143                         /* Verify support for flow meter parameters. */
 144                         if (profile->srtcm_rfc2697.cir > 0 &&
 145                             profile->srtcm_rfc2697.cir <= MLX5_SRTCM_CIR_MAX &&
 146                             profile->srtcm_rfc2697.cbs > 0 &&
 147                             profile->srtcm_rfc2697.cbs <= MLX5_SRTCM_CBS_MAX &&
 148                             profile->srtcm_rfc2697.ebs <= MLX5_SRTCM_EBS_MAX)
 149                                 return 0;
 150                         else
 151                                 return -rte_mtr_error_set
 152                                              (error, ENOTSUP,
 153                                               RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 154                                               NULL,
 155                                               profile->srtcm_rfc2697.ebs ?
 156                                               "Metering value ebs must be 0." :
 157                                               "Invalid metering parameters.");
 158                 }
 159         }
 160         return -rte_mtr_error_set(error, ENOTSUP,
 161                                   RTE_MTR_ERROR_TYPE_METER_PROFILE,
 162                                   NULL, "Metering algorithm not supported.");
 163 }
 164
 165 /**
 166  * Calculate mantissa and exponent for cir.
 167  *
 168  * @param[in] cir
 169  *   Value to be calculated.
 170  * @param[out] man
 171  *   Pointer to the mantissa.
 172  * @param[out] exp
 173  *   Pointer to the exp.
 174  */
 175 static void
 176 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
 177 {
 178         int64_t _cir;
 179         int64_t delta = INT64_MAX;
 180         uint8_t _man = 0;
 181         uint8_t _exp = 0;
 182         uint64_t m, e;
 183
 184         for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
 185                 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
 186                         _cir = (1000000000ULL * m) >> e;
 187                         if (llabs(cir - _cir) <= delta) {
 188                                 delta = llabs(cir - _cir);
 189                                 _man = m;
 190                                 _exp = e;
 191                         }
 192                 }
 193         }
 194         *man = _man;
 195         *exp = _exp;
 196 }
 197
 198 /**
 199  * Calculate mantissa and exponent for xbs.
 200  *
 201  * @param[in] xbs
 202  *   Value to be calculated.
 203  * @param[out] man
 204  *   Pointer to the mantissa.
 205  * @param[out] exp
 206  *   Pointer to the exp.
 207  */
 208 static void
 209 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
 210 {
 211         int _exp;
 212         double _man;
 213
 214         /* Special case xbs == 0 ? both exp and matissa are 0. */
 215         if (xbs == 0) {
 216                 *man = 0;
 217                 *exp = 0;
 218                 return;
 219         }
 220         /* xbs = xbs_mantissa * 2^xbs_exponent */
 221         _man = frexp(xbs, &_exp);
 222         _man = _man * pow(2, MLX5_MAN_WIDTH);
 223         _exp = _exp - MLX5_MAN_WIDTH;
 224         *man = (uint8_t)ceil(_man);
 225         *exp = _exp;
 226 }
 227
 228 /**
 229  * Fill the prm meter parameter.
 230  *
 231  * @param[in,out] fmp
 232  *   Pointer to meter profie to be converted.
 233  * @param[out] error
 234  *   Pointer to the error structure.
 235  *
 236  * @return
 237  *   0 on success, a negative errno value otherwise and rte_errno is set.
 238  */
 239 static int
 240 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
 241                           struct rte_mtr_error *error)
 242 {
 243         struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
 244         uint8_t man, exp;
 245         uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
 246         uint32_t ebs_exp, ebs_man;
 247
 248         if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
 249                 return -rte_mtr_error_set(error, ENOTSUP,
 250                                 RTE_MTR_ERROR_TYPE_METER_PROFILE,
 251                                 NULL, "Metering algorithm not supported.");
 252         /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
 253         mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir,
 254                                     &man, &exp);
 255         /* Check if cir mantissa is too large. */
 256         if (exp > ASO_DSEG_CIR_EXP_MASK)
 257                 return -rte_mtr_error_set(error, ENOTSUP,
 258                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 259                                           "meter profile parameter cir is"
 260                                           " not supported.");
 261         cir_man = man;
 262         cir_exp = exp;
 263          /* cbs = cbs_mantissa * 2^cbs_exponent */
 264         mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.cbs,
 265                                     &man, &exp);
 266         /* Check if cbs mantissa is too large. */
 267         if (exp > ASO_DSEG_EXP_MASK)
 268                 return -rte_mtr_error_set(error, ENOTSUP,
 269                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 270                                           "meter profile parameter cbs is"
 271                                           " not supported.");
 272         cbs_man = man;
 273         cbs_exp = exp;
 274         srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
 275                                 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
 276                                 cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
 277                                 cir_man);
 278         mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.ebs,
 279                                     &man, &exp);
 280         /* Check if ebs mantissa is too large. */
 281         if (exp > ASO_DSEG_EXP_MASK)
 282                 return -rte_mtr_error_set(error, ENOTSUP,
 283                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
 284                                           "meter profile parameter ebs is"
 285                                           " not supported.");
 286         ebs_man = man;
 287         ebs_exp = exp;
 288         srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
 289                                         ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
 290         return 0;
 291 }
 292
 293 /**
 294  * Callback to get MTR capabilities.
 295  *
 296  * @param[in] dev
 297  *   Pointer to Ethernet device.
 298  * @param[out] cap
 299  *   Pointer to save MTR capabilities.
 300  * @param[out] error
 301  *   Pointer to the error structure.
 302  *
 303  * @return
 304  *   0 on success, a negative errno value otherwise and rte_errno is set.
 305  */
 306 static int
 307 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
 308                  struct rte_mtr_capabilities *cap,
 309                  struct rte_mtr_error *error __rte_unused)
 310 {
 311         struct mlx5_priv *priv = dev->data->dev_private;
 312         struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
 313
 314         if (!priv->mtr_en)
 315                 return -rte_mtr_error_set(error, ENOTSUP,
 316                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 317                                           "Meter is not supported");
 318         memset(cap, 0, sizeof(*cap));
 319         if (priv->sh->meter_aso_en)
 320             /* 2 meters per one ASO cache line. */
 321                 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
 322         else
 323                 cap->n_max = 1 << qattr->log_max_flow_meter;
 324         cap->n_shared_max = cap->n_max;
 325         cap->identical = 1;
 326         cap->shared_identical = 1;
 327         cap->shared_n_flows_per_mtr_max = 4 << 20;
 328         /* 2M flows can share the same meter. */
 329         cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
 330         cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
 331         cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
 332         cap->policer_action_drop_supported = 1;
 333         cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
 334                           RTE_MTR_STATS_N_PKTS_DROPPED;
 335         return 0;
 336 }
 337
 338 /**
 339  * Callback to add MTR profile.
 340  *
 341  * @param[in] dev
 342  *   Pointer to Ethernet device.
 343  * @param[in] meter_profile_id
 344  *   Meter profile id.
 345  * @param[in] profile
 346  *   Pointer to meter profile detail.
 347  * @param[out] error
 348  *   Pointer to the error structure.
 349  *
 350  * @return
 351  *   0 on success, a negative errno value otherwise and rte_errno is set.
 352  */
 353 static int
 354 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
 355                        uint32_t meter_profile_id,
 356                        struct rte_mtr_meter_profile *profile,
 357                        struct rte_mtr_error *error)
 358 {
 359         struct mlx5_priv *priv = dev->data->dev_private;
 360         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
 361         struct mlx5_flow_meter_profile *fmp;
 362         int ret;
 363
 364         if (!priv->mtr_en)
 365                 return -rte_mtr_error_set(error, ENOTSUP,
 366                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 367                                           "Meter is not supported");
 368         /* Check input params. */
 369         ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
 370                                                profile, error);
 371         if (ret)
 372                 return ret;
 373         /* Meter profile memory allocation. */
 374         fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
 375                          RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
 376         if (fmp == NULL)
 377                 return -rte_mtr_error_set(error, ENOMEM,
 378                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 379                                           NULL, "Meter profile memory "
 380                                           "alloc failed.");
 381         /* Fill profile info. */
 382         fmp->id = meter_profile_id;
 383         fmp->profile = *profile;
 384         /* Fill the flow meter parameters for the PRM. */
 385         ret = mlx5_flow_meter_param_fill(fmp, error);
 386         if (ret)
 387                 goto error;
 388         /* Add to list. */
 389         TAILQ_INSERT_TAIL(fmps, fmp, next);
 390         return 0;
 391 error:
 392         mlx5_free(fmp);
 393         return ret;
 394 }
 395
 396 /**
 397  * Callback to delete MTR profile.
 398  *
 399  * @param[in] dev
 400  *   Pointer to Ethernet device.
 401  * @param[in] meter_profile_id
 402  *   Meter profile id.
 403  * @param[out] error
 404  *   Pointer to the error structure.
 405  *
 406  * @return
 407  *   0 on success, a negative errno value otherwise and rte_errno is set.
 408  */
 409 static int
 410 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
 411                           uint32_t meter_profile_id,
 412                           struct rte_mtr_error *error)
 413 {
 414         struct mlx5_priv *priv = dev->data->dev_private;
 415         struct mlx5_flow_meter_profile *fmp;
 416
 417         if (!priv->mtr_en)
 418                 return -rte_mtr_error_set(error, ENOTSUP,
 419                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 420                                           "Meter is not supported");
 421         /* Meter profile must exist. */
 422         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
 423         if (fmp == NULL)
 424                 return -rte_mtr_error_set(error, ENOENT,
 425                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 426                                           &meter_profile_id,
 427                                           "Meter profile id is invalid.");
 428         /* Check profile is unused. */
 429         if (fmp->ref_cnt)
 430                 return -rte_mtr_error_set(error, EBUSY,
 431                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 432                                           NULL, "Meter profile is in use.");
 433         /* Remove from list. */
 434         TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
 435         mlx5_free(fmp);
 436         return 0;
 437 }
 438
 439 /**
 440  * Convert wrong color setting action to verbose error.
 441  *
 442  * @param[in] action
 443  *   Policy color action.
 444  *
 445  * @return
 446  *   Verbose meter color error type.
 447  */
 448 static inline enum rte_mtr_error_type
 449 action2error(enum rte_mtr_policer_action action)
 450 {
 451         switch (action) {
 452         case MTR_POLICER_ACTION_COLOR_GREEN:
 453                 return RTE_MTR_ERROR_TYPE_POLICER_ACTION_GREEN;
 454         case MTR_POLICER_ACTION_COLOR_YELLOW:
 455                 return RTE_MTR_ERROR_TYPE_POLICER_ACTION_YELLOW;
 456         case MTR_POLICER_ACTION_COLOR_RED:
 457                 return RTE_MTR_ERROR_TYPE_POLICER_ACTION_RED;
 458         default:
 459                 break;
 460         }
 461         return RTE_MTR_ERROR_TYPE_UNSPECIFIED;
 462 }
 463
 464 /**
 465  * Check meter validation.
 466  *
 467  * @param[in] priv
 468  *   Pointer to mlx5 private data structure.
 469  * @param[in] meter_id
 470  *   Meter id.
 471  * @param[in] params
 472  *   Pointer to rte meter parameters.
 473  * @param[out] error
 474  *   Pointer to rte meter error structure.
 475  *
 476  * @return
 477  *   0 on success, a negative errno value otherwise and rte_errno is set.
 478  */
 479 static int
 480 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
 481                          struct rte_mtr_params *params,
 482                          struct rte_mtr_error *error)
 483 {
 484         /* Meter must use global drop action. */
 485         if (!priv->sh->dr_drop_action)
 486                 return -rte_mtr_error_set(error, ENOTSUP,
 487                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 488                                           NULL,
 489                                           "No drop action ready for meter.");
 490         /* Meter params must not be NULL. */
 491         if (params == NULL)
 492                 return -rte_mtr_error_set(error, EINVAL,
 493                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 494                                           NULL, "Meter object params null.");
 495         /* Previous meter color is not supported. */
 496         if (params->use_prev_mtr_color)
 497                 return -rte_mtr_error_set(error, ENOTSUP,
 498                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 499                                           NULL,
 500                                           "Previous meter color "
 501                                           "not supported.");
 502         /* Validate policer settings. */
 503         if (params->action[RTE_COLOR_RED] != MTR_POLICER_ACTION_DROP)
 504                 return -rte_mtr_error_set
 505                                 (error, ENOTSUP,
 506                                  action2error(params->action[RTE_COLOR_RED]),
 507                                  NULL,
 508                                  "Red color only supports drop action.");
 509         if (params->action[RTE_COLOR_GREEN] != MTR_POLICER_ACTION_COLOR_GREEN)
 510                 return -rte_mtr_error_set
 511                                 (error, ENOTSUP,
 512                                  action2error(params->action[RTE_COLOR_GREEN]),
 513                                  NULL,
 514                                  "Green color only supports recolor green action.");
 515         /* Validate meter id. */
 516         if (mlx5_flow_meter_find(priv, meter_id, NULL))
 517                 return -rte_mtr_error_set(error, EEXIST,
 518                                           RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
 519                                           "Meter object already exists.");
 520         return 0;
 521 }
 522
 523 /**
 524  * Modify the flow meter action.
 525  *
 526  * @param[in] priv
 527  *   Pointer to mlx5 private data structure.
 528  * @param[in] fm
 529  *   Pointer to flow meter to be modified.
 530  * @param[in] srtcm
 531  *   Pointer to meter srtcm description parameter.
 532  * @param[in] modify_bits
 533  *   The bit in srtcm to be updated.
 534  * @param[in] active_state
 535  *   The state to be updated.
 536  * @return
 537  *   0 on success, o negative value otherwise.
 538  */
 539 static int
 540 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
 541                 struct mlx5_flow_meter_info *fm,
 542                 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
 543                 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
 544 {
 545 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
 546         uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
 547         uint32_t *attr;
 548         struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
 549         int ret;
 550         struct mlx5_aso_mtr *aso_mtr = NULL;
 551         uint32_t cbs_cir, ebs_eir, val;
 552
 553         if (priv->sh->meter_aso_en) {
 554                 fm->is_enable = !!is_enable;
 555                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 556                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 557                 if (ret)
 558                         return ret;
 559                 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
 560                 if (ret)
 561                         return ret;
 562         } else {
 563                 /* Fill command parameters. */
 564                 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
 565                 mod_attr.flow_meter_parameter = in;
 566                 mod_attr.flow_meter_parameter_sz =
 567                                 MLX5_ST_SZ_BYTES(flow_meter_parameters);
 568                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 569                         mod_attr.active = !!active_state;
 570                 else
 571                         mod_attr.active = 0;
 572                 attr = in;
 573                 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
 574                 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
 575                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
 576                         val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
 577                                 ASO_DSEG_EXP_MASK;
 578                         MLX5_SET(flow_meter_parameters, attr,
 579                                 cbs_exponent, val);
 580                         val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
 581                                 ASO_DSEG_MAN_MASK;
 582                         MLX5_SET(flow_meter_parameters, attr,
 583                                 cbs_mantissa, val);
 584                 }
 585                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
 586                         val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
 587                                 ASO_DSEG_EXP_MASK;
 588                         MLX5_SET(flow_meter_parameters, attr,
 589                                 cir_exponent, val);
 590                         val = cbs_cir & ASO_DSEG_MAN_MASK;
 591                         MLX5_SET(flow_meter_parameters, attr,
 592                                 cir_mantissa, val);
 593                 }
 594                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
 595                         val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
 596                                 ASO_DSEG_EXP_MASK;
 597                         MLX5_SET(flow_meter_parameters, attr,
 598                                 ebs_exponent, val);
 599                         val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
 600                                 ASO_DSEG_MAN_MASK;
 601                         MLX5_SET(flow_meter_parameters, attr,
 602                                 ebs_mantissa, val);
 603                 }
 604                 /* Apply modifications to meter only if it was created. */
 605                 if (fm->meter_action) {
 606                         ret = mlx5_glue->dv_modify_flow_action_meter
 607                                         (fm->meter_action, &mod_attr,
 608                                         rte_cpu_to_be_64(modify_bits));
 609                         if (ret)
 610                                 return ret;
 611                 }
 612                 /* Update succeedded modify meter parameters. */
 613                 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
 614                         fm->active_state = !!active_state;
 615         }
 616         return 0;
 617 #else
 618         (void)priv;
 619         (void)fm;
 620         (void)srtcm;
 621         (void)modify_bits;
 622         (void)active_state;
 623         (void)is_enable;
 624         return -ENOTSUP;
 625 #endif
 626 }
 627
 628 static void
 629 mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter_info *fm,
 630                                 uint64_t stats_mask)
 631 {
 632         fm->green_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_GREEN) ? 1 : 0;
 633         fm->green_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_GREEN) ? 1 : 0;
 634         fm->red_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_RED) ? 1 : 0;
 635         fm->red_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_RED) ? 1 : 0;
 636         fm->bytes_dropped =
 637                 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
 638         fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
 639 }
 640
 641 /**
 642  * Create meter rules.
 643  *
 644  * @param[in] dev
 645  *   Pointer to Ethernet device.
 646  * @param[in] meter_id
 647  *   Meter id.
 648  * @param[in] params
 649  *   Pointer to rte meter parameters.
 650  * @param[in] shared
 651  *   Meter shared with other flow or not.
 652  * @param[out] error
 653  *   Pointer to rte meter error structure.
 654  *
 655  * @return
 656  *   0 on success, a negative errno value otherwise and rte_errno is set.
 657  */
 658 static int
 659 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
 660                        struct rte_mtr_params *params, int shared,
 661                        struct rte_mtr_error *error)
 662 {
 663         struct mlx5_priv *priv = dev->data->dev_private;
 664         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
 665         struct mlx5_flow_meter_profile *fmp;
 666         struct mlx5_legacy_flow_meter *legacy_fm;
 667         struct mlx5_flow_meter_info *fm;
 668         const struct rte_flow_attr attr = {
 669                                 .ingress = 1,
 670                                 .egress = 1,
 671                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
 672                         };
 673         struct mlx5_indexed_pool_config flow_ipool_cfg = {
 674                 .size = 0,
 675                 .trunk_size = 64,
 676                 .need_lock = 1,
 677                 .type = "mlx5_flow_mtr_flow_id_pool",
 678         };
 679         struct mlx5_aso_mtr *aso_mtr;
 680         union mlx5_l3t_data data;
 681         uint32_t mtr_idx;
 682         int ret;
 683         uint8_t mtr_id_bits;
 684         uint8_t mtr_reg_bits = priv->mtr_reg_share ?
 685                                 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
 686
 687         if (!priv->mtr_en)
 688                 return -rte_mtr_error_set(error, ENOTSUP,
 689                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 690                                           "Meter is not supported");
 691         /* Validate the parameters. */
 692         ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
 693         if (ret)
 694                 return ret;
 695         /* Meter profile must exist. */
 696         fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
 697         if (fmp == NULL)
 698                 return -rte_mtr_error_set(error, ENOENT,
 699                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 700                                           NULL, "Meter profile id not valid.");
 701         /* Allocate the flow meter memory. */
 702         if (priv->sh->meter_aso_en) {
 703                 mtr_idx = mlx5_flow_mtr_alloc(dev);
 704                 if (!mtr_idx)
 705                         return -rte_mtr_error_set(error, ENOMEM,
 706                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 707                                 "Memory alloc failed for meter.");
 708                 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
 709                 fm = &aso_mtr->fm;
 710         } else {
 711                 legacy_fm = mlx5_ipool_zmalloc
 712                                 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
 713                 if (legacy_fm == NULL)
 714                         return -rte_mtr_error_set(error, ENOMEM,
 715                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 716                                 "Memory alloc failed for meter.");
 717                 legacy_fm->idx = mtr_idx;
 718                 fm = &legacy_fm->fm;
 719         }
 720         mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
 721         if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) {
 722                 DRV_LOG(ERR, "Meter number exceeds max limit.");
 723                 goto error;
 724         }
 725         if (mtr_id_bits > priv->max_mtr_bits)
 726                 priv->max_mtr_bits = mtr_id_bits;
 727         /* Fill the flow meter parameters. */
 728         fm->meter_id = meter_id;
 729         fm->profile = fmp;
 730         memcpy(fm->action, params->action, sizeof(params->action));
 731         mlx5_flow_meter_stats_enable_update(fm, params->stats_mask);
 732         /* Alloc policer counters. */
 733         if (fm->green_bytes || fm->green_pkts) {
 734                 fm->policer_stats.pass_cnt = mlx5_counter_alloc(dev);
 735                 if (!fm->policer_stats.pass_cnt)
 736                         goto error;
 737         }
 738         if (fm->red_bytes || fm->red_pkts ||
 739             fm->bytes_dropped || fm->pkts_dropped) {
 740                 fm->policer_stats.drop_cnt = mlx5_counter_alloc(dev);
 741                 if (!fm->policer_stats.drop_cnt)
 742                         goto error;
 743         }
 744         fm->mfts = mlx5_flow_create_mtr_tbls(dev);
 745         if (!fm->mfts)
 746                 goto error;
 747         ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr);
 748         if (ret)
 749                 goto error;
 750         /* Add to the flow meter list. */
 751         if (!priv->sh->meter_aso_en)
 752                 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
 753         fm->active_state = 1; /* Config meter starts as active. */
 754         fm->is_enable = 1;
 755         fm->shared = !!shared;
 756         __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
 757         fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
 758         if (!fm->flow_ipool)
 759                 goto error;
 760         rte_spinlock_init(&fm->sl);
 761         /* If ASO meter supported, allocate ASO flow meter. */
 762         if (priv->sh->meter_aso_en) {
 763                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
 764                 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
 765                 if (ret)
 766                         goto error;
 767                 data.dword = mtr_idx;
 768                 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
 769                         goto error;
 770         }
 771         return 0;
 772 error:
 773         mlx5_flow_destroy_policer_rules(dev, fm, &attr);
 774         mlx5_flow_destroy_mtr_tbls(dev, fm->mfts);
 775         /* Free policer counters. */
 776         if (fm->policer_stats.pass_cnt)
 777                 mlx5_counter_free(dev, fm->policer_stats.pass_cnt);
 778         if (fm->policer_stats.drop_cnt)
 779                 mlx5_counter_free(dev, fm->policer_stats.drop_cnt);
 780         if (priv->sh->meter_aso_en)
 781                 mlx5_flow_mtr_free(dev, mtr_idx);
 782         else
 783                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
 784         return -rte_mtr_error_set(error, -ret,
 785                                   RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 786                                   NULL, "Failed to create devx meter.");
 787 }
 788
 789 static int
 790 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
 791                         struct mlx5_flow_meter_info *fm,
 792                         const struct rte_flow_attr *attr,
 793                         uint32_t mtr_idx)
 794 {
 795         struct mlx5_priv *priv = dev->data->dev_private;
 796         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
 797         struct mlx5_flow_meter_profile *fmp;
 798         struct mlx5_legacy_flow_meter *legacy_fm = NULL;
 799
 800         /* Meter object must not have any owner. */
 801         MLX5_ASSERT(!fm->ref_cnt);
 802         /* Get meter profile. */
 803         fmp = fm->profile;
 804         if (fmp == NULL)
 805                 return -1;
 806         /* Update dependencies. */
 807         __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
 808         /* Remove from list. */
 809         if (!priv->sh->meter_aso_en) {
 810                 legacy_fm = container_of(fm, struct mlx5_legacy_flow_meter, fm);
 811                 TAILQ_REMOVE(fms, legacy_fm, next);
 812         }
 813         /* Free policer counters. */
 814         if (fm->policer_stats.pass_cnt)
 815                 mlx5_counter_free(dev, fm->policer_stats.pass_cnt);
 816         if (fm->policer_stats.drop_cnt)
 817                 mlx5_counter_free(dev, fm->policer_stats.drop_cnt);
 818         /* Free meter flow table. */
 819         if (fm->flow_ipool)
 820                 mlx5_ipool_destroy(fm->flow_ipool);
 821         mlx5_flow_destroy_policer_rules(dev, fm, attr);
 822         mlx5_flow_destroy_mtr_tbls(dev, fm->mfts);
 823         if (priv->sh->meter_aso_en)
 824                 mlx5_flow_mtr_free(dev, mtr_idx);
 825         else
 826                 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
 827                                         legacy_fm->idx);
 828         return 0;
 829 }
 830
 831 /**
 832  * Destroy meter rules.
 833  *
 834  * @param[in] dev
 835  *   Pointer to Ethernet device.
 836  * @param[in] meter_id
 837  *   Meter id.
 838  * @param[out] error
 839  *   Pointer to rte meter error structure.
 840  *
 841  * @return
 842  *   0 on success, a negative errno value otherwise and rte_errno is set.
 843  */
 844 static int
 845 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
 846                         struct rte_mtr_error *error)
 847 {
 848         struct mlx5_priv *priv = dev->data->dev_private;
 849         struct mlx5_flow_meter_info *fm;
 850         const struct rte_flow_attr attr = {
 851                                 .ingress = 1,
 852                                 .egress = 1,
 853                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
 854                         };
 855         uint32_t mtr_idx = 0;
 856
 857         if (!priv->mtr_en)
 858                 return -rte_mtr_error_set(error, ENOTSUP,
 859                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 860                                           "Meter is not supported");
 861         /* Meter object must exist. */
 862         fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
 863         if (fm == NULL)
 864                 return -rte_mtr_error_set(error, ENOENT,
 865                                           RTE_MTR_ERROR_TYPE_MTR_ID,
 866                                           NULL, "Meter object id not valid.");
 867         /* Meter object must not have any owner. */
 868         if (fm->ref_cnt > 0)
 869                 return -rte_mtr_error_set(error, EBUSY,
 870                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED,
 871                                           NULL, "Meter object is being used.");
 872         if (priv->sh->meter_aso_en) {
 873                 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id))
 874                         return -rte_mtr_error_set(error, EBUSY,
 875                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 876                                 "Fail to delete ASO Meter in index table.");
 877         }
 878         /* Destroy the meter profile. */
 879         if (mlx5_flow_meter_params_flush(dev, fm, &attr, mtr_idx))
 880                 return -rte_mtr_error_set(error, EINVAL,
 881                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
 882                                         NULL, "MTR object meter profile invalid.");
 883         return 0;
 884 }
 885
 886 /**
 887  * Modify meter state.
 888  *
 889  * @param[in] priv
 890  *   Pointer to mlx5 private data structure.
 891  * @param[in] fm
 892  *   Pointer to flow meter.
 893  * @param[in] new_state
 894  *   New state to update.
 895  * @param[out] error
 896  *   Pointer to rte meter error structure.
 897  *
 898  * @return
 899  *   0 on success, a negative errno value otherwise and rte_errno is set.
 900  */
 901 static int
 902 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
 903                              struct mlx5_flow_meter_info *fm,
 904                              uint32_t new_state,
 905                              struct rte_mtr_error *error)
 906 {
 907         static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
 908                 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
 909                 .ebs_eir = 0,
 910         };
 911         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
 912                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
 913         int ret;
 914
 915         if (new_state == MLX5_FLOW_METER_DISABLE)
 916                 ret = mlx5_flow_meter_action_modify(priv, fm,
 917                                 &srtcm, modify_bits, 0, 0);
 918         else
 919                 ret = mlx5_flow_meter_action_modify(priv, fm,
 920                                                    &fm->profile->srtcm_prm,
 921                                                     modify_bits, 0, 1);
 922         if (ret)
 923                 return -rte_mtr_error_set(error, -ret,
 924                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
 925                                           NULL,
 926                                           new_state ?
 927                                           "Failed to enable meter." :
 928                                           "Failed to disable meter.");
 929         return 0;
 930 }
 931
 932 /**
 933  * Callback to enable flow meter.
 934  *
 935  * @param[in] dev
 936  *   Pointer to Ethernet device.
 937  * @param[in] meter_id
 938  *   Meter id.
 939  * @param[out] error
 940  *   Pointer to rte meter error structure.
 941  *
 942  * @return
 943  *   0 on success, a negative errno value otherwise and rte_errno is set.
 944  */
 945 static int
 946 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
 947                        uint32_t meter_id,
 948                        struct rte_mtr_error *error)
 949 {
 950         struct mlx5_priv *priv = dev->data->dev_private;
 951         struct mlx5_flow_meter_info *fm;
 952         int ret;
 953
 954         if (!priv->mtr_en)
 955                 return -rte_mtr_error_set(error, ENOTSUP,
 956                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 957                                           "Meter is not supported");
 958         /* Meter object must exist. */
 959         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
 960         if (fm == NULL)
 961                 return -rte_mtr_error_set(error, ENOENT,
 962                                           RTE_MTR_ERROR_TYPE_MTR_ID,
 963                                           NULL, "Meter not found.");
 964         if (fm->active_state == MLX5_FLOW_METER_ENABLE)
 965                 return 0;
 966         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
 967                                            error);
 968         if (!ret)
 969                 fm->active_state = MLX5_FLOW_METER_ENABLE;
 970         return ret;
 971 }
 972
 973 /**
 974  * Callback to disable flow meter.
 975  *
 976  * @param[in] dev
 977  *   Pointer to Ethernet device.
 978  * @param[in] meter_id
 979  *   Meter id.
 980  * @param[out] error
 981  *   Pointer to rte meter error structure.
 982  *
 983  * @return
 984  *   0 on success, a negative errno value otherwise and rte_errno is set.
 985  */
 986 static int
 987 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
 988                         uint32_t meter_id,
 989                         struct rte_mtr_error *error)
 990 {
 991         struct mlx5_priv *priv = dev->data->dev_private;
 992         struct mlx5_flow_meter_info *fm;
 993         int ret;
 994
 995         if (!priv->mtr_en)
 996                 return -rte_mtr_error_set(error, ENOTSUP,
 997                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
 998                                           "Meter is not supported");
 999         /* Meter object must exist. */
1000         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1001         if (fm == NULL)
1002                 return -rte_mtr_error_set(error, ENOENT,
1003                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1004                                           NULL, "Meter not found.");
1005         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1006                 return 0;
1007         ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1008                                            error);
1009         if (!ret)
1010                 fm->active_state = MLX5_FLOW_METER_DISABLE;
1011         return ret;
1012 }
1013
1014 /**
1015  * Callback to update meter profile.
1016  *
1017  * @param[in] dev
1018  *   Pointer to Ethernet device.
1019  * @param[in] meter_id
1020  *   Meter id.
1021  * @param[in] meter_profile_id
1022  *   To be updated meter profile id.
1023  * @param[out] error
1024  *   Pointer to rte meter error structure.
1025  *
1026  * @return
1027  *   0 on success, a negative errno value otherwise and rte_errno is set.
1028  */
1029 static int
1030 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1031                                uint32_t meter_id,
1032                                uint32_t meter_profile_id,
1033                                struct rte_mtr_error *error)
1034 {
1035         struct mlx5_priv *priv = dev->data->dev_private;
1036         struct mlx5_flow_meter_profile *fmp;
1037         struct mlx5_flow_meter_profile *old_fmp;
1038         struct mlx5_flow_meter_info *fm;
1039         uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1040                                MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1041         int ret;
1042
1043         if (!priv->mtr_en)
1044                 return -rte_mtr_error_set(error, ENOTSUP,
1045                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1046                                           "Meter is not supported");
1047         /* Meter profile must exist. */
1048         fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1049         if (fmp == NULL)
1050                 return -rte_mtr_error_set(error, ENOENT,
1051                                           RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1052                                           NULL, "Meter profile not found.");
1053         /* Meter object must exist. */
1054         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1055         if (fm == NULL)
1056                 return -rte_mtr_error_set(error, ENOENT,
1057                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1058                                           NULL, "Meter not found.");
1059         /* MTR object already set to meter profile id. */
1060         old_fmp = fm->profile;
1061         if (fmp == old_fmp)
1062                 return 0;
1063         /* Update the profile. */
1064         fm->profile = fmp;
1065         /* Update meter params in HW (if not disabled). */
1066         if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1067                 return 0;
1068         ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1069                                               modify_bits, fm->active_state, 1);
1070         if (ret) {
1071                 fm->profile = old_fmp;
1072                 return -rte_mtr_error_set(error, -ret,
1073                                           RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1074                                           NULL, "Failed to update meter"
1075                                           " parmeters in hardware.");
1076         }
1077         old_fmp->ref_cnt--;
1078         fmp->ref_cnt++;
1079         return 0;
1080 }
1081
1082 /**
1083  * Callback to update meter stats mask.
1084  *
1085  * @param[in] dev
1086  *   Pointer to Ethernet device.
1087  * @param[in] meter_id
1088  *   Meter id.
1089  * @param[in] stats_mask
1090  *   To be updated stats_mask.
1091  * @param[out] error
1092  *   Pointer to rte meter error structure.
1093  *
1094  * @return
1095  *   0 on success, a negative errno value otherwise and rte_errno is set.
1096  */
1097 static int
1098 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1099                              uint32_t meter_id,
1100                              uint64_t stats_mask,
1101                              struct rte_mtr_error *error)
1102 {
1103         struct mlx5_priv *priv = dev->data->dev_private;
1104         struct mlx5_flow_meter_info *fm;
1105         const struct rte_flow_attr attr = {
1106                                 .ingress = 1,
1107                                 .egress = 1,
1108                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
1109                         };
1110         bool need_updated = false;
1111         struct mlx5_flow_policer_stats old_policer_stats;
1112
1113         if (!priv->mtr_en)
1114                 return -rte_mtr_error_set(error, ENOTSUP,
1115                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1116                                           "Meter is not supported");
1117         /* Meter object must exist. */
1118         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1119         if (fm == NULL)
1120                 return -rte_mtr_error_set(error, ENOENT,
1121                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1122                                           NULL, "Meter object id not valid.");
1123         old_policer_stats.pass_cnt = 0;
1124         old_policer_stats.drop_cnt = 0;
1125         if (!!((RTE_MTR_STATS_N_PKTS_GREEN |
1126                                 RTE_MTR_STATS_N_BYTES_GREEN) & stats_mask) !=
1127                 !!fm->policer_stats.pass_cnt) {
1128                 need_updated = true;
1129                 if (fm->policer_stats.pass_cnt) {
1130                         old_policer_stats.pass_cnt = fm->policer_stats.pass_cnt;
1131                         fm->policer_stats.pass_cnt = 0;
1132                 } else {
1133                         fm->policer_stats.pass_cnt =
1134                                 mlx5_counter_alloc(dev);
1135                         if (!fm->policer_stats.pass_cnt)
1136                                 return -rte_mtr_error_set(error, ENOMEM,
1137                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1138                                           "Counter alloc failed for meter.");
1139                 }
1140         }
1141         if (!!((RTE_MTR_STATS_N_PKTS_RED | RTE_MTR_STATS_N_BYTES_RED |
1142                 RTE_MTR_STATS_N_PKTS_DROPPED | RTE_MTR_STATS_N_BYTES_DROPPED) &
1143                 stats_mask) !=
1144                 !!fm->policer_stats.drop_cnt) {
1145                 need_updated = true;
1146                 if (fm->policer_stats.drop_cnt) {
1147                         old_policer_stats.drop_cnt = fm->policer_stats.drop_cnt;
1148                         fm->policer_stats.drop_cnt = 0;
1149                 } else {
1150                         fm->policer_stats.drop_cnt =
1151                                 mlx5_counter_alloc(dev);
1152                         if (!fm->policer_stats.drop_cnt)
1153                                 return -rte_mtr_error_set(error, ENOMEM,
1154                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1155                                           "Counter alloc failed for meter.");
1156                 }
1157         }
1158         if (need_updated) {
1159                 if (mlx5_flow_prepare_policer_rules(dev, fm, &attr)) {
1160                         if (fm->policer_stats.pass_cnt &&
1161                                 fm->policer_stats.pass_cnt !=
1162                                 old_policer_stats.pass_cnt)
1163                                 mlx5_counter_free(dev,
1164                                         fm->policer_stats.pass_cnt);
1165                         fm->policer_stats.pass_cnt =
1166                                         old_policer_stats.pass_cnt;
1167                         if (fm->policer_stats.drop_cnt &&
1168                                 fm->policer_stats.drop_cnt !=
1169                                 old_policer_stats.drop_cnt)
1170                                 mlx5_counter_free(dev,
1171                                         fm->policer_stats.drop_cnt);
1172                         fm->policer_stats.pass_cnt =
1173                                         old_policer_stats.pass_cnt;
1174                         return -rte_mtr_error_set(error, ENOTSUP,
1175                                 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1176                                 NULL, "Failed to create meter policer rules.");
1177                 }
1178                 /* Free old policer counters. */
1179                 if (old_policer_stats.pass_cnt)
1180                         mlx5_counter_free(dev,
1181                                 old_policer_stats.pass_cnt);
1182                 if (old_policer_stats.drop_cnt)
1183                         mlx5_counter_free(dev,
1184                                 old_policer_stats.drop_cnt);
1185         }
1186         mlx5_flow_meter_stats_enable_update(fm, stats_mask);
1187         return 0;
1188 }
1189
1190 /**
1191  * Callback to read meter statistics.
1192  *
1193  * @param[in] dev
1194  *   Pointer to Ethernet device.
1195  * @param[in] meter_id
1196  *   Meter id.
1197  * @param[out] stats
1198  *   Pointer to store the statistics.
1199  * @param[out] stats_mask
1200  *   Pointer to store the stats_mask.
1201  * @param[in] clear
1202  *   Statistic to be cleared after read or not.
1203  * @param[out] error
1204  *   Pointer to rte meter error structure.
1205  *
1206  * @return
1207  *   0 on success, a negative errno value otherwise and rte_errno is set.
1208  */
1209 static int
1210 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1211                            uint32_t meter_id,
1212                            struct rte_mtr_stats *stats,
1213                            uint64_t *stats_mask,
1214                            int clear,
1215                            struct rte_mtr_error *error)
1216 {
1217         struct mlx5_priv *priv = dev->data->dev_private;
1218         struct mlx5_flow_meter_info *fm;
1219         struct mlx5_flow_policer_stats *ps;
1220         uint64_t pkts;
1221         uint64_t bytes;
1222         int ret = 0;
1223
1224         if (!priv->mtr_en)
1225                 return -rte_mtr_error_set(error, ENOTSUP,
1226                                           RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1227                                           "Meter is not supported");
1228         /* Meter object must exist. */
1229         fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1230         if (fm == NULL)
1231                 return -rte_mtr_error_set(error, ENOENT,
1232                                           RTE_MTR_ERROR_TYPE_MTR_ID,
1233                                           NULL, "Meter object id not valid.");
1234         ps = &fm->policer_stats;
1235         *stats_mask = 0;
1236         if (fm->green_bytes)
1237                 *stats_mask |= RTE_MTR_STATS_N_BYTES_GREEN;
1238         if (fm->green_pkts)
1239                 *stats_mask |= RTE_MTR_STATS_N_PKTS_GREEN;
1240         if (fm->red_bytes)
1241                 *stats_mask |= RTE_MTR_STATS_N_BYTES_RED;
1242         if (fm->red_pkts)
1243                 *stats_mask |= RTE_MTR_STATS_N_PKTS_RED;
1244         if (fm->bytes_dropped)
1245                 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1246         if (fm->pkts_dropped)
1247                 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1248         memset(stats, 0, sizeof(*stats));
1249         if (ps->pass_cnt) {
1250                 ret = mlx5_counter_query(dev, ps->pass_cnt, clear, &pkts,
1251                                                  &bytes);
1252                 if (ret)
1253                         goto error;
1254                 /* If need to read the packets, set it. */
1255                 if (fm->green_pkts)
1256                         stats->n_pkts[RTE_COLOR_GREEN] = pkts;
1257                 /* If need to read the bytes, set it. */
1258                 if (fm->green_bytes)
1259                         stats->n_bytes[RTE_COLOR_GREEN] = bytes;
1260         }
1261         if (ps->drop_cnt) {
1262                 ret = mlx5_counter_query(dev, ps->drop_cnt, clear, &pkts,
1263                                                  &bytes);
1264                 if (ret)
1265                         goto error;
1266                 /* If need to read the packets, set it. */
1267                 if (fm->pkts_dropped)
1268                         stats->n_pkts_dropped = pkts;
1269                 /* If need to read the bytes, set it. */
1270                 if (fm->bytes_dropped)
1271                         stats->n_bytes_dropped = bytes;
1272         }
1273         return 0;
1274 error:
1275         return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1276                                  "Failed to read policer counters.");
1277 }
1278
1279 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1280         .capabilities_get = mlx5_flow_mtr_cap_get,
1281         .meter_profile_add = mlx5_flow_meter_profile_add,
1282         .meter_profile_delete = mlx5_flow_meter_profile_delete,
1283         .create = mlx5_flow_meter_create,
1284         .destroy = mlx5_flow_meter_destroy,
1285         .meter_enable = mlx5_flow_meter_enable,
1286         .meter_disable = mlx5_flow_meter_disable,
1287         .meter_profile_update = mlx5_flow_meter_profile_update,
1288         .meter_dscp_table_update = NULL,
1289         .policer_actions_update = NULL,
1290         .stats_update = mlx5_flow_meter_stats_update,
1291         .stats_read = mlx5_flow_meter_stats_read,
1292 };
1293
1294 /**
1295  * Get meter operations.
1296  *
1297  * @param dev
1298  *   Pointer to Ethernet device structure.
1299  * @param arg
1300  *   Pointer to set the mtr operations.
1301  *
1302  * @return
1303  *   Always 0.
1304  */
1305 int
1306 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1307 {
1308         *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1309         return 0;
1310 }
1311
1312 /**
1313  * Find meter by id.
1314  *
1315  * @param priv
1316  *   Pointer to mlx5_priv.
1317  * @param meter_id
1318  *   Meter id.
1319  * @param mtr_idx
1320  *   Pointer to Meter index.
1321  *
1322  * @return
1323  *   Pointer to the profile found on success, NULL otherwise.
1324  */
1325 struct mlx5_flow_meter_info *
1326 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1327                 uint32_t *mtr_idx)
1328 {
1329         struct mlx5_legacy_flow_meter *legacy_fm;
1330         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1331         struct mlx5_aso_mtr *aso_mtr;
1332         struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng;
1333         union mlx5_l3t_data data;
1334
1335         if (priv->sh->meter_aso_en) {
1336                 rte_spinlock_lock(&mtrmng->mtrsl);
1337                 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1338                         !data.dword) {
1339                         rte_spinlock_unlock(&mtrmng->mtrsl);
1340                         return NULL;
1341                 }
1342                 if (mtr_idx)
1343                         *mtr_idx = data.dword;
1344                 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1345                 /* Remove reference taken by the mlx5_l3t_get_entry. */
1346                 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1347                 MLX5_ASSERT(meter_id == aso_mtr->fm.meter_id);
1348                 rte_spinlock_unlock(&mtrmng->mtrsl);
1349                 return &aso_mtr->fm;
1350         }
1351         TAILQ_FOREACH(legacy_fm, fms, next)
1352                 if (meter_id == legacy_fm->fm.meter_id) {
1353                         if (mtr_idx)
1354                                 *mtr_idx = legacy_fm->idx;
1355                         return &legacy_fm->fm;
1356                 }
1357         return NULL;
1358 }
1359
1360 /**
1361  * Find meter by index.
1362  *
1363  * @param priv
1364  *   Pointer to mlx5_priv.
1365  * @param idx
1366  *   Meter index.
1367  *
1368  * @return
1369  *   Pointer to the profile found on success, NULL otherwise.
1370  */
1371 struct mlx5_flow_meter_info *
1372 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1373 {
1374         struct mlx5_aso_mtr *aso_mtr;
1375
1376         if (priv->sh->meter_aso_en) {
1377                 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1378                 return &aso_mtr->fm;
1379         } else {
1380                 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1381         }
1382 }
1383
1384 /**
1385  * Attach meter to flow.
1386  * Unidirectional Meter creation can only be done
1387  * when flow direction is known, i.e. when calling meter_attach.
1388  *
1389  * @param [in] priv
1390  *  Pointer to mlx5 private data.
1391  * @param[in] fm
1392  *   Pointer to flow meter.
1393  * @param [in] attr
1394  *  Pointer to flow attributes.
1395  * @param [out] error
1396  *  Pointer to error structure.
1397  *
1398  * @return
1399  *   0 on success, a negative errno value otherwise and rte_errno is set.
1400  */
1401 int
1402 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1403                        struct mlx5_flow_meter_info *fm,
1404                        const struct rte_flow_attr *attr,
1405                        struct rte_flow_error *error)
1406 {
1407         int ret = 0;
1408
1409         if (priv->sh->meter_aso_en) {
1410                 struct mlx5_aso_mtr *aso_mtr;
1411
1412                 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1413                 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1414                         return rte_flow_error_set(error, ENOENT,
1415                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1416                                         NULL,
1417                                         "Timeout in meter configuration");
1418                 }
1419                 rte_spinlock_lock(&fm->sl);
1420                 if (fm->shared || !fm->ref_cnt) {
1421                         fm->ref_cnt++;
1422                 } else {
1423                         rte_flow_error_set(error, EINVAL,
1424                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1425                                    "Meter cannot be shared");
1426                         ret = -1;
1427                 }
1428                 rte_spinlock_unlock(&fm->sl);
1429         } else {
1430                 rte_spinlock_lock(&fm->sl);
1431                 if (fm->meter_action) {
1432                         if (fm->shared &&
1433                             attr->transfer == fm->transfer &&
1434                             attr->ingress == fm->ingress &&
1435                             attr->egress == fm->egress) {
1436                                 fm->ref_cnt++;
1437                         } else {
1438                                 rte_flow_error_set(error, EINVAL,
1439                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1440                                         fm->shared ?
1441                                         "Meter attr not match." :
1442                                         "Meter cannot be shared.");
1443                                 ret = -1;
1444                         }
1445                 } else {
1446                         fm->ingress = attr->ingress;
1447                         fm->egress = attr->egress;
1448                         fm->transfer = attr->transfer;
1449                         fm->ref_cnt = 1;
1450                         /* This also creates the meter object. */
1451                         fm->meter_action = mlx5_flow_meter_action_create(priv,
1452                                                                          fm);
1453                         if (!fm->meter_action) {
1454                                 fm->ref_cnt = 0;
1455                                 fm->ingress = 0;
1456                                 fm->egress = 0;
1457                                 fm->transfer = 0;
1458                                 rte_flow_error_set(error, EINVAL,
1459                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1460                                         "Meter action create failed.");
1461                                 ret = -1;
1462                         }
1463                 }
1464                 rte_spinlock_unlock(&fm->sl);
1465         }
1466         return ret ? -rte_errno : 0;
1467 }
1468
1469 /**
1470  * Detach meter from flow.
1471  *
1472  * @param [in] priv
1473  *  Pointer to mlx5 private data.
1474  * @param [in] fm
1475  *  Pointer to flow meter.
1476  */
1477 void
1478 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1479                        struct mlx5_flow_meter_info *fm)
1480 {
1481 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1482         rte_spinlock_lock(&fm->sl);
1483         MLX5_ASSERT(fm->ref_cnt);
1484         if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1485                 mlx5_glue->destroy_flow_action(fm->meter_action);
1486                 fm->meter_action = NULL;
1487                 fm->ingress = 0;
1488                 fm->egress = 0;
1489                 fm->transfer = 0;
1490         }
1491         rte_spinlock_unlock(&fm->sl);
1492 #else
1493         (void)priv;
1494         (void)fm;
1495 #endif
1496 }
1497
1498 /**
1499  * Flush meter configuration.
1500  *
1501  * @param[in] dev
1502  *   Pointer to Ethernet device.
1503  * @param[out] error
1504  *   Pointer to rte meter error structure.
1505  *
1506  * @return
1507  *   0 on success, a negative errno value otherwise and rte_errno is set.
1508  */
1509 int
1510 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1511 {
1512         struct mlx5_priv *priv = dev->data->dev_private;
1513         struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng;
1514         struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1515         struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
1516         struct mlx5_flow_meter_profile *fmp;
1517         struct mlx5_legacy_flow_meter *legacy_fm;
1518         struct mlx5_flow_meter_info *fm;
1519         struct mlx5_aso_mtr_pool *mtr_pool;
1520         const struct rte_flow_attr attr = {
1521                                 .ingress = 1,
1522                                 .egress = 1,
1523                                 .transfer = priv->config.dv_esw_en ? 1 : 0,
1524                         };
1525         void *tmp;
1526         uint32_t i, offset, mtr_idx;
1527
1528         if (priv->sh->meter_aso_en) {
1529                 i = mtrmng->n_valid;
1530                 while (i--) {
1531                         mtr_pool = mtrmng->pools[i];
1532                         for (offset = 0; offset < MLX5_ASO_MTRS_PER_POOL;
1533                                 offset++) {
1534                                 fm = &mtr_pool->mtrs[offset].fm;
1535                                 mtr_idx = MLX5_MAKE_MTR_IDX(i, offset);
1536                                 if (fm->meter_id != UINT32_MAX &&
1537                                         mlx5_flow_meter_params_flush(dev,
1538                                                 fm, &attr, mtr_idx))
1539                                         return -rte_mtr_error_set
1540                                         (error, EINVAL,
1541                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1542                                         NULL, "MTR object meter profile invalid.");
1543                         }
1544                 }
1545         } else {
1546                 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1547                         fm = &legacy_fm->fm;
1548                         if (mlx5_flow_meter_params_flush(dev, fm, &attr, 0))
1549                                 return -rte_mtr_error_set(error, EINVAL,
1550                                         RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1551                                         NULL, "MTR object meter profile invalid.");
1552                 }
1553         }
1554         TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) {
1555                 /* Check unused. */
1556                 MLX5_ASSERT(!fmp->ref_cnt);
1557                 /* Remove from list. */
1558                 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
1559                 mlx5_free(fmp);
1560         }
1561         return 0;
1562 }