1 // SPDX-License-Identifier: BSD-3-Clause
3 * Copyright 2018 Mellanox Technologies, Ltd
8 #include <rte_malloc.h>
10 #include <rte_mtr_driver.h>
12 #include <mlx5_devx_cmds.h>
13 #include <mlx5_malloc.h>
16 #include "mlx5_flow.h"
18 static int mlx5_flow_meter_disable(struct rte_eth_dev *dev,
19 uint32_t meter_id, struct rte_mtr_error *error);
22 * Create the meter action.
25 * Pointer to mlx5_priv.
27 * Pointer to flow meter to be converted.
30 * Pointer to the meter action on success, NULL otherwise.
33 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
34 struct mlx5_flow_meter_info *fm)
36 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
37 struct mlx5dv_dr_flow_meter_attr mtr_init;
38 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
39 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
40 &fm->profile->srtcm_prm;
41 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
42 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
44 enum mlx5_meter_domain domain =
45 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
46 fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
47 MLX5_MTR_DOMAIN_INGRESS;
48 struct mlx5_flow_meter_def_policy *def_policy =
49 priv->sh->mtrmng->def_policy[domain];
51 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
52 MLX5_SET(flow_meter_parameters, fmp, valid, 1);
53 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
54 MLX5_SET(flow_meter_parameters, fmp,
55 start_color, MLX5_FLOW_COLOR_GREEN);
56 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
57 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
58 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
59 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
60 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
61 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
62 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
63 val = (cbs_cir & ASO_DSEG_MAN_MASK);
64 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
65 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
66 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
67 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
68 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
69 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
70 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
71 mtr_init.flow_meter_parameter = fmp;
72 mtr_init.flow_meter_parameter_sz =
73 MLX5_ST_SZ_BYTES(flow_meter_parameters);
74 mtr_init.active = fm->active_state;
75 return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
84 * Find meter profile by id.
87 * Pointer to mlx5_priv.
88 * @param meter_profile_id
92 * Pointer to the profile found on success, NULL otherwise.
94 static struct mlx5_flow_meter_profile *
95 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
97 struct mlx5_flow_meter_profile *fmp;
98 union mlx5_l3t_data data;
101 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
102 meter_profile_id, &data) || !data.ptr)
105 /* Remove reference taken by the mlx5_l3t_get_entry. */
106 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
108 if (!ret || ret == -1)
114 * Validate the MTR profile.
117 * Pointer to Ethernet device.
118 * @param[in] meter_profile_id
121 * Pointer to meter profile detail.
123 * Pointer to the error structure.
126 * 0 on success, a negative errno value otherwise and rte_errno is set.
129 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
130 uint32_t meter_profile_id,
131 struct rte_mtr_meter_profile *profile,
132 struct rte_mtr_error *error)
134 struct mlx5_priv *priv = dev->data->dev_private;
135 struct mlx5_flow_meter_profile *fmp;
142 /* Profile must not be NULL. */
144 return -rte_mtr_error_set(error, EINVAL,
145 RTE_MTR_ERROR_TYPE_METER_PROFILE,
146 NULL, "Meter profile is null.");
147 /* Meter profile ID must be valid. */
148 if (meter_profile_id == UINT32_MAX)
149 return -rte_mtr_error_set(error, EINVAL,
150 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
151 NULL, "Meter profile id not valid.");
152 /* Meter profile must not exist. */
153 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
155 return -rte_mtr_error_set(error, EEXIST,
156 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
158 "Meter profile already exists.");
159 if (!priv->sh->meter_aso_en) {
160 /* Old version is even not supported. */
161 if (!priv->sh->cdev->config.hca_attr.qos.flow_meter_old)
162 return -rte_mtr_error_set(error, ENOTSUP,
163 RTE_MTR_ERROR_TYPE_METER_PROFILE,
164 NULL, "Metering is not supported.");
165 /* Old FW metering only supports srTCM. */
166 if (profile->alg != RTE_MTR_SRTCM_RFC2697) {
167 return -rte_mtr_error_set(error, ENOTSUP,
168 RTE_MTR_ERROR_TYPE_METER_PROFILE,
169 NULL, "Metering algorithm is not supported.");
170 } else if (profile->srtcm_rfc2697.ebs) {
171 /* EBS is not supported for old metering. */
172 return -rte_mtr_error_set(error, ENOTSUP,
173 RTE_MTR_ERROR_TYPE_METER_PROFILE,
174 NULL, "EBS is not supported.");
176 if (profile->packet_mode)
177 return -rte_mtr_error_set(error, ENOTSUP,
178 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
179 "Metering algorithm packet mode is not supported.");
181 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0;
182 switch (profile->alg) {
183 case RTE_MTR_SRTCM_RFC2697:
184 cir = profile->srtcm_rfc2697.cir << ls_factor;
185 cbs = profile->srtcm_rfc2697.cbs << ls_factor;
186 ebs = profile->srtcm_rfc2697.ebs << ls_factor;
187 /* EBS could be zero for old metering. */
188 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
189 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
190 ebs <= MLX5_SRTCM_XBS_MAX) {
193 ret = -rte_mtr_error_set(error, ENOTSUP,
194 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
195 "Profile values out of range.");
198 case RTE_MTR_TRTCM_RFC2698:
199 cir = profile->trtcm_rfc2698.cir << ls_factor;
200 cbs = profile->trtcm_rfc2698.cbs << ls_factor;
201 pir = profile->trtcm_rfc2698.pir << ls_factor;
202 pbs = profile->trtcm_rfc2698.pbs << ls_factor;
203 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
204 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
205 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) &&
206 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) {
209 ret = -rte_mtr_error_set(error, ENOTSUP,
210 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
211 "Profile values out of range.");
214 case RTE_MTR_TRTCM_RFC4115:
215 cir = profile->trtcm_rfc4115.cir << ls_factor;
216 cbs = profile->trtcm_rfc4115.cbs << ls_factor;
217 eir = profile->trtcm_rfc4115.eir << ls_factor;
218 ebs = profile->trtcm_rfc4115.ebs << ls_factor;
219 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
220 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
221 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) {
224 ret = -rte_mtr_error_set(error, ENOTSUP,
225 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
226 "Profile values out of range.");
230 ret = -rte_mtr_error_set(error, ENOTSUP,
231 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
232 "Unknown metering algorithm.");
239 * Calculate mantissa and exponent for cir / eir.
242 * Value to be calculated.
244 * Pointer to the mantissa.
246 * Pointer to the exp.
249 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp)
252 int64_t delta = INT64_MAX;
257 /* Special case xir == 0 ? both exp and mantissa are 0. */
263 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
264 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
265 _xir = (1000000000ULL * m) >> e;
266 if (llabs(xir - _xir) <= delta) {
267 delta = llabs(xir - _xir);
278 * Calculate mantissa and exponent for xbs.
281 * Value to be calculated.
283 * Pointer to the mantissa.
285 * Pointer to the exp.
288 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
293 /* Special case xbs == 0 ? both exp and mantissa are 0. */
299 /* xbs = xbs_mantissa * 2^xbs_exponent */
300 _man = frexp(xbs, &_exp);
301 if (_exp >= MLX5_MAN_WIDTH) {
302 _man = _man * pow(2, MLX5_MAN_WIDTH);
303 _exp = _exp - MLX5_MAN_WIDTH;
305 *man = (uint8_t)ceil(_man);
310 * Fill the prm meter parameter.
313 * Pointer to meter profile to be converted.
315 * Pointer to the error structure.
318 * 0 on success, a negative errno value otherwise and rte_errno is set.
321 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
322 struct rte_mtr_error *error)
324 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
326 uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
327 uint32_t eir_exp, eir_man, ebs_exp, ebs_man;
328 uint64_t cir, cbs, eir, ebs;
330 switch (fmp->profile.alg) {
331 case RTE_MTR_SRTCM_RFC2697:
332 cir = fmp->profile.srtcm_rfc2697.cir;
333 cbs = fmp->profile.srtcm_rfc2697.cbs;
335 ebs = fmp->profile.srtcm_rfc2697.ebs;
337 case RTE_MTR_TRTCM_RFC2698:
338 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir >
339 fmp->profile.trtcm_rfc2698.cir &&
340 fmp->profile.trtcm_rfc2698.pbs >
341 fmp->profile.trtcm_rfc2698.cbs);
342 cir = fmp->profile.trtcm_rfc2698.cir;
343 cbs = fmp->profile.trtcm_rfc2698.cbs;
344 /* EIR / EBS are filled with PIR / PBS. */
345 eir = fmp->profile.trtcm_rfc2698.pir;
346 ebs = fmp->profile.trtcm_rfc2698.pbs;
348 case RTE_MTR_TRTCM_RFC4115:
349 cir = fmp->profile.trtcm_rfc4115.cir;
350 cbs = fmp->profile.trtcm_rfc4115.cbs;
351 eir = fmp->profile.trtcm_rfc4115.eir;
352 ebs = fmp->profile.trtcm_rfc4115.ebs;
355 return -rte_mtr_error_set(error, EINVAL,
356 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
357 "Metering algorithm mode is invalid");
359 /* Adjust the values for PPS mode. */
360 if (fmp->profile.packet_mode) {
361 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
362 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
363 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
364 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
366 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
367 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp);
368 /* Check if cir mantissa is too large. */
369 if (exp > ASO_DSEG_XIR_EXP_MASK)
370 return -rte_mtr_error_set(error, ENOTSUP,
371 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
372 "meter profile parameter cir is not supported.");
375 /* cbs = cbs_mantissa * 2^cbs_exponent */
376 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
377 /* Check if cbs mantissa is too large. */
378 if (exp > ASO_DSEG_EXP_MASK)
379 return -rte_mtr_error_set(error, ENOTSUP,
380 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
381 "meter profile parameter cbs is not supported.");
384 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
385 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
386 cir_exp << ASO_DSEG_XIR_EXP_OFFSET |
388 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp);
389 /* Check if eir mantissa is too large. */
390 if (exp > ASO_DSEG_XIR_EXP_MASK)
391 return -rte_mtr_error_set(error, ENOTSUP,
392 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
393 "meter profile parameter eir is not supported.");
396 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
397 /* Check if ebs mantissa is too large. */
398 if (exp > ASO_DSEG_EXP_MASK)
399 return -rte_mtr_error_set(error, ENOTSUP,
400 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
401 "meter profile parameter ebs is not supported.");
404 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
405 ebs_man << ASO_DSEG_EBS_MAN_OFFSET |
406 eir_exp << ASO_DSEG_XIR_EXP_OFFSET |
416 * Callback to get MTR capabilities.
419 * Pointer to Ethernet device.
421 * Pointer to save MTR capabilities.
423 * Pointer to the error structure.
426 * 0 on success, a negative errno value otherwise and rte_errno is set.
429 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
430 struct rte_mtr_capabilities *cap,
431 struct rte_mtr_error *error __rte_unused)
433 struct mlx5_priv *priv = dev->data->dev_private;
434 struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos;
437 return -rte_mtr_error_set(error, ENOTSUP,
438 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
439 "Meter is not supported");
440 memset(cap, 0, sizeof(*cap));
441 if (priv->sh->meter_aso_en) {
442 /* 2 meters per one ASO cache line. */
443 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
444 cap->srtcm_rfc2697_packet_mode_supported = 1;
445 cap->trtcm_rfc2698_packet_mode_supported = 1;
446 cap->trtcm_rfc4115_packet_mode_supported = 1;
448 cap->n_max = 1 << qattr->log_max_flow_meter;
450 cap->srtcm_rfc2697_byte_mode_supported = 1;
451 cap->trtcm_rfc2698_byte_mode_supported = 1;
452 cap->trtcm_rfc4115_byte_mode_supported = 1;
453 cap->n_shared_max = cap->n_max;
455 cap->shared_identical = 1;
456 cap->shared_n_flows_per_mtr_max = 4 << 20;
457 /* 2M flows can share the same meter. */
458 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
459 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
460 cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0;
461 cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0;
462 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
463 cap->meter_policy_n_max = cap->n_max;
464 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
465 RTE_MTR_STATS_N_PKTS_DROPPED;
470 * Callback to add MTR profile.
473 * Pointer to Ethernet device.
474 * @param[in] meter_profile_id
477 * Pointer to meter profile detail.
479 * Pointer to the error structure.
482 * 0 on success, a negative errno value otherwise and rte_errno is set.
485 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
486 uint32_t meter_profile_id,
487 struct rte_mtr_meter_profile *profile,
488 struct rte_mtr_error *error)
490 struct mlx5_priv *priv = dev->data->dev_private;
491 struct mlx5_flow_meter_profile *fmp;
492 union mlx5_l3t_data data;
496 return -rte_mtr_error_set(error, ENOTSUP,
497 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
498 "Meter is not supported");
499 /* Check input params. */
500 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
504 /* Meter profile memory allocation. */
505 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
506 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
508 return -rte_mtr_error_set(error, ENOMEM,
509 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
510 NULL, "Meter profile memory "
512 /* Fill profile info. */
513 fmp->id = meter_profile_id;
514 fmp->profile = *profile;
515 /* Fill the flow meter parameters for the PRM. */
516 ret = mlx5_flow_meter_param_fill(fmp, error);
520 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
521 meter_profile_id, &data);
523 return -rte_mtr_error_set(error, ENOTSUP,
524 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
525 NULL, "Meter profile insert fail.");
533 * Callback to delete MTR profile.
536 * Pointer to Ethernet device.
537 * @param[in] meter_profile_id
540 * Pointer to the error structure.
543 * 0 on success, a negative errno value otherwise and rte_errno is set.
546 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
547 uint32_t meter_profile_id,
548 struct rte_mtr_error *error)
550 struct mlx5_priv *priv = dev->data->dev_private;
551 struct mlx5_flow_meter_profile *fmp;
554 return -rte_mtr_error_set(error, ENOTSUP,
555 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
556 "Meter is not supported");
557 /* Meter profile must exist. */
558 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
560 return -rte_mtr_error_set(error, ENOENT,
561 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
563 "Meter profile id is invalid.");
564 /* Check profile is unused. */
566 return -rte_mtr_error_set(error, EBUSY,
567 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
568 NULL, "Meter profile is in use.");
569 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
570 return -rte_mtr_error_set(error, EBUSY,
571 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
572 NULL, "Meter profile remove fail.");
581 * Pointer to Ethernet device.
586 * Pointer to the policy found on success, NULL otherwise.
588 struct mlx5_flow_meter_policy *
589 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
591 uint32_t *policy_idx)
593 struct mlx5_priv *priv = dev->data->dev_private;
594 struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
595 union mlx5_l3t_data data;
597 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
599 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
603 *policy_idx = data.dword;
604 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
606 /* Remove reference taken by the mlx5_l3t_get_entry. */
607 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
609 if (sub_policy->main_policy_id)
610 return sub_policy->main_policy;
615 * Get the next meter from one meter's policy in hierarchy chain.
616 * Lock free, mutex should be acquired by caller.
619 * Pointer to mlx5_priv.
621 * Pointer to flow meter policy.
622 * @param[out] mtr_idx
623 * Pointer to Meter index.
626 * Pointer to the next meter, or NULL when fail.
628 struct mlx5_flow_meter_info *
629 mlx5_flow_meter_hierarchy_next_meter(struct mlx5_priv *priv,
630 struct mlx5_flow_meter_policy *policy,
635 for (i = 0; i < MLX5_MTR_RTE_COLORS; i++) {
636 if (policy->act_cnt[i].fate_action == MLX5_FLOW_FATE_MTR)
637 return mlx5_flow_meter_find(priv,
638 policy->act_cnt[i].next_mtr_id,
645 * Get the last meter's policy from one meter's policy in hierarchy.
648 * Pointer to Ethernet device.
650 * Pointer to flow meter policy.
653 * Pointer to the final meter's policy, or NULL when fail.
655 struct mlx5_flow_meter_policy *
656 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
657 struct mlx5_flow_meter_policy *policy)
659 struct mlx5_priv *priv = dev->data->dev_private;
660 struct mlx5_flow_meter_info *next_fm;
661 struct mlx5_flow_meter_policy *next_policy = policy;
663 while (next_policy->is_hierarchy) {
664 rte_spinlock_lock(&next_policy->sl);
665 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, next_policy, NULL);
666 rte_spinlock_unlock(&next_policy->sl);
667 if (!next_fm || next_fm->def_policy)
669 next_policy = mlx5_flow_meter_policy_find(dev,
670 next_fm->policy_id, NULL);
671 MLX5_ASSERT(next_policy);
677 * Callback to check MTR policy action validate
680 * Pointer to Ethernet device.
682 * Pointer to meter policy action detail.
684 * Pointer to the error structure.
687 * 0 on success, a negative errno value otherwise and rte_errno is set.
690 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
691 struct rte_mtr_meter_policy_params *policy,
692 struct rte_mtr_error *error)
694 struct mlx5_priv *priv = dev->data->dev_private;
695 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ?
699 uint8_t domain_bitmap;
702 if (!priv->mtr_en || !priv->sh->meter_aso_en)
703 return -rte_mtr_error_set(error, ENOTSUP,
704 RTE_MTR_ERROR_TYPE_METER_POLICY,
705 NULL, "meter policy unsupported.");
706 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
707 &is_rss, &domain_bitmap, &policy_mode, error);
714 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
716 struct mlx5_flow_meter_policy *mtr_policy,
717 struct rte_mtr_error *error,
720 struct mlx5_priv *priv = dev->data->dev_private;
721 struct mlx5_flow_meter_sub_policy *sub_policy;
723 uint16_t sub_policy_num;
725 rte_spinlock_lock(&mtr_policy->sl);
726 if (mtr_policy->ref_cnt) {
727 rte_spinlock_unlock(&mtr_policy->sl);
728 return -rte_mtr_error_set(error, EBUSY,
729 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
731 "Meter policy object is being used.");
733 mlx5_flow_destroy_policy_rules(dev, mtr_policy);
734 mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
735 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
736 sub_policy_num = (mtr_policy->sub_policy_num >>
737 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
738 MLX5_MTR_SUB_POLICY_NUM_MASK;
739 if (sub_policy_num) {
740 for (j = 0; j < sub_policy_num; j++) {
741 sub_policy = mtr_policy->sub_policys[i][j];
744 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
749 if (priv->policy_idx_tbl && clear_l3t) {
750 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
751 rte_spinlock_unlock(&mtr_policy->sl);
752 return -rte_mtr_error_set(error, ENOTSUP,
753 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
754 "Fail to delete policy in index table.");
757 rte_spinlock_unlock(&mtr_policy->sl);
762 * Callback to add MTR policy.
765 * Pointer to Ethernet device.
766 * @param[out] policy_id
767 * Pointer to policy id
769 * Pointer to meter policy action detail.
771 * Pointer to the error structure.
774 * 0 on success, a negative errno value otherwise and rte_errno is set.
777 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
779 struct rte_mtr_meter_policy_params *policy,
780 struct rte_mtr_error *error)
782 struct mlx5_priv *priv = dev->data->dev_private;
783 struct rte_flow_attr attr = { .transfer = priv->sh->config.dv_esw_en ?
785 uint32_t sub_policy_idx = 0;
786 uint32_t policy_idx = 0;
787 struct mlx5_flow_meter_policy *mtr_policy = NULL;
788 struct mlx5_flow_meter_sub_policy *sub_policy;
793 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
794 uint16_t sub_policy_num;
795 uint8_t domain_bitmap = 0;
796 union mlx5_l3t_data data;
797 bool skip_rule = false;
800 return -rte_mtr_error_set(error, ENOTSUP,
801 RTE_MTR_ERROR_TYPE_METER_POLICY,
802 NULL, "meter policy unsupported. ");
803 if (policy_id == MLX5_INVALID_POLICY_ID)
804 return -rte_mtr_error_set(error, ENOTSUP,
805 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
806 NULL, "policy ID is invalid. ");
807 if (policy_id == priv->sh->mtrmng->def_policy_id)
808 return -rte_mtr_error_set(error, EEXIST,
809 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
810 NULL, "default policy ID exists. ");
811 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
813 return -rte_mtr_error_set(error, EEXIST,
814 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
815 NULL, "policy ID exists. ");
816 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
817 &is_rss, &domain_bitmap,
818 &policy_mode, error);
822 return -rte_mtr_error_set(error, ENOTSUP,
823 RTE_MTR_ERROR_TYPE_METER_POLICY,
824 NULL, "fail to find policy domain.");
825 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
826 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
827 return -rte_mtr_error_set(error, EEXIST,
828 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
829 NULL, "a policy with similar actions "
830 "is already configured");
831 if (mlx5_flow_create_def_policy(dev))
832 return -rte_mtr_error_set(error, ENOTSUP,
833 RTE_MTR_ERROR_TYPE_METER_POLICY,
835 "fail to create non-terminated policy.");
836 priv->sh->mtrmng->def_policy_id = policy_id;
839 if (!priv->sh->meter_aso_en)
840 return -rte_mtr_error_set(error, ENOTSUP,
841 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
842 "no ASO capability to support the policy ");
843 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
844 if (!(domain_bitmap & (1 << i)))
847 * If RSS is found, it means that only the ingress domain can
848 * be supported. It is invalid to support RSS for one color
849 * and egress / transfer domain actions for another. Drop and
850 * jump action should have no impact.
854 sizeof(struct mlx5_flow_meter_sub_policy *) *
855 MLX5_MTR_RSS_MAX_SUB_POLICY;
858 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
860 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
861 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
863 return -rte_mtr_error_set(error, ENOMEM,
864 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
865 "Memory alloc failed for meter policy.");
866 if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
867 mtr_policy->skip_y = 1;
868 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
869 mtr_policy->skip_g = 1;
870 policy_size = sizeof(struct mlx5_flow_meter_policy);
871 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
872 if (!(domain_bitmap & (1 << i)))
874 if (i == MLX5_MTR_DOMAIN_INGRESS)
875 mtr_policy->ingress = 1;
876 if (i == MLX5_MTR_DOMAIN_EGRESS)
877 mtr_policy->egress = 1;
878 if (i == MLX5_MTR_DOMAIN_TRANSFER)
879 mtr_policy->transfer = 1;
880 sub_policy = mlx5_ipool_zmalloc
881 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
884 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
886 sub_policy->idx = sub_policy_idx;
887 sub_policy->main_policy = mtr_policy;
889 policy_idx = sub_policy_idx;
890 sub_policy->main_policy_id = 1;
892 mtr_policy->sub_policys[i] =
893 (struct mlx5_flow_meter_sub_policy **)
894 ((uint8_t *)mtr_policy + policy_size);
895 mtr_policy->sub_policys[i][0] = sub_policy;
896 sub_policy_num = (mtr_policy->sub_policy_num >>
897 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
898 MLX5_MTR_SUB_POLICY_NUM_MASK;
900 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
901 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
902 mtr_policy->sub_policy_num |=
903 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
904 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
906 * If RSS is found, it means that only the ingress domain can
907 * be supported. It is invalid to support RSS for one color
908 * and egress / transfer domain actions for another. Drop and
909 * jump action should have no impact.
912 mtr_policy->is_rss = 1;
915 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
917 rte_spinlock_init(&mtr_policy->sl);
918 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
919 policy->actions, error);
922 if (mtr_policy->is_hierarchy) {
923 struct mlx5_flow_meter_policy *final_policy;
926 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
929 skip_rule = (final_policy->is_rss || final_policy->is_queue);
932 * If either Green or Yellow has queue / RSS action, all the policy
933 * rules will be created later in the flow splitting stage.
935 if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
936 /* Create policy rules in HW. */
937 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
941 data.dword = policy_idx;
942 if (!priv->policy_idx_tbl) {
943 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
944 if (!priv->policy_idx_tbl)
947 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
952 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
953 mtr_policy, error, false);
954 mlx5_free(mtr_policy);
958 return -rte_mtr_error_set(error, ENOTSUP,
959 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
960 NULL, "Failed to create devx policy.");
964 * Callback to delete MTR policy.
967 * Pointer to Ethernet device.
968 * @param[in] policy_id
971 * Pointer to the error structure.
974 * 0 on success, a negative errno value otherwise and rte_errno is set.
977 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
979 struct rte_mtr_error *error)
981 struct mlx5_priv *priv = dev->data->dev_private;
982 struct mlx5_flow_meter_policy *mtr_policy;
986 if (policy_id == priv->sh->mtrmng->def_policy_id) {
987 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
988 return -rte_mtr_error_set(error, ENOTSUP,
989 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
990 "Meter policy object is being used.");
991 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
994 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
996 return -rte_mtr_error_set(error, ENOTSUP,
997 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
998 "Meter policy id is invalid. ");
999 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
1003 mlx5_free(mtr_policy);
1008 * Check meter validation.
1011 * Pointer to mlx5 private data structure.
1012 * @param[in] meter_id
1015 * Pointer to rte meter parameters.
1017 * Pointer to rte meter error structure.
1020 * 0 on success, a negative errno value otherwise and rte_errno is set.
1023 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
1024 struct rte_mtr_params *params,
1025 struct rte_mtr_error *error)
1027 /* Meter must use global drop action. */
1028 if (!priv->sh->dr_drop_action)
1029 return -rte_mtr_error_set(error, ENOTSUP,
1030 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1032 "No drop action ready for meter.");
1033 /* Meter params must not be NULL. */
1035 return -rte_mtr_error_set(error, EINVAL,
1036 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1037 NULL, "Meter object params null.");
1038 /* Previous meter color is not supported. */
1039 if (params->use_prev_mtr_color && !priv->sh->meter_aso_en)
1040 return -rte_mtr_error_set(error, ENOTSUP,
1041 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1043 "Previous meter color "
1045 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
1046 return -rte_mtr_error_set(error, ENOENT,
1047 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1048 NULL, "Meter policy id not valid.");
1049 /* Validate meter id. */
1050 if (mlx5_flow_meter_find(priv, meter_id, NULL))
1051 return -rte_mtr_error_set(error, EEXIST,
1052 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
1053 "Meter object already exists.");
1058 * Modify the flow meter action.
1061 * Pointer to mlx5 private data structure.
1063 * Pointer to flow meter to be modified.
1065 * Pointer to meter srtcm description parameter.
1066 * @param[in] modify_bits
1067 * The bit in srtcm to be updated.
1068 * @param[in] active_state
1069 * The state to be updated.
1071 * 0 on success, o negative value otherwise.
1074 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
1075 struct mlx5_flow_meter_info *fm,
1076 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
1077 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
1079 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1080 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
1082 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
1084 struct mlx5_aso_mtr *aso_mtr = NULL;
1085 uint32_t cbs_cir, ebs_eir, val;
1087 if (priv->sh->meter_aso_en) {
1088 fm->is_enable = !!is_enable;
1089 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1090 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1093 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
1097 /* Fill command parameters. */
1098 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
1099 mod_attr.flow_meter_parameter = in;
1100 mod_attr.flow_meter_parameter_sz =
1101 MLX5_ST_SZ_BYTES(flow_meter_parameters);
1102 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1103 mod_attr.active = !!active_state;
1105 mod_attr.active = 0;
1107 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
1108 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1109 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1110 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1112 MLX5_SET(flow_meter_parameters, attr,
1114 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1116 MLX5_SET(flow_meter_parameters, attr,
1119 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1120 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) &
1122 MLX5_SET(flow_meter_parameters, attr,
1124 val = cbs_cir & ASO_DSEG_MAN_MASK;
1125 MLX5_SET(flow_meter_parameters, attr,
1128 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1129 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1131 MLX5_SET(flow_meter_parameters, attr,
1133 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1135 MLX5_SET(flow_meter_parameters, attr,
1138 /* Apply modifications to meter only if it was created. */
1139 if (fm->meter_action_g) {
1140 ret = mlx5_glue->dv_modify_flow_action_meter
1141 (fm->meter_action_g, &mod_attr,
1142 rte_cpu_to_be_64(modify_bits));
1146 /* Update succeeded modify meter parameters. */
1147 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1148 fm->active_state = !!active_state;
1163 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1164 struct mlx5_flow_meter_info *fm,
1165 uint64_t stats_mask)
1168 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1169 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1170 if (fm->bytes_dropped || fm->pkts_dropped) {
1171 if (!fm->drop_cnt) {
1172 /* Alloc policer counters. */
1173 fm->drop_cnt = mlx5_counter_alloc(dev);
1179 mlx5_counter_free(dev, fm->drop_cnt);
1187 * Create meter rules.
1190 * Pointer to Ethernet device.
1191 * @param[in] meter_id
1194 * Pointer to rte meter parameters.
1196 * Meter shared with other flow or not.
1198 * Pointer to rte meter error structure.
1201 * 0 on success, a negative errno value otherwise and rte_errno is set.
1204 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1205 struct rte_mtr_params *params, int shared,
1206 struct rte_mtr_error *error)
1208 struct mlx5_priv *priv = dev->data->dev_private;
1209 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1210 struct mlx5_flow_meter_profile *fmp;
1211 struct mlx5_flow_meter_info *fm;
1212 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */
1213 struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1214 struct mlx5_flow_meter_policy *mtr_policy = NULL;
1215 struct mlx5_indexed_pool_config flow_ipool_cfg = {
1219 .type = "mlx5_flow_mtr_flow_id_pool",
1221 struct mlx5_aso_mtr *aso_mtr;
1222 uint32_t mtr_idx, policy_idx;
1223 union mlx5_l3t_data data;
1225 uint8_t domain_bitmap;
1226 uint8_t mtr_id_bits;
1227 uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1228 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1231 return -rte_mtr_error_set(error, ENOTSUP,
1232 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1233 "Meter is not supported");
1234 /* Validate the parameters. */
1235 ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1238 /* Meter profile must exist. */
1239 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1241 return -rte_mtr_error_set(error, ENOENT,
1242 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1243 NULL, "Meter profile id not valid.");
1244 /* Meter policy must exist. */
1245 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1247 (&priv->sh->mtrmng->def_policy_ref_cnt,
1248 1, __ATOMIC_RELAXED);
1249 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1250 if (!priv->sh->config.dv_esw_en)
1251 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1253 if (!priv->sh->meter_aso_en)
1254 return -rte_mtr_error_set(error, ENOTSUP,
1255 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1256 "Part of the policies cannot be "
1257 "supported without ASO ");
1258 mtr_policy = mlx5_flow_meter_policy_find(dev,
1259 params->meter_policy_id, &policy_idx);
1261 return -rte_mtr_error_set(error, ENOENT,
1262 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1263 NULL, "Meter policy id not valid.");
1264 domain_bitmap = (mtr_policy->ingress ?
1265 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1266 (mtr_policy->egress ?
1267 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1268 (mtr_policy->transfer ?
1269 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1270 if (fmp->g_support && mtr_policy->skip_g)
1271 return -rte_mtr_error_set(error, ENOTSUP,
1272 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1273 NULL, "Meter green policy is empty.");
1274 if (fmp->y_support && mtr_policy->skip_y)
1275 return -rte_mtr_error_set(error, ENOTSUP,
1276 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1277 NULL, "Meter yellow policy is empty.");
1279 /* Allocate the flow meter memory. */
1280 if (priv->sh->meter_aso_en) {
1281 mtr_idx = mlx5_flow_mtr_alloc(dev);
1283 return -rte_mtr_error_set(error, ENOMEM,
1284 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1285 "Memory alloc failed for meter.");
1286 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1290 return -rte_mtr_error_set(error, ENOMEM,
1291 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1292 "Unsupported profile with yellow.");
1293 legacy_fm = mlx5_ipool_zmalloc
1294 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1295 if (legacy_fm == NULL)
1296 return -rte_mtr_error_set(error, ENOMEM,
1297 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1298 "Memory alloc failed for meter.");
1299 legacy_fm->idx = mtr_idx;
1300 fm = &legacy_fm->fm;
1302 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1303 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1305 DRV_LOG(ERR, "Meter number exceeds max limit.");
1308 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1309 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1310 /* Fill the flow meter parameters. */
1311 fm->meter_id = meter_id;
1312 fm->policy_id = params->meter_policy_id;
1314 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1316 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1318 /* Add to the flow meter list. */
1319 if (!priv->sh->meter_aso_en) {
1320 MLX5_ASSERT(legacy_fm != NULL);
1321 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1323 /* Add to the flow meter list. */
1324 fm->active_state = 1; /* Config meter starts as active. */
1325 fm->is_enable = params->meter_enable;
1326 fm->shared = !!shared;
1327 fm->color_aware = !!params->use_prev_mtr_color;
1328 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1329 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1331 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1332 if (!fm->flow_ipool)
1335 rte_spinlock_init(&fm->sl);
1336 /* If ASO meter supported, update ASO flow meter by wqe. */
1337 if (priv->sh->meter_aso_en) {
1338 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1339 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1342 if (!priv->mtr_idx_tbl) {
1344 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1345 if (!priv->mtr_idx_tbl)
1348 data.dword = mtr_idx;
1349 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1351 } else if (!params->meter_enable && mlx5_flow_meter_disable(dev, meter_id, error)) {
1354 fm->active_state = params->meter_enable;
1356 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1359 mlx5_flow_destroy_mtr_tbls(dev, fm);
1360 /* Free policer counters. */
1362 mlx5_counter_free(dev, fm->drop_cnt);
1363 if (priv->sh->meter_aso_en)
1364 mlx5_flow_mtr_free(dev, mtr_idx);
1366 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1367 return -rte_mtr_error_set(error, ENOTSUP,
1368 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1369 NULL, "Failed to create devx meter.");
1373 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1374 struct mlx5_flow_meter_info *fm,
1377 struct mlx5_priv *priv = dev->data->dev_private;
1378 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1379 struct mlx5_flow_meter_profile *fmp;
1380 struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1381 struct mlx5_flow_meter_policy *mtr_policy;
1383 /* Meter object must not have any owner. */
1384 MLX5_ASSERT(!fm->ref_cnt);
1385 /* Get meter profile. */
1389 /* Update dependencies. */
1390 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1392 /* Remove from list. */
1393 if (!priv->sh->meter_aso_en) {
1394 legacy_fm = container_of(fm,
1395 struct mlx5_legacy_flow_meter, fm);
1396 TAILQ_REMOVE(fms, legacy_fm, next);
1398 /* Free drop counters. */
1400 mlx5_counter_free(dev, fm->drop_cnt);
1401 /* Free meter flow table. */
1402 if (fm->flow_ipool) {
1403 mlx5_ipool_destroy(fm->flow_ipool);
1406 mlx5_flow_destroy_mtr_tbls(dev, fm);
1408 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1409 1, __ATOMIC_RELAXED);
1410 if (priv->sh->meter_aso_en) {
1411 if (!fm->def_policy) {
1412 mtr_policy = mlx5_flow_meter_policy_find(dev,
1413 fm->policy_id, NULL);
1415 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1416 1, __ATOMIC_RELAXED);
1420 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1422 mlx5_flow_mtr_free(dev, mtr_idx);
1424 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1431 * Destroy meter rules.
1434 * Pointer to Ethernet device.
1435 * @param[in] meter_id
1438 * Pointer to rte meter error structure.
1441 * 0 on success, a negative errno value otherwise and rte_errno is set.
1444 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1445 struct rte_mtr_error *error)
1447 struct mlx5_priv *priv = dev->data->dev_private;
1448 struct mlx5_flow_meter_info *fm;
1449 uint32_t mtr_idx = 0;
1452 return -rte_mtr_error_set(error, ENOTSUP,
1453 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1455 "Meter is not supported");
1456 /* Meter object must exist. */
1457 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1459 return -rte_mtr_error_set(error, ENOENT,
1460 RTE_MTR_ERROR_TYPE_MTR_ID,
1462 "Meter object id not valid.");
1463 /* Meter object must not have any owner. */
1464 if (fm->ref_cnt > 0)
1465 return -rte_mtr_error_set(error, EBUSY,
1466 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1468 "Meter object is being used.");
1469 /* Destroy the meter profile. */
1470 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1471 return -rte_mtr_error_set(error, EINVAL,
1472 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1474 "MTR object meter profile invalid.");
1479 * Modify meter state.
1482 * Pointer to mlx5 private data structure.
1484 * Pointer to flow meter.
1485 * @param[in] new_state
1486 * New state to update.
1488 * Pointer to rte meter error structure.
1491 * 0 on success, a negative errno value otherwise and rte_errno is set.
1494 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1495 struct mlx5_flow_meter_info *fm,
1497 struct rte_mtr_error *error)
1499 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1500 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1503 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1504 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1507 if (new_state == MLX5_FLOW_METER_DISABLE)
1508 ret = mlx5_flow_meter_action_modify(priv, fm,
1509 &srtcm, modify_bits, 0, 0);
1511 ret = mlx5_flow_meter_action_modify(priv, fm,
1512 &fm->profile->srtcm_prm,
1515 return -rte_mtr_error_set(error, -ret,
1516 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1519 "Failed to enable meter." :
1520 "Failed to disable meter.");
1525 * Callback to enable flow meter.
1528 * Pointer to Ethernet device.
1529 * @param[in] meter_id
1532 * Pointer to rte meter error structure.
1535 * 0 on success, a negative errno value otherwise and rte_errno is set.
1538 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1540 struct rte_mtr_error *error)
1542 struct mlx5_priv *priv = dev->data->dev_private;
1543 struct mlx5_flow_meter_info *fm;
1547 return -rte_mtr_error_set(error, ENOTSUP,
1548 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1549 "Meter is not supported");
1550 /* Meter object must exist. */
1551 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1553 return -rte_mtr_error_set(error, ENOENT,
1554 RTE_MTR_ERROR_TYPE_MTR_ID,
1555 NULL, "Meter not found.");
1556 if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1558 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1561 fm->active_state = MLX5_FLOW_METER_ENABLE;
1566 * Callback to disable flow meter.
1569 * Pointer to Ethernet device.
1570 * @param[in] meter_id
1573 * Pointer to rte meter error structure.
1576 * 0 on success, a negative errno value otherwise and rte_errno is set.
1579 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1581 struct rte_mtr_error *error)
1583 struct mlx5_priv *priv = dev->data->dev_private;
1584 struct mlx5_flow_meter_info *fm;
1588 return -rte_mtr_error_set(error, ENOTSUP,
1589 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1590 "Meter is not supported");
1591 /* Meter object must exist. */
1592 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1594 return -rte_mtr_error_set(error, ENOENT,
1595 RTE_MTR_ERROR_TYPE_MTR_ID,
1596 NULL, "Meter not found.");
1597 if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1599 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1602 fm->active_state = MLX5_FLOW_METER_DISABLE;
1607 * Callback to update meter profile.
1610 * Pointer to Ethernet device.
1611 * @param[in] meter_id
1613 * @param[in] meter_profile_id
1614 * To be updated meter profile id.
1616 * Pointer to rte meter error structure.
1619 * 0 on success, a negative errno value otherwise and rte_errno is set.
1622 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1624 uint32_t meter_profile_id,
1625 struct rte_mtr_error *error)
1627 struct mlx5_priv *priv = dev->data->dev_private;
1628 struct mlx5_flow_meter_profile *fmp;
1629 struct mlx5_flow_meter_profile *old_fmp;
1630 struct mlx5_flow_meter_info *fm;
1631 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1632 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1636 return -rte_mtr_error_set(error, ENOTSUP,
1637 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1638 "Meter is not supported");
1639 /* Meter profile must exist. */
1640 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1642 return -rte_mtr_error_set(error, ENOENT,
1643 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1644 NULL, "Meter profile not found.");
1645 /* Meter object must exist. */
1646 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1648 return -rte_mtr_error_set(error, ENOENT,
1649 RTE_MTR_ERROR_TYPE_MTR_ID,
1650 NULL, "Meter not found.");
1651 /* MTR object already set to meter profile id. */
1652 old_fmp = fm->profile;
1655 /* Update the profile. */
1657 /* Update meter params in HW (if not disabled). */
1658 if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1660 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1661 modify_bits, fm->active_state, 1);
1663 fm->profile = old_fmp;
1664 return -rte_mtr_error_set(error, -ret,
1665 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1666 NULL, "Failed to update meter"
1667 " parameters in hardware.");
1675 * Callback to update meter stats mask.
1678 * Pointer to Ethernet device.
1679 * @param[in] meter_id
1681 * @param[in] stats_mask
1682 * To be updated stats_mask.
1684 * Pointer to rte meter error structure.
1687 * 0 on success, a negative errno value otherwise and rte_errno is set.
1690 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1692 uint64_t stats_mask,
1693 struct rte_mtr_error *error)
1695 struct mlx5_priv *priv = dev->data->dev_private;
1696 struct mlx5_flow_meter_info *fm;
1699 return -rte_mtr_error_set(error, ENOTSUP,
1700 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1701 "Meter is not supported");
1702 /* Meter object must exist. */
1703 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1705 return -rte_mtr_error_set(error, ENOENT,
1706 RTE_MTR_ERROR_TYPE_MTR_ID,
1707 NULL, "Meter object id not valid.");
1708 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1709 return -rte_mtr_error_set(error, ENOENT,
1710 RTE_MTR_ERROR_TYPE_MTR_ID,
1711 NULL, "Fail to allocate "
1712 "counter for meter.");
1717 * Callback to read meter statistics.
1720 * Pointer to Ethernet device.
1721 * @param[in] meter_id
1724 * Pointer to store the statistics.
1725 * @param[out] stats_mask
1726 * Pointer to store the stats_mask.
1728 * Statistic to be cleared after read or not.
1730 * Pointer to rte meter error structure.
1733 * 0 on success, a negative errno value otherwise and rte_errno is set.
1736 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1738 struct rte_mtr_stats *stats,
1739 uint64_t *stats_mask,
1741 struct rte_mtr_error *error)
1743 struct mlx5_priv *priv = dev->data->dev_private;
1744 struct mlx5_flow_meter_info *fm;
1750 return -rte_mtr_error_set(error, ENOTSUP,
1751 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1752 "Meter is not supported");
1753 /* Meter object must exist. */
1754 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1756 return -rte_mtr_error_set(error, ENOENT,
1757 RTE_MTR_ERROR_TYPE_MTR_ID,
1758 NULL, "Meter object id not valid.");
1760 if (fm->bytes_dropped)
1761 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1762 if (fm->pkts_dropped)
1763 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1764 memset(stats, 0, sizeof(*stats));
1766 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1770 /* If need to read the packets, set it. */
1771 if (fm->pkts_dropped)
1772 stats->n_pkts_dropped = pkts;
1773 /* If need to read the bytes, set it. */
1774 if (fm->bytes_dropped)
1775 stats->n_bytes_dropped = bytes;
1779 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1780 "Failed to read meter drop counters.");
1783 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1784 .capabilities_get = mlx5_flow_mtr_cap_get,
1785 .meter_profile_add = mlx5_flow_meter_profile_add,
1786 .meter_profile_delete = mlx5_flow_meter_profile_delete,
1787 .meter_policy_validate = mlx5_flow_meter_policy_validate,
1788 .meter_policy_add = mlx5_flow_meter_policy_add,
1789 .meter_policy_delete = mlx5_flow_meter_policy_delete,
1790 .create = mlx5_flow_meter_create,
1791 .destroy = mlx5_flow_meter_destroy,
1792 .meter_enable = mlx5_flow_meter_enable,
1793 .meter_disable = mlx5_flow_meter_disable,
1794 .meter_profile_update = mlx5_flow_meter_profile_update,
1795 .meter_dscp_table_update = NULL,
1796 .stats_update = mlx5_flow_meter_stats_update,
1797 .stats_read = mlx5_flow_meter_stats_read,
1801 * Get meter operations.
1804 * Pointer to Ethernet device structure.
1806 * Pointer to set the mtr operations.
1812 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1814 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1822 * Pointer to mlx5_priv.
1826 * Pointer to Meter index.
1829 * Pointer to the meter info found on success, NULL otherwise.
1831 struct mlx5_flow_meter_info *
1832 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1835 struct mlx5_legacy_flow_meter *legacy_fm;
1836 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1837 struct mlx5_aso_mtr *aso_mtr;
1838 struct mlx5_aso_mtr_pools_mng *pools_mng =
1839 &priv->sh->mtrmng->pools_mng;
1840 union mlx5_l3t_data data;
1843 if (priv->sh->meter_aso_en) {
1844 rte_rwlock_read_lock(&pools_mng->resize_mtrwl);
1845 n_valid = pools_mng->n_valid;
1846 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl);
1847 if (!n_valid || !priv->mtr_idx_tbl ||
1848 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1852 *mtr_idx = data.dword;
1853 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1854 /* Remove reference taken by the mlx5_l3t_get_entry. */
1855 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1856 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1858 return &aso_mtr->fm;
1860 TAILQ_FOREACH(legacy_fm, fms, next)
1861 if (meter_id == legacy_fm->fm.meter_id) {
1863 *mtr_idx = legacy_fm->idx;
1864 return &legacy_fm->fm;
1870 * Find meter by index.
1873 * Pointer to mlx5_priv.
1878 * Pointer to the meter info found on success, NULL otherwise.
1880 struct mlx5_flow_meter_info *
1881 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1883 struct mlx5_aso_mtr *aso_mtr;
1885 if (priv->sh->meter_aso_en) {
1886 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1889 return &aso_mtr->fm;
1891 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1896 * Attach meter to flow.
1897 * Unidirectional Meter creation can only be done
1898 * when flow direction is known, i.e. when calling meter_attach.
1901 * Pointer to mlx5 private data.
1903 * Pointer to flow meter.
1905 * Pointer to flow attributes.
1906 * @param [out] error
1907 * Pointer to error structure.
1910 * 0 on success, a negative errno value otherwise and rte_errno is set.
1913 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1914 struct mlx5_flow_meter_info *fm,
1915 const struct rte_flow_attr *attr,
1916 struct rte_flow_error *error)
1920 if (priv->sh->meter_aso_en) {
1921 struct mlx5_aso_mtr *aso_mtr;
1923 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1924 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1925 return rte_flow_error_set(error, ENOENT,
1926 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1928 "Timeout in meter configuration");
1930 rte_spinlock_lock(&fm->sl);
1931 if (fm->shared || !fm->ref_cnt) {
1934 rte_flow_error_set(error, EINVAL,
1935 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1936 "Meter cannot be shared");
1939 rte_spinlock_unlock(&fm->sl);
1941 rte_spinlock_lock(&fm->sl);
1942 if (fm->meter_action_g) {
1944 attr->transfer == fm->transfer &&
1945 attr->ingress == fm->ingress &&
1946 attr->egress == fm->egress) {
1949 rte_flow_error_set(error, EINVAL,
1950 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1952 "Meter attr not match." :
1953 "Meter cannot be shared.");
1957 fm->ingress = attr->ingress;
1958 fm->egress = attr->egress;
1959 fm->transfer = attr->transfer;
1961 /* This also creates the meter object. */
1962 fm->meter_action_g = mlx5_flow_meter_action_create(priv,
1964 if (!fm->meter_action_g) {
1969 rte_flow_error_set(error, EINVAL,
1970 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1971 "Meter action create failed.");
1975 rte_spinlock_unlock(&fm->sl);
1977 return ret ? -rte_errno : 0;
1981 * Detach meter from flow.
1984 * Pointer to mlx5 private data.
1986 * Pointer to flow meter.
1989 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1990 struct mlx5_flow_meter_info *fm)
1992 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1993 rte_spinlock_lock(&fm->sl);
1994 MLX5_ASSERT(fm->ref_cnt);
1995 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1996 mlx5_glue->destroy_flow_action(fm->meter_action_g);
1997 fm->meter_action_g = NULL;
2002 rte_spinlock_unlock(&fm->sl);
2010 * Flush meter with Rx queue configuration.
2013 * Pointer to Ethernet device.
2016 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
2018 struct mlx5_priv *priv = dev->data->dev_private;
2019 struct mlx5_flow_meter_sub_policy *sub_policy;
2020 struct mlx5_flow_meter_policy *mtr_policy;
2022 uint32_t i, policy_idx;
2026 if (priv->policy_idx_tbl) {
2027 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2028 policy_idx = *(uint32_t *)entry;
2029 sub_policy = mlx5_ipool_get
2030 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2032 if (!sub_policy || !sub_policy->main_policy)
2034 mtr_policy = sub_policy->main_policy;
2035 if (mtr_policy->is_queue || mtr_policy->is_rss)
2036 mlx5_flow_destroy_sub_policy_with_rxq(dev,
2043 * Iterate a meter hierarchy and flush all meters and policies if possible.
2046 * Pointer to Ethernet device.
2048 * Pointer to flow meter.
2049 * @param[in] mtr_idx
2052 * Pointer to rte meter error structure.
2055 * 0 on success, a negative errno value otherwise and rte_errno is set.
2058 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
2059 struct mlx5_flow_meter_info *fm,
2061 struct rte_mtr_error *error)
2063 struct mlx5_priv *priv = dev->data->dev_private;
2064 struct mlx5_flow_meter_policy *policy;
2066 struct mlx5_flow_meter_info *next_fm;
2067 uint32_t next_mtr_idx;
2068 struct mlx5_flow_meter_policy *next_policy = NULL;
2070 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
2071 MLX5_ASSERT(policy);
2072 while (!fm->ref_cnt && policy->is_hierarchy) {
2073 policy_id = fm->policy_id;
2074 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &next_mtr_idx);
2076 next_policy = mlx5_flow_meter_policy_find(dev,
2079 MLX5_ASSERT(next_policy);
2081 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
2082 return -rte_mtr_error_set(error, ENOTSUP,
2083 RTE_MTR_ERROR_TYPE_MTR_ID,
2085 "Failed to flush meter.");
2086 if (policy->ref_cnt)
2088 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
2089 policy, error, true))
2092 if (!next_fm || !next_policy)
2095 mtr_idx = next_mtr_idx;
2096 policy = next_policy;
2102 * Flush all the hierarchy meters and their policies.
2105 * Pointer to Ethernet device.
2107 * Pointer to rte meter error structure.
2110 * 0 on success, a negative errno value otherwise and rte_errno is set.
2113 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
2114 struct rte_mtr_error *error)
2116 struct mlx5_priv *priv = dev->data->dev_private;
2117 struct mlx5_flow_meter_info *fm;
2118 struct mlx5_flow_meter_policy *policy;
2119 struct mlx5_flow_meter_sub_policy *sub_policy;
2120 struct mlx5_flow_meter_info *next_fm;
2121 struct mlx5_aso_mtr *aso_mtr;
2122 uint32_t mtr_idx = 0;
2123 uint32_t i, policy_idx;
2126 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2128 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2129 mtr_idx = *(uint32_t *)entry;
2132 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2134 if (fm->ref_cnt || fm->def_policy)
2136 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2139 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2140 policy_idx = *(uint32_t *)entry;
2141 sub_policy = mlx5_ipool_get
2142 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2145 return -rte_mtr_error_set(error,
2147 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2148 NULL, "Meter policy invalid.");
2149 policy = sub_policy->main_policy;
2150 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2152 next_fm = mlx5_flow_meter_hierarchy_next_meter(priv, policy, &mtr_idx);
2153 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2155 return -rte_mtr_error_set(error,
2157 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2158 NULL, "Meter policy invalid.");
2160 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2162 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2169 * Flush meter configuration.
2172 * Pointer to Ethernet device.
2174 * Pointer to rte meter error structure.
2177 * 0 on success, a negative errno value otherwise and rte_errno is set.
2180 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2182 struct mlx5_priv *priv = dev->data->dev_private;
2183 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2184 struct mlx5_flow_meter_profile *fmp;
2185 struct mlx5_legacy_flow_meter *legacy_fm;
2186 struct mlx5_flow_meter_info *fm;
2187 struct mlx5_flow_meter_sub_policy *sub_policy;
2189 uint32_t i, mtr_idx, policy_idx;
2191 struct mlx5_aso_mtr *aso_mtr;
2195 if (priv->sh->meter_aso_en) {
2196 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2198 if (priv->mtr_idx_tbl) {
2199 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2200 mtr_idx = *(uint32_t *)entry;
2203 mlx5_aso_meter_by_idx(priv, mtr_idx);
2205 (void)mlx5_flow_meter_params_flush(dev,
2209 mlx5_l3t_destroy(priv->mtr_idx_tbl);
2210 priv->mtr_idx_tbl = NULL;
2213 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2214 fm = &legacy_fm->fm;
2215 if (mlx5_flow_meter_params_flush(dev, fm, 0))
2216 return -rte_mtr_error_set(error, EINVAL,
2217 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2218 NULL, "MTR object meter profile invalid.");
2221 if (priv->policy_idx_tbl) {
2222 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2223 policy_idx = *(uint32_t *)entry;
2224 sub_policy = mlx5_ipool_get
2225 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2228 return -rte_mtr_error_set(error,
2230 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2232 "meter policy invalid.");
2233 if (__mlx5_flow_meter_policy_delete(dev, i,
2234 sub_policy->main_policy,
2236 return -rte_mtr_error_set(error,
2238 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2240 "meter policy invalid.");
2241 mlx5_free(sub_policy->main_policy);
2243 mlx5_l3t_destroy(priv->policy_idx_tbl);
2244 priv->policy_idx_tbl = NULL;
2246 if (priv->mtr_profile_tbl) {
2247 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2249 if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2251 return -rte_mtr_error_set(error, EINVAL,
2252 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2253 NULL, "Fail to destroy "
2256 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2257 priv->mtr_profile_tbl = NULL;
2259 /* Delete default policy table. */
2260 mlx5_flow_destroy_def_policy(dev);
2261 if (priv->sh->refcnt == 1)
2262 mlx5_flow_destroy_mtr_drop_tbls(dev);