1 // SPDX-License-Identifier: BSD-3-Clause
3 * Copyright 2018 Mellanox Technologies, Ltd
8 #include <rte_malloc.h>
10 #include <rte_mtr_driver.h>
12 #include <mlx5_devx_cmds.h>
13 #include <mlx5_malloc.h>
16 #include "mlx5_flow.h"
19 * Create the meter action.
22 * Pointer to mlx5_priv.
24 * Pointer to flow meter to be converted.
27 * Pointer to the meter action on success, NULL otherwise.
30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
31 struct mlx5_flow_meter_info *fm)
33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
34 struct mlx5dv_dr_flow_meter_attr mtr_init;
35 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
36 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
37 &fm->profile->srtcm_prm;
38 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
39 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
41 enum mlx5_meter_domain domain =
42 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
43 fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
44 MLX5_MTR_DOMAIN_INGRESS;
45 struct mlx5_flow_meter_def_policy *def_policy =
46 priv->sh->mtrmng->def_policy[domain];
48 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
49 MLX5_SET(flow_meter_parameters, fmp, valid, 1);
50 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
51 MLX5_SET(flow_meter_parameters, fmp,
52 start_color, MLX5_FLOW_COLOR_GREEN);
53 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
54 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
55 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
56 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
57 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
58 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
59 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
60 val = (cbs_cir & ASO_DSEG_MAN_MASK);
61 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
62 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
63 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
64 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
65 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
66 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
67 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
68 mtr_init.flow_meter_parameter = fmp;
69 mtr_init.flow_meter_parameter_sz =
70 MLX5_ST_SZ_BYTES(flow_meter_parameters);
71 mtr_init.active = fm->active_state;
72 return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
81 * Find meter profile by id.
84 * Pointer to mlx5_priv.
85 * @param meter_profile_id
89 * Pointer to the profile found on success, NULL otherwise.
91 static struct mlx5_flow_meter_profile *
92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
94 struct mlx5_flow_meter_profile *fmp;
95 union mlx5_l3t_data data;
98 if (mlx5_l3t_get_entry(priv->mtr_profile_tbl,
99 meter_profile_id, &data) || !data.ptr)
102 /* Remove reference taken by the mlx5_l3t_get_entry. */
103 ret = mlx5_l3t_clear_entry(priv->mtr_profile_tbl,
105 if (!ret || ret == -1)
111 * Validate the MTR profile.
114 * Pointer to Ethernet device.
115 * @param[in] meter_profile_id
118 * Pointer to meter profile detail.
120 * Pointer to the error structure.
123 * 0 on success, a negative errno value otherwise and rte_errno is set.
126 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
127 uint32_t meter_profile_id,
128 struct rte_mtr_meter_profile *profile,
129 struct rte_mtr_error *error)
131 struct mlx5_priv *priv = dev->data->dev_private;
132 struct mlx5_flow_meter_profile *fmp;
139 /* Profile must not be NULL. */
141 return -rte_mtr_error_set(error, EINVAL,
142 RTE_MTR_ERROR_TYPE_METER_PROFILE,
143 NULL, "Meter profile is null.");
144 /* Meter profile ID must be valid. */
145 if (meter_profile_id == UINT32_MAX)
146 return -rte_mtr_error_set(error, EINVAL,
147 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
148 NULL, "Meter profile id not valid.");
149 /* Meter profile must not exist. */
150 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
152 return -rte_mtr_error_set(error, EEXIST,
153 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
155 "Meter profile already exists.");
156 if (!priv->sh->meter_aso_en) {
157 /* Old version is even not supported. */
158 if (!priv->config.hca_attr.qos.flow_meter_old)
159 return -rte_mtr_error_set(error, ENOTSUP,
160 RTE_MTR_ERROR_TYPE_METER_PROFILE,
161 NULL, "Metering is not supported.");
162 /* Old FW metering only supports srTCM. */
163 if (profile->alg != RTE_MTR_SRTCM_RFC2697) {
164 return -rte_mtr_error_set(error, ENOTSUP,
165 RTE_MTR_ERROR_TYPE_METER_PROFILE,
166 NULL, "Metering algorithm is not supported.");
167 } else if (profile->srtcm_rfc2697.ebs) {
168 /* EBS is not supported for old metering. */
169 return -rte_mtr_error_set(error, ENOTSUP,
170 RTE_MTR_ERROR_TYPE_METER_PROFILE,
171 NULL, "EBS is not supported.");
173 if (profile->packet_mode)
174 return -rte_mtr_error_set(error, ENOTSUP,
175 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
176 "Metering algorithm packet mode is not supported.");
178 ls_factor = profile->packet_mode ? MLX5_MTRS_PPS_MAP_BPS_SHIFT : 0;
179 switch (profile->alg) {
180 case RTE_MTR_SRTCM_RFC2697:
181 cir = profile->srtcm_rfc2697.cir << ls_factor;
182 cbs = profile->srtcm_rfc2697.cbs << ls_factor;
183 ebs = profile->srtcm_rfc2697.ebs << ls_factor;
184 /* EBS could be zero for old metering. */
185 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
186 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
187 ebs <= MLX5_SRTCM_XBS_MAX) {
190 ret = -rte_mtr_error_set(error, ENOTSUP,
191 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
192 "Profile values out of range.");
195 case RTE_MTR_TRTCM_RFC2698:
196 cir = profile->trtcm_rfc2698.cir << ls_factor;
197 cbs = profile->trtcm_rfc2698.cbs << ls_factor;
198 pir = profile->trtcm_rfc2698.pir << ls_factor;
199 pbs = profile->trtcm_rfc2698.pbs << ls_factor;
200 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
201 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
202 pir >= cir && pir <= (MLX5_SRTCM_XIR_MAX * 2) &&
203 pbs >= cbs && pbs <= (MLX5_SRTCM_XBS_MAX * 2)) {
206 ret = -rte_mtr_error_set(error, ENOTSUP,
207 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
208 "Profile values out of range.");
211 case RTE_MTR_TRTCM_RFC4115:
212 cir = profile->trtcm_rfc4115.cir << ls_factor;
213 cbs = profile->trtcm_rfc4115.cbs << ls_factor;
214 eir = profile->trtcm_rfc4115.eir << ls_factor;
215 ebs = profile->trtcm_rfc4115.ebs << ls_factor;
216 if (cir > 0 && cir <= MLX5_SRTCM_XIR_MAX &&
217 cbs > 0 && cbs <= MLX5_SRTCM_XBS_MAX &&
218 eir <= MLX5_SRTCM_XIR_MAX && ebs <= MLX5_SRTCM_XBS_MAX) {
221 ret = -rte_mtr_error_set(error, ENOTSUP,
222 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
223 "Profile values out of range.");
227 ret = -rte_mtr_error_set(error, ENOTSUP,
228 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
229 "Unknown metering algorithm.");
236 * Calculate mantissa and exponent for cir / eir.
239 * Value to be calculated.
241 * Pointer to the mantissa.
243 * Pointer to the exp.
246 mlx5_flow_meter_xir_man_exp_calc(int64_t xir, uint8_t *man, uint8_t *exp)
249 int64_t delta = INT64_MAX;
254 /* Special case xir == 0 ? both exp and mantissa are 0. */
260 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
261 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
262 _xir = (1000000000ULL * m) >> e;
263 if (llabs(xir - _xir) <= delta) {
264 delta = llabs(xir - _xir);
275 * Calculate mantissa and exponent for xbs.
278 * Value to be calculated.
280 * Pointer to the mantissa.
282 * Pointer to the exp.
285 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
290 /* Special case xbs == 0 ? both exp and mantissa are 0. */
296 /* xbs = xbs_mantissa * 2^xbs_exponent */
297 _man = frexp(xbs, &_exp);
298 if (_exp >= MLX5_MAN_WIDTH) {
299 _man = _man * pow(2, MLX5_MAN_WIDTH);
300 _exp = _exp - MLX5_MAN_WIDTH;
302 *man = (uint8_t)ceil(_man);
307 * Fill the prm meter parameter.
310 * Pointer to meter profile to be converted.
312 * Pointer to the error structure.
315 * 0 on success, a negative errno value otherwise and rte_errno is set.
318 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
319 struct rte_mtr_error *error)
321 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
323 uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
324 uint32_t eir_exp, eir_man, ebs_exp, ebs_man;
325 uint64_t cir, cbs, eir, ebs;
327 switch (fmp->profile.alg) {
328 case RTE_MTR_SRTCM_RFC2697:
329 cir = fmp->profile.srtcm_rfc2697.cir;
330 cbs = fmp->profile.srtcm_rfc2697.cbs;
332 ebs = fmp->profile.srtcm_rfc2697.ebs;
334 case RTE_MTR_TRTCM_RFC2698:
335 MLX5_ASSERT(fmp->profile.trtcm_rfc2698.pir >
336 fmp->profile.trtcm_rfc2698.cir &&
337 fmp->profile.trtcm_rfc2698.pbs >
338 fmp->profile.trtcm_rfc2698.cbs);
339 cir = fmp->profile.trtcm_rfc2698.cir;
340 cbs = fmp->profile.trtcm_rfc2698.cbs;
341 /* EIR / EBS are filled with PIR / PBS. */
342 eir = fmp->profile.trtcm_rfc2698.pir;
343 ebs = fmp->profile.trtcm_rfc2698.pbs;
345 case RTE_MTR_TRTCM_RFC4115:
346 cir = fmp->profile.trtcm_rfc4115.cir;
347 cbs = fmp->profile.trtcm_rfc4115.cbs;
348 eir = fmp->profile.trtcm_rfc4115.eir;
349 ebs = fmp->profile.trtcm_rfc4115.ebs;
352 return -rte_mtr_error_set(error, EINVAL,
353 RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL,
354 "Metering algorithm mode is invalid");
356 /* Adjust the values for PPS mode. */
357 if (fmp->profile.packet_mode) {
358 cir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
359 cbs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
360 eir <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
361 ebs <<= MLX5_MTRS_PPS_MAP_BPS_SHIFT;
363 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
364 mlx5_flow_meter_xir_man_exp_calc(cir, &man, &exp);
365 /* Check if cir mantissa is too large. */
366 if (exp > ASO_DSEG_XIR_EXP_MASK)
367 return -rte_mtr_error_set(error, ENOTSUP,
368 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
369 "meter profile parameter cir is not supported.");
372 /* cbs = cbs_mantissa * 2^cbs_exponent */
373 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
374 /* Check if cbs mantissa is too large. */
375 if (exp > ASO_DSEG_EXP_MASK)
376 return -rte_mtr_error_set(error, ENOTSUP,
377 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
378 "meter profile parameter cbs is not supported.");
381 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
382 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
383 cir_exp << ASO_DSEG_XIR_EXP_OFFSET |
385 mlx5_flow_meter_xir_man_exp_calc(eir, &man, &exp);
386 /* Check if eir mantissa is too large. */
387 if (exp > ASO_DSEG_XIR_EXP_MASK)
388 return -rte_mtr_error_set(error, ENOTSUP,
389 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
390 "meter profile parameter eir is not supported.");
393 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
394 /* Check if ebs mantissa is too large. */
395 if (exp > ASO_DSEG_EXP_MASK)
396 return -rte_mtr_error_set(error, ENOTSUP,
397 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
398 "meter profile parameter ebs is not supported.");
401 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
402 ebs_man << ASO_DSEG_EBS_MAN_OFFSET |
403 eir_exp << ASO_DSEG_XIR_EXP_OFFSET |
413 * Callback to get MTR capabilities.
416 * Pointer to Ethernet device.
418 * Pointer to save MTR capabilities.
420 * Pointer to the error structure.
423 * 0 on success, a negative errno value otherwise and rte_errno is set.
426 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
427 struct rte_mtr_capabilities *cap,
428 struct rte_mtr_error *error __rte_unused)
430 struct mlx5_priv *priv = dev->data->dev_private;
431 struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
434 return -rte_mtr_error_set(error, ENOTSUP,
435 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
436 "Meter is not supported");
437 memset(cap, 0, sizeof(*cap));
438 if (priv->sh->meter_aso_en) {
439 /* 2 meters per one ASO cache line. */
440 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
441 cap->srtcm_rfc2697_packet_mode_supported = 1;
442 cap->trtcm_rfc2698_packet_mode_supported = 1;
443 cap->trtcm_rfc4115_packet_mode_supported = 1;
445 cap->n_max = 1 << qattr->log_max_flow_meter;
447 cap->srtcm_rfc2697_byte_mode_supported = 1;
448 cap->trtcm_rfc2698_byte_mode_supported = 1;
449 cap->trtcm_rfc4115_byte_mode_supported = 1;
450 cap->n_shared_max = cap->n_max;
452 cap->shared_identical = 1;
453 cap->shared_n_flows_per_mtr_max = 4 << 20;
454 /* 2M flows can share the same meter. */
455 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
456 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
457 cap->meter_trtcm_rfc2698_n_max = qattr->flow_meter_old ? cap->n_max : 0;
458 cap->meter_trtcm_rfc4115_n_max = qattr->flow_meter_old ? cap->n_max : 0;
459 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
460 cap->meter_policy_n_max = cap->n_max;
461 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
462 RTE_MTR_STATS_N_PKTS_DROPPED;
467 * Callback to add MTR profile.
470 * Pointer to Ethernet device.
471 * @param[in] meter_profile_id
474 * Pointer to meter profile detail.
476 * Pointer to the error structure.
479 * 0 on success, a negative errno value otherwise and rte_errno is set.
482 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
483 uint32_t meter_profile_id,
484 struct rte_mtr_meter_profile *profile,
485 struct rte_mtr_error *error)
487 struct mlx5_priv *priv = dev->data->dev_private;
488 struct mlx5_flow_meter_profile *fmp;
489 union mlx5_l3t_data data;
493 return -rte_mtr_error_set(error, ENOTSUP,
494 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
495 "Meter is not supported");
496 /* Check input params. */
497 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
501 /* Meter profile memory allocation. */
502 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
503 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
505 return -rte_mtr_error_set(error, ENOMEM,
506 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
507 NULL, "Meter profile memory "
509 /* Fill profile info. */
510 fmp->id = meter_profile_id;
511 fmp->profile = *profile;
512 /* Fill the flow meter parameters for the PRM. */
513 ret = mlx5_flow_meter_param_fill(fmp, error);
517 ret = mlx5_l3t_set_entry(priv->mtr_profile_tbl,
518 meter_profile_id, &data);
520 return -rte_mtr_error_set(error, ENOTSUP,
521 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
522 NULL, "Meter profile insert fail.");
530 * Callback to delete MTR profile.
533 * Pointer to Ethernet device.
534 * @param[in] meter_profile_id
537 * Pointer to the error structure.
540 * 0 on success, a negative errno value otherwise and rte_errno is set.
543 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
544 uint32_t meter_profile_id,
545 struct rte_mtr_error *error)
547 struct mlx5_priv *priv = dev->data->dev_private;
548 struct mlx5_flow_meter_profile *fmp;
551 return -rte_mtr_error_set(error, ENOTSUP,
552 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
553 "Meter is not supported");
554 /* Meter profile must exist. */
555 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
557 return -rte_mtr_error_set(error, ENOENT,
558 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
560 "Meter profile id is invalid.");
561 /* Check profile is unused. */
563 return -rte_mtr_error_set(error, EBUSY,
564 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
565 NULL, "Meter profile is in use.");
566 if (mlx5_l3t_clear_entry(priv->mtr_profile_tbl, meter_profile_id))
567 return -rte_mtr_error_set(error, EBUSY,
568 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
569 NULL, "Meter profile remove fail.");
578 * Pointer to Ethernet device.
583 * Pointer to the policy found on success, NULL otherwise.
585 struct mlx5_flow_meter_policy *
586 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
588 uint32_t *policy_idx)
590 struct mlx5_priv *priv = dev->data->dev_private;
591 struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
592 union mlx5_l3t_data data;
594 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM || !priv->policy_idx_tbl)
596 if (mlx5_l3t_get_entry(priv->policy_idx_tbl, policy_id, &data) ||
600 *policy_idx = data.dword;
601 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
603 /* Remove reference taken by the mlx5_l3t_get_entry. */
604 mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id);
606 if (sub_policy->main_policy_id)
607 return sub_policy->main_policy;
612 * Get the last meter's policy from one meter's policy in hierarchy.
615 * Pointer to Ethernet device.
617 * Pointer to flow meter policy.
620 * Pointer to the final meter's policy, or NULL when fail.
622 struct mlx5_flow_meter_policy *
623 mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
624 struct mlx5_flow_meter_policy *policy)
626 struct mlx5_priv *priv = dev->data->dev_private;
627 struct mlx5_flow_meter_info *next_fm;
628 struct mlx5_flow_meter_policy *next_policy = policy;
630 while (next_policy->is_hierarchy) {
631 next_fm = mlx5_flow_meter_find(priv,
632 next_policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id, NULL);
633 if (!next_fm || next_fm->def_policy)
635 next_policy = mlx5_flow_meter_policy_find(dev,
636 next_fm->policy_id, NULL);
637 MLX5_ASSERT(next_policy);
643 * Callback to check MTR policy action validate
646 * Pointer to Ethernet device.
648 * Pointer to meter policy action detail.
650 * Pointer to the error structure.
653 * 0 on success, a negative errno value otherwise and rte_errno is set.
656 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
657 struct rte_mtr_meter_policy_params *policy,
658 struct rte_mtr_error *error)
660 struct mlx5_priv *priv = dev->data->dev_private;
661 struct rte_flow_attr attr = { .transfer =
662 priv->config.dv_esw_en ? 1 : 0};
665 uint8_t domain_bitmap;
668 if (!priv->mtr_en || !priv->sh->meter_aso_en)
669 return -rte_mtr_error_set(error, ENOTSUP,
670 RTE_MTR_ERROR_TYPE_METER_POLICY,
671 NULL, "meter policy unsupported.");
672 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
673 &is_rss, &domain_bitmap, &policy_mode, error);
680 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
682 struct mlx5_flow_meter_policy *mtr_policy,
683 struct rte_mtr_error *error,
686 struct mlx5_priv *priv = dev->data->dev_private;
687 struct mlx5_flow_meter_sub_policy *sub_policy;
689 uint16_t sub_policy_num;
691 rte_spinlock_lock(&mtr_policy->sl);
692 if (mtr_policy->ref_cnt) {
693 rte_spinlock_unlock(&mtr_policy->sl);
694 return -rte_mtr_error_set(error, EBUSY,
695 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
697 "Meter policy object is being used.");
699 mlx5_flow_destroy_policy_rules(dev, mtr_policy);
700 mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
701 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
702 sub_policy_num = (mtr_policy->sub_policy_num >>
703 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
704 MLX5_MTR_SUB_POLICY_NUM_MASK;
705 if (sub_policy_num) {
706 for (j = 0; j < sub_policy_num; j++) {
707 sub_policy = mtr_policy->sub_policys[i][j];
710 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
715 if (priv->policy_idx_tbl && clear_l3t) {
716 if (mlx5_l3t_clear_entry(priv->policy_idx_tbl, policy_id)) {
717 rte_spinlock_unlock(&mtr_policy->sl);
718 return -rte_mtr_error_set(error, ENOTSUP,
719 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
720 "Fail to delete policy in index table.");
723 rte_spinlock_unlock(&mtr_policy->sl);
728 * Callback to add MTR policy.
731 * Pointer to Ethernet device.
732 * @param[out] policy_id
733 * Pointer to policy id
735 * Pointer to meter policy action detail.
737 * Pointer to the error structure.
740 * 0 on success, a negative errno value otherwise and rte_errno is set.
743 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
745 struct rte_mtr_meter_policy_params *policy,
746 struct rte_mtr_error *error)
748 struct mlx5_priv *priv = dev->data->dev_private;
749 struct rte_flow_attr attr = { .transfer =
750 priv->config.dv_esw_en ? 1 : 0};
751 uint32_t sub_policy_idx = 0;
752 uint32_t policy_idx = 0;
753 struct mlx5_flow_meter_policy *mtr_policy = NULL;
754 struct mlx5_flow_meter_sub_policy *sub_policy;
759 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
760 uint16_t sub_policy_num;
761 uint8_t domain_bitmap = 0;
762 union mlx5_l3t_data data;
763 bool skip_rule = false;
766 return -rte_mtr_error_set(error, ENOTSUP,
767 RTE_MTR_ERROR_TYPE_METER_POLICY,
768 NULL, "meter policy unsupported. ");
769 if (policy_id == MLX5_INVALID_POLICY_ID)
770 return -rte_mtr_error_set(error, ENOTSUP,
771 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
772 NULL, "policy ID is invalid. ");
773 if (policy_id == priv->sh->mtrmng->def_policy_id)
774 return -rte_mtr_error_set(error, EEXIST,
775 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
776 NULL, "default policy ID exists. ");
777 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
779 return -rte_mtr_error_set(error, EEXIST,
780 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
781 NULL, "policy ID exists. ");
782 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
783 &is_rss, &domain_bitmap,
784 &policy_mode, error);
788 return -rte_mtr_error_set(error, ENOTSUP,
789 RTE_MTR_ERROR_TYPE_METER_POLICY,
790 NULL, "fail to find policy domain.");
791 if (policy_mode == MLX5_MTR_POLICY_MODE_DEF) {
792 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
793 return -rte_mtr_error_set(error, EEXIST,
794 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
795 NULL, "a policy with similar actions "
796 "is already configured");
797 if (mlx5_flow_create_def_policy(dev))
798 return -rte_mtr_error_set(error, ENOTSUP,
799 RTE_MTR_ERROR_TYPE_METER_POLICY,
801 "fail to create non-terminated policy.");
802 priv->sh->mtrmng->def_policy_id = policy_id;
805 if (!priv->sh->meter_aso_en)
806 return -rte_mtr_error_set(error, ENOTSUP,
807 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
808 "no ASO capability to support the policy ");
809 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
810 if (!(domain_bitmap & (1 << i)))
813 * If RSS is found, it means that only the ingress domain can
814 * be supported. It is invalid to support RSS for one color
815 * and egress / transfer domain actions for another. Drop and
816 * jump action should have no impact.
820 sizeof(struct mlx5_flow_meter_sub_policy *) *
821 MLX5_MTR_RSS_MAX_SUB_POLICY;
824 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
826 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
827 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
829 return -rte_mtr_error_set(error, ENOMEM,
830 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
831 "Memory alloc failed for meter policy.");
832 if (policy_mode == MLX5_MTR_POLICY_MODE_OG)
833 mtr_policy->skip_y = 1;
834 else if (policy_mode == MLX5_MTR_POLICY_MODE_OY)
835 mtr_policy->skip_g = 1;
836 policy_size = sizeof(struct mlx5_flow_meter_policy);
837 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
838 if (!(domain_bitmap & (1 << i)))
840 if (i == MLX5_MTR_DOMAIN_INGRESS)
841 mtr_policy->ingress = 1;
842 if (i == MLX5_MTR_DOMAIN_EGRESS)
843 mtr_policy->egress = 1;
844 if (i == MLX5_MTR_DOMAIN_TRANSFER)
845 mtr_policy->transfer = 1;
846 sub_policy = mlx5_ipool_zmalloc
847 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
850 sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
852 sub_policy->idx = sub_policy_idx;
853 sub_policy->main_policy = mtr_policy;
855 policy_idx = sub_policy_idx;
856 sub_policy->main_policy_id = 1;
858 mtr_policy->sub_policys[i] =
859 (struct mlx5_flow_meter_sub_policy **)
860 ((uint8_t *)mtr_policy + policy_size);
861 mtr_policy->sub_policys[i][0] = sub_policy;
862 sub_policy_num = (mtr_policy->sub_policy_num >>
863 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
864 MLX5_MTR_SUB_POLICY_NUM_MASK;
866 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
867 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
868 mtr_policy->sub_policy_num |=
869 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
870 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
872 * If RSS is found, it means that only the ingress domain can
873 * be supported. It is invalid to support RSS for one color
874 * and egress / transfer domain actions for another. Drop and
875 * jump action should have no impact.
878 mtr_policy->is_rss = 1;
881 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
883 rte_spinlock_init(&mtr_policy->sl);
884 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
885 policy->actions, error);
888 if (mtr_policy->is_hierarchy) {
889 struct mlx5_flow_meter_policy *final_policy;
892 mlx5_flow_meter_hierarchy_get_final_policy(dev, mtr_policy);
895 skip_rule = (final_policy->is_rss || final_policy->is_queue);
898 * If either Green or Yellow has queue / RSS action, all the policy
899 * rules will be created later in the flow splitting stage.
901 if (!is_rss && !mtr_policy->is_queue && !skip_rule) {
902 /* Create policy rules in HW. */
903 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
907 data.dword = policy_idx;
908 if (!priv->policy_idx_tbl) {
909 priv->policy_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
910 if (!priv->policy_idx_tbl)
913 if (mlx5_l3t_set_entry(priv->policy_idx_tbl, policy_id, &data))
918 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
919 mtr_policy, error, false);
920 mlx5_free(mtr_policy);
924 return -rte_mtr_error_set(error, ENOTSUP,
925 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
926 NULL, "Failed to create devx policy.");
930 * Callback to delete MTR policy.
933 * Pointer to Ethernet device.
934 * @param[in] policy_id
937 * Pointer to the error structure.
940 * 0 on success, a negative errno value otherwise and rte_errno is set.
943 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
945 struct rte_mtr_error *error)
947 struct mlx5_priv *priv = dev->data->dev_private;
948 struct mlx5_flow_meter_policy *mtr_policy;
952 if (policy_id == priv->sh->mtrmng->def_policy_id) {
953 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
954 return -rte_mtr_error_set(error, ENOTSUP,
955 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
956 "Meter policy object is being used.");
957 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
960 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
962 return -rte_mtr_error_set(error, ENOTSUP,
963 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
964 "Meter policy id is invalid. ");
965 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
969 mlx5_free(mtr_policy);
974 * Check meter validation.
977 * Pointer to mlx5 private data structure.
978 * @param[in] meter_id
981 * Pointer to rte meter parameters.
983 * Pointer to rte meter error structure.
986 * 0 on success, a negative errno value otherwise and rte_errno is set.
989 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
990 struct rte_mtr_params *params,
991 struct rte_mtr_error *error)
993 /* Meter must use global drop action. */
994 if (!priv->sh->dr_drop_action)
995 return -rte_mtr_error_set(error, ENOTSUP,
996 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
998 "No drop action ready for meter.");
999 /* Meter params must not be NULL. */
1001 return -rte_mtr_error_set(error, EINVAL,
1002 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1003 NULL, "Meter object params null.");
1004 /* Previous meter color is not supported. */
1005 if (params->use_prev_mtr_color)
1006 return -rte_mtr_error_set(error, ENOTSUP,
1007 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1009 "Previous meter color "
1011 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
1012 return -rte_mtr_error_set(error, ENOENT,
1013 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1014 NULL, "Meter policy id not valid.");
1015 /* Validate meter id. */
1016 if (mlx5_flow_meter_find(priv, meter_id, NULL))
1017 return -rte_mtr_error_set(error, EEXIST,
1018 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
1019 "Meter object already exists.");
1024 * Modify the flow meter action.
1027 * Pointer to mlx5 private data structure.
1029 * Pointer to flow meter to be modified.
1031 * Pointer to meter srtcm description parameter.
1032 * @param[in] modify_bits
1033 * The bit in srtcm to be updated.
1034 * @param[in] active_state
1035 * The state to be updated.
1037 * 0 on success, o negative value otherwise.
1040 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
1041 struct mlx5_flow_meter_info *fm,
1042 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
1043 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
1045 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1046 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
1048 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
1050 struct mlx5_aso_mtr *aso_mtr = NULL;
1051 uint32_t cbs_cir, ebs_eir, val;
1053 if (priv->sh->meter_aso_en) {
1054 fm->is_enable = !!is_enable;
1055 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1056 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1059 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
1063 /* Fill command parameters. */
1064 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
1065 mod_attr.flow_meter_parameter = in;
1066 mod_attr.flow_meter_parameter_sz =
1067 MLX5_ST_SZ_BYTES(flow_meter_parameters);
1068 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1069 mod_attr.active = !!active_state;
1071 mod_attr.active = 0;
1073 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
1074 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
1075 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
1076 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
1078 MLX5_SET(flow_meter_parameters, attr,
1080 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
1082 MLX5_SET(flow_meter_parameters, attr,
1085 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
1086 val = (cbs_cir >> ASO_DSEG_XIR_EXP_OFFSET) &
1088 MLX5_SET(flow_meter_parameters, attr,
1090 val = cbs_cir & ASO_DSEG_MAN_MASK;
1091 MLX5_SET(flow_meter_parameters, attr,
1094 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
1095 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
1097 MLX5_SET(flow_meter_parameters, attr,
1099 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
1101 MLX5_SET(flow_meter_parameters, attr,
1104 /* Apply modifications to meter only if it was created. */
1105 if (fm->meter_action) {
1106 ret = mlx5_glue->dv_modify_flow_action_meter
1107 (fm->meter_action, &mod_attr,
1108 rte_cpu_to_be_64(modify_bits));
1112 /* Update succeeded modify meter parameters. */
1113 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
1114 fm->active_state = !!active_state;
1129 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
1130 struct mlx5_flow_meter_info *fm,
1131 uint64_t stats_mask)
1134 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
1135 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
1136 if (fm->bytes_dropped || fm->pkts_dropped) {
1137 if (!fm->drop_cnt) {
1138 /* Alloc policer counters. */
1139 fm->drop_cnt = mlx5_counter_alloc(dev);
1145 mlx5_counter_free(dev, fm->drop_cnt);
1153 * Create meter rules.
1156 * Pointer to Ethernet device.
1157 * @param[in] meter_id
1160 * Pointer to rte meter parameters.
1162 * Meter shared with other flow or not.
1164 * Pointer to rte meter error structure.
1167 * 0 on success, a negative errno value otherwise and rte_errno is set.
1170 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1171 struct rte_mtr_params *params, int shared,
1172 struct rte_mtr_error *error)
1174 struct mlx5_priv *priv = dev->data->dev_private;
1175 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1176 struct mlx5_flow_meter_profile *fmp;
1177 struct mlx5_flow_meter_info *fm;
1178 /* GCC fails to infer legacy_fm is set when !priv->sh->meter_aso_en. */
1179 struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1180 struct mlx5_flow_meter_policy *mtr_policy = NULL;
1181 struct mlx5_indexed_pool_config flow_ipool_cfg = {
1185 .type = "mlx5_flow_mtr_flow_id_pool",
1187 struct mlx5_aso_mtr *aso_mtr;
1188 uint32_t mtr_idx, policy_idx;
1189 union mlx5_l3t_data data;
1191 uint8_t domain_bitmap;
1192 uint8_t mtr_id_bits;
1193 uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1194 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1197 return -rte_mtr_error_set(error, ENOTSUP,
1198 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1199 "Meter is not supported");
1200 /* Validate the parameters. */
1201 ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1204 /* Meter profile must exist. */
1205 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1207 return -rte_mtr_error_set(error, ENOENT,
1208 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1209 NULL, "Meter profile id not valid.");
1210 /* Meter policy must exist. */
1211 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1213 (&priv->sh->mtrmng->def_policy_ref_cnt,
1214 1, __ATOMIC_RELAXED);
1215 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1216 if (!priv->config.dv_esw_en)
1217 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1219 if (!priv->sh->meter_aso_en)
1220 return -rte_mtr_error_set(error, ENOTSUP,
1221 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1222 "Part of the policies cannot be "
1223 "supported without ASO ");
1224 mtr_policy = mlx5_flow_meter_policy_find(dev,
1225 params->meter_policy_id, &policy_idx);
1227 return -rte_mtr_error_set(error, ENOENT,
1228 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1229 NULL, "Meter policy id not valid.");
1230 domain_bitmap = (mtr_policy->ingress ?
1231 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1232 (mtr_policy->egress ?
1233 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1234 (mtr_policy->transfer ?
1235 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1236 if (fmp->g_support && mtr_policy->skip_g)
1237 return -rte_mtr_error_set(error, ENOTSUP,
1238 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1239 NULL, "Meter green policy is empty.");
1240 if (fmp->y_support && mtr_policy->skip_y)
1241 return -rte_mtr_error_set(error, ENOTSUP,
1242 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1243 NULL, "Meter yellow policy is empty.");
1245 /* Allocate the flow meter memory. */
1246 if (priv->sh->meter_aso_en) {
1247 mtr_idx = mlx5_flow_mtr_alloc(dev);
1249 return -rte_mtr_error_set(error, ENOMEM,
1250 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1251 "Memory alloc failed for meter.");
1252 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1256 return -rte_mtr_error_set(error, ENOMEM,
1257 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1258 "Unsupported profile with yellow.");
1259 legacy_fm = mlx5_ipool_zmalloc
1260 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1261 if (legacy_fm == NULL)
1262 return -rte_mtr_error_set(error, ENOMEM,
1263 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1264 "Memory alloc failed for meter.");
1265 legacy_fm->idx = mtr_idx;
1266 fm = &legacy_fm->fm;
1268 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1269 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1271 DRV_LOG(ERR, "Meter number exceeds max limit.");
1274 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1275 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1276 /* Fill the flow meter parameters. */
1277 fm->meter_id = meter_id;
1278 fm->policy_id = params->meter_policy_id;
1280 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1282 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1284 /* Add to the flow meter list. */
1285 if (!priv->sh->meter_aso_en) {
1286 MLX5_ASSERT(legacy_fm != NULL);
1287 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1289 /* Add to the flow meter list. */
1290 fm->active_state = 1; /* Config meter starts as active. */
1292 fm->shared = !!shared;
1293 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1294 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1296 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1297 if (!fm->flow_ipool)
1300 rte_spinlock_init(&fm->sl);
1301 /* If ASO meter supported, update ASO flow meter by wqe. */
1302 if (priv->sh->meter_aso_en) {
1303 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1304 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1307 if (!priv->mtr_idx_tbl) {
1309 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1310 if (!priv->mtr_idx_tbl)
1313 data.dword = mtr_idx;
1314 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1318 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1321 mlx5_flow_destroy_mtr_tbls(dev, fm);
1322 /* Free policer counters. */
1324 mlx5_counter_free(dev, fm->drop_cnt);
1325 if (priv->sh->meter_aso_en)
1326 mlx5_flow_mtr_free(dev, mtr_idx);
1328 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1329 return -rte_mtr_error_set(error, ENOTSUP,
1330 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1331 NULL, "Failed to create devx meter.");
1335 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1336 struct mlx5_flow_meter_info *fm,
1339 struct mlx5_priv *priv = dev->data->dev_private;
1340 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1341 struct mlx5_flow_meter_profile *fmp;
1342 struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1343 struct mlx5_flow_meter_policy *mtr_policy;
1345 /* Meter object must not have any owner. */
1346 MLX5_ASSERT(!fm->ref_cnt);
1347 /* Get meter profile. */
1351 /* Update dependencies. */
1352 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1354 /* Remove from list. */
1355 if (!priv->sh->meter_aso_en) {
1356 legacy_fm = container_of(fm,
1357 struct mlx5_legacy_flow_meter, fm);
1358 TAILQ_REMOVE(fms, legacy_fm, next);
1360 /* Free drop counters. */
1362 mlx5_counter_free(dev, fm->drop_cnt);
1363 /* Free meter flow table. */
1364 if (fm->flow_ipool) {
1365 mlx5_ipool_destroy(fm->flow_ipool);
1368 mlx5_flow_destroy_mtr_tbls(dev, fm);
1370 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1371 1, __ATOMIC_RELAXED);
1372 if (priv->sh->meter_aso_en) {
1373 if (!fm->def_policy) {
1374 mtr_policy = mlx5_flow_meter_policy_find(dev,
1375 fm->policy_id, NULL);
1377 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1378 1, __ATOMIC_RELAXED);
1382 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1384 mlx5_flow_mtr_free(dev, mtr_idx);
1386 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1393 * Destroy meter rules.
1396 * Pointer to Ethernet device.
1397 * @param[in] meter_id
1400 * Pointer to rte meter error structure.
1403 * 0 on success, a negative errno value otherwise and rte_errno is set.
1406 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1407 struct rte_mtr_error *error)
1409 struct mlx5_priv *priv = dev->data->dev_private;
1410 struct mlx5_flow_meter_info *fm;
1411 uint32_t mtr_idx = 0;
1414 return -rte_mtr_error_set(error, ENOTSUP,
1415 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1417 "Meter is not supported");
1418 /* Meter object must exist. */
1419 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1421 return -rte_mtr_error_set(error, ENOENT,
1422 RTE_MTR_ERROR_TYPE_MTR_ID,
1424 "Meter object id not valid.");
1425 /* Meter object must not have any owner. */
1426 if (fm->ref_cnt > 0)
1427 return -rte_mtr_error_set(error, EBUSY,
1428 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1430 "Meter object is being used.");
1431 /* Destroy the meter profile. */
1432 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1433 return -rte_mtr_error_set(error, EINVAL,
1434 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1436 "MTR object meter profile invalid.");
1441 * Modify meter state.
1444 * Pointer to mlx5 private data structure.
1446 * Pointer to flow meter.
1447 * @param[in] new_state
1448 * New state to update.
1450 * Pointer to rte meter error structure.
1453 * 0 on success, a negative errno value otherwise and rte_errno is set.
1456 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1457 struct mlx5_flow_meter_info *fm,
1459 struct rte_mtr_error *error)
1461 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1462 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1465 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1466 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1469 if (new_state == MLX5_FLOW_METER_DISABLE)
1470 ret = mlx5_flow_meter_action_modify(priv, fm,
1471 &srtcm, modify_bits, 0, 0);
1473 ret = mlx5_flow_meter_action_modify(priv, fm,
1474 &fm->profile->srtcm_prm,
1477 return -rte_mtr_error_set(error, -ret,
1478 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1481 "Failed to enable meter." :
1482 "Failed to disable meter.");
1487 * Callback to enable flow meter.
1490 * Pointer to Ethernet device.
1491 * @param[in] meter_id
1494 * Pointer to rte meter error structure.
1497 * 0 on success, a negative errno value otherwise and rte_errno is set.
1500 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1502 struct rte_mtr_error *error)
1504 struct mlx5_priv *priv = dev->data->dev_private;
1505 struct mlx5_flow_meter_info *fm;
1509 return -rte_mtr_error_set(error, ENOTSUP,
1510 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1511 "Meter is not supported");
1512 /* Meter object must exist. */
1513 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1515 return -rte_mtr_error_set(error, ENOENT,
1516 RTE_MTR_ERROR_TYPE_MTR_ID,
1517 NULL, "Meter not found.");
1518 if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1520 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1523 fm->active_state = MLX5_FLOW_METER_ENABLE;
1528 * Callback to disable flow meter.
1531 * Pointer to Ethernet device.
1532 * @param[in] meter_id
1535 * Pointer to rte meter error structure.
1538 * 0 on success, a negative errno value otherwise and rte_errno is set.
1541 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1543 struct rte_mtr_error *error)
1545 struct mlx5_priv *priv = dev->data->dev_private;
1546 struct mlx5_flow_meter_info *fm;
1550 return -rte_mtr_error_set(error, ENOTSUP,
1551 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1552 "Meter is not supported");
1553 /* Meter object must exist. */
1554 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1556 return -rte_mtr_error_set(error, ENOENT,
1557 RTE_MTR_ERROR_TYPE_MTR_ID,
1558 NULL, "Meter not found.");
1559 if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1561 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1564 fm->active_state = MLX5_FLOW_METER_DISABLE;
1569 * Callback to update meter profile.
1572 * Pointer to Ethernet device.
1573 * @param[in] meter_id
1575 * @param[in] meter_profile_id
1576 * To be updated meter profile id.
1578 * Pointer to rte meter error structure.
1581 * 0 on success, a negative errno value otherwise and rte_errno is set.
1584 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1586 uint32_t meter_profile_id,
1587 struct rte_mtr_error *error)
1589 struct mlx5_priv *priv = dev->data->dev_private;
1590 struct mlx5_flow_meter_profile *fmp;
1591 struct mlx5_flow_meter_profile *old_fmp;
1592 struct mlx5_flow_meter_info *fm;
1593 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1594 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1598 return -rte_mtr_error_set(error, ENOTSUP,
1599 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1600 "Meter is not supported");
1601 /* Meter profile must exist. */
1602 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1604 return -rte_mtr_error_set(error, ENOENT,
1605 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1606 NULL, "Meter profile not found.");
1607 /* Meter object must exist. */
1608 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1610 return -rte_mtr_error_set(error, ENOENT,
1611 RTE_MTR_ERROR_TYPE_MTR_ID,
1612 NULL, "Meter not found.");
1613 /* MTR object already set to meter profile id. */
1614 old_fmp = fm->profile;
1617 /* Update the profile. */
1619 /* Update meter params in HW (if not disabled). */
1620 if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1622 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1623 modify_bits, fm->active_state, 1);
1625 fm->profile = old_fmp;
1626 return -rte_mtr_error_set(error, -ret,
1627 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1628 NULL, "Failed to update meter"
1629 " parameters in hardware.");
1637 * Callback to update meter stats mask.
1640 * Pointer to Ethernet device.
1641 * @param[in] meter_id
1643 * @param[in] stats_mask
1644 * To be updated stats_mask.
1646 * Pointer to rte meter error structure.
1649 * 0 on success, a negative errno value otherwise and rte_errno is set.
1652 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1654 uint64_t stats_mask,
1655 struct rte_mtr_error *error)
1657 struct mlx5_priv *priv = dev->data->dev_private;
1658 struct mlx5_flow_meter_info *fm;
1661 return -rte_mtr_error_set(error, ENOTSUP,
1662 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1663 "Meter is not supported");
1664 /* Meter object must exist. */
1665 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1667 return -rte_mtr_error_set(error, ENOENT,
1668 RTE_MTR_ERROR_TYPE_MTR_ID,
1669 NULL, "Meter object id not valid.");
1670 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1671 return -rte_mtr_error_set(error, ENOENT,
1672 RTE_MTR_ERROR_TYPE_MTR_ID,
1673 NULL, "Fail to allocate "
1674 "counter for meter.");
1679 * Callback to read meter statistics.
1682 * Pointer to Ethernet device.
1683 * @param[in] meter_id
1686 * Pointer to store the statistics.
1687 * @param[out] stats_mask
1688 * Pointer to store the stats_mask.
1690 * Statistic to be cleared after read or not.
1692 * Pointer to rte meter error structure.
1695 * 0 on success, a negative errno value otherwise and rte_errno is set.
1698 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1700 struct rte_mtr_stats *stats,
1701 uint64_t *stats_mask,
1703 struct rte_mtr_error *error)
1705 struct mlx5_priv *priv = dev->data->dev_private;
1706 struct mlx5_flow_meter_info *fm;
1712 return -rte_mtr_error_set(error, ENOTSUP,
1713 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1714 "Meter is not supported");
1715 /* Meter object must exist. */
1716 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1718 return -rte_mtr_error_set(error, ENOENT,
1719 RTE_MTR_ERROR_TYPE_MTR_ID,
1720 NULL, "Meter object id not valid.");
1722 if (fm->bytes_dropped)
1723 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1724 if (fm->pkts_dropped)
1725 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1726 memset(stats, 0, sizeof(*stats));
1728 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1732 /* If need to read the packets, set it. */
1733 if (fm->pkts_dropped)
1734 stats->n_pkts_dropped = pkts;
1735 /* If need to read the bytes, set it. */
1736 if (fm->bytes_dropped)
1737 stats->n_bytes_dropped = bytes;
1741 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1742 "Failed to read meter drop counters.");
1745 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1746 .capabilities_get = mlx5_flow_mtr_cap_get,
1747 .meter_profile_add = mlx5_flow_meter_profile_add,
1748 .meter_profile_delete = mlx5_flow_meter_profile_delete,
1749 .meter_policy_validate = mlx5_flow_meter_policy_validate,
1750 .meter_policy_add = mlx5_flow_meter_policy_add,
1751 .meter_policy_delete = mlx5_flow_meter_policy_delete,
1752 .create = mlx5_flow_meter_create,
1753 .destroy = mlx5_flow_meter_destroy,
1754 .meter_enable = mlx5_flow_meter_enable,
1755 .meter_disable = mlx5_flow_meter_disable,
1756 .meter_profile_update = mlx5_flow_meter_profile_update,
1757 .meter_dscp_table_update = NULL,
1758 .stats_update = mlx5_flow_meter_stats_update,
1759 .stats_read = mlx5_flow_meter_stats_read,
1763 * Get meter operations.
1766 * Pointer to Ethernet device structure.
1768 * Pointer to set the mtr operations.
1774 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1776 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1784 * Pointer to mlx5_priv.
1788 * Pointer to Meter index.
1791 * Pointer to the meter info found on success, NULL otherwise.
1793 struct mlx5_flow_meter_info *
1794 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1797 struct mlx5_legacy_flow_meter *legacy_fm;
1798 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1799 struct mlx5_aso_mtr *aso_mtr;
1800 struct mlx5_aso_mtr_pools_mng *pools_mng =
1801 &priv->sh->mtrmng->pools_mng;
1802 union mlx5_l3t_data data;
1805 if (priv->sh->meter_aso_en) {
1806 rte_rwlock_read_lock(&pools_mng->resize_mtrwl);
1807 n_valid = pools_mng->n_valid;
1808 rte_rwlock_read_unlock(&pools_mng->resize_mtrwl);
1809 if (!n_valid || !priv->mtr_idx_tbl ||
1810 (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1814 *mtr_idx = data.dword;
1815 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1816 /* Remove reference taken by the mlx5_l3t_get_entry. */
1817 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1818 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1820 return &aso_mtr->fm;
1822 TAILQ_FOREACH(legacy_fm, fms, next)
1823 if (meter_id == legacy_fm->fm.meter_id) {
1825 *mtr_idx = legacy_fm->idx;
1826 return &legacy_fm->fm;
1832 * Find meter by index.
1835 * Pointer to mlx5_priv.
1840 * Pointer to the meter info found on success, NULL otherwise.
1842 struct mlx5_flow_meter_info *
1843 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1845 struct mlx5_aso_mtr *aso_mtr;
1847 if (priv->sh->meter_aso_en) {
1848 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1851 return &aso_mtr->fm;
1853 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1858 * Attach meter to flow.
1859 * Unidirectional Meter creation can only be done
1860 * when flow direction is known, i.e. when calling meter_attach.
1863 * Pointer to mlx5 private data.
1865 * Pointer to flow meter.
1867 * Pointer to flow attributes.
1868 * @param [out] error
1869 * Pointer to error structure.
1872 * 0 on success, a negative errno value otherwise and rte_errno is set.
1875 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1876 struct mlx5_flow_meter_info *fm,
1877 const struct rte_flow_attr *attr,
1878 struct rte_flow_error *error)
1882 if (priv->sh->meter_aso_en) {
1883 struct mlx5_aso_mtr *aso_mtr;
1885 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1886 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1887 return rte_flow_error_set(error, ENOENT,
1888 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1890 "Timeout in meter configuration");
1892 rte_spinlock_lock(&fm->sl);
1893 if (fm->shared || !fm->ref_cnt) {
1896 rte_flow_error_set(error, EINVAL,
1897 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1898 "Meter cannot be shared");
1901 rte_spinlock_unlock(&fm->sl);
1903 rte_spinlock_lock(&fm->sl);
1904 if (fm->meter_action) {
1906 attr->transfer == fm->transfer &&
1907 attr->ingress == fm->ingress &&
1908 attr->egress == fm->egress) {
1911 rte_flow_error_set(error, EINVAL,
1912 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1914 "Meter attr not match." :
1915 "Meter cannot be shared.");
1919 fm->ingress = attr->ingress;
1920 fm->egress = attr->egress;
1921 fm->transfer = attr->transfer;
1923 /* This also creates the meter object. */
1924 fm->meter_action = mlx5_flow_meter_action_create(priv,
1926 if (!fm->meter_action) {
1931 rte_flow_error_set(error, EINVAL,
1932 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1933 "Meter action create failed.");
1937 rte_spinlock_unlock(&fm->sl);
1939 return ret ? -rte_errno : 0;
1943 * Detach meter from flow.
1946 * Pointer to mlx5 private data.
1948 * Pointer to flow meter.
1951 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1952 struct mlx5_flow_meter_info *fm)
1954 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1955 rte_spinlock_lock(&fm->sl);
1956 MLX5_ASSERT(fm->ref_cnt);
1957 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1958 mlx5_glue->destroy_flow_action(fm->meter_action);
1959 fm->meter_action = NULL;
1964 rte_spinlock_unlock(&fm->sl);
1972 * Flush meter with Rx queue configuration.
1975 * Pointer to Ethernet device.
1978 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1980 struct mlx5_priv *priv = dev->data->dev_private;
1981 struct mlx5_flow_meter_sub_policy *sub_policy;
1982 struct mlx5_flow_meter_policy *mtr_policy;
1984 uint32_t i, policy_idx;
1988 if (priv->policy_idx_tbl) {
1989 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
1990 policy_idx = *(uint32_t *)entry;
1991 sub_policy = mlx5_ipool_get
1992 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1994 if (!sub_policy || !sub_policy->main_policy)
1996 mtr_policy = sub_policy->main_policy;
1997 if (mtr_policy->is_queue || mtr_policy->is_rss)
1998 mlx5_flow_destroy_sub_policy_with_rxq(dev,
2005 * Iterate a meter hierarchy and flush all meters and policies if possible.
2008 * Pointer to Ethernet device.
2010 * Pointer to flow meter.
2011 * @param[in] mtr_idx
2014 * Pointer to rte meter error structure.
2017 * 0 on success, a negative errno value otherwise and rte_errno is set.
2020 mlx5_flow_meter_flush_hierarchy(struct rte_eth_dev *dev,
2021 struct mlx5_flow_meter_info *fm,
2023 struct rte_mtr_error *error)
2025 struct mlx5_priv *priv = dev->data->dev_private;
2026 struct mlx5_flow_meter_policy *policy;
2028 struct mlx5_flow_meter_info *next_fm;
2029 uint32_t next_mtr_idx;
2030 struct mlx5_flow_meter_policy *next_policy = NULL;
2032 policy = mlx5_flow_meter_policy_find(dev, fm->policy_id, NULL);
2033 MLX5_ASSERT(policy);
2034 while (!fm->ref_cnt && policy->is_hierarchy) {
2035 policy_id = fm->policy_id;
2036 next_fm = mlx5_flow_meter_find(priv,
2037 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2040 next_policy = mlx5_flow_meter_policy_find(dev,
2043 MLX5_ASSERT(next_policy);
2045 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
2046 return -rte_mtr_error_set(error, ENOTSUP,
2047 RTE_MTR_ERROR_TYPE_MTR_ID,
2049 "Failed to flush meter.");
2050 if (policy->ref_cnt)
2052 if (__mlx5_flow_meter_policy_delete(dev, policy_id,
2053 policy, error, true))
2056 if (!next_fm || !next_policy)
2059 mtr_idx = next_mtr_idx;
2060 policy = next_policy;
2066 * Flush all the hierarchy meters and their policies.
2069 * Pointer to Ethernet device.
2071 * Pointer to rte meter error structure.
2074 * 0 on success, a negative errno value otherwise and rte_errno is set.
2077 mlx5_flow_meter_flush_all_hierarchies(struct rte_eth_dev *dev,
2078 struct rte_mtr_error *error)
2080 struct mlx5_priv *priv = dev->data->dev_private;
2081 struct mlx5_flow_meter_info *fm;
2082 struct mlx5_flow_meter_policy *policy;
2083 struct mlx5_flow_meter_sub_policy *sub_policy;
2084 struct mlx5_flow_meter_info *next_fm;
2085 struct mlx5_aso_mtr *aso_mtr;
2086 uint32_t mtr_idx = 0;
2087 uint32_t i, policy_idx;
2090 if (!priv->mtr_idx_tbl || !priv->policy_idx_tbl)
2092 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2093 mtr_idx = *(uint32_t *)entry;
2096 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
2098 if (fm->ref_cnt || fm->def_policy)
2100 if (mlx5_flow_meter_flush_hierarchy(dev, fm, mtr_idx, error))
2103 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2104 policy_idx = *(uint32_t *)entry;
2105 sub_policy = mlx5_ipool_get
2106 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2109 return -rte_mtr_error_set(error,
2111 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2112 NULL, "Meter policy invalid.");
2113 policy = sub_policy->main_policy;
2114 if (!policy || !policy->is_hierarchy || policy->ref_cnt)
2116 next_fm = mlx5_flow_meter_find(priv,
2117 policy->act_cnt[RTE_COLOR_GREEN].next_mtr_id,
2119 if (__mlx5_flow_meter_policy_delete(dev, i, policy,
2121 return -rte_mtr_error_set(error,
2123 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2124 NULL, "Meter policy invalid.");
2126 if (!next_fm || next_fm->ref_cnt || next_fm->def_policy)
2128 if (mlx5_flow_meter_flush_hierarchy(dev, next_fm,
2135 * Flush meter configuration.
2138 * Pointer to Ethernet device.
2140 * Pointer to rte meter error structure.
2143 * 0 on success, a negative errno value otherwise and rte_errno is set.
2146 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
2148 struct mlx5_priv *priv = dev->data->dev_private;
2149 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
2150 struct mlx5_flow_meter_profile *fmp;
2151 struct mlx5_legacy_flow_meter *legacy_fm;
2152 struct mlx5_flow_meter_info *fm;
2153 struct mlx5_flow_meter_sub_policy *sub_policy;
2155 uint32_t i, mtr_idx, policy_idx;
2157 struct mlx5_aso_mtr *aso_mtr;
2161 if (priv->sh->meter_aso_en) {
2162 if (mlx5_flow_meter_flush_all_hierarchies(dev, error))
2164 if (priv->mtr_idx_tbl) {
2165 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
2166 mtr_idx = *(uint32_t *)entry;
2169 mlx5_aso_meter_by_idx(priv, mtr_idx);
2171 (void)mlx5_flow_meter_params_flush(dev,
2175 mlx5_l3t_destroy(priv->mtr_idx_tbl);
2176 priv->mtr_idx_tbl = NULL;
2179 RTE_TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
2180 fm = &legacy_fm->fm;
2181 if (mlx5_flow_meter_params_flush(dev, fm, 0))
2182 return -rte_mtr_error_set(error, EINVAL,
2183 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
2184 NULL, "MTR object meter profile invalid.");
2187 if (priv->policy_idx_tbl) {
2188 MLX5_L3T_FOREACH(priv->policy_idx_tbl, i, entry) {
2189 policy_idx = *(uint32_t *)entry;
2190 sub_policy = mlx5_ipool_get
2191 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
2194 return -rte_mtr_error_set(error,
2196 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2198 "meter policy invalid.");
2199 if (__mlx5_flow_meter_policy_delete(dev, i,
2200 sub_policy->main_policy,
2202 return -rte_mtr_error_set(error,
2204 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2206 "meter policy invalid.");
2207 mlx5_free(sub_policy->main_policy);
2209 mlx5_l3t_destroy(priv->policy_idx_tbl);
2210 priv->policy_idx_tbl = NULL;
2212 if (priv->mtr_profile_tbl) {
2213 MLX5_L3T_FOREACH(priv->mtr_profile_tbl, i, entry) {
2215 if (mlx5_flow_meter_profile_delete(dev, fmp->id,
2217 return -rte_mtr_error_set(error, EINVAL,
2218 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
2219 NULL, "Fail to destroy "
2222 mlx5_l3t_destroy(priv->mtr_profile_tbl);
2223 priv->mtr_profile_tbl = NULL;
2225 /* Delete default policy table. */
2226 mlx5_flow_destroy_def_policy(dev);
2227 if (priv->sh->refcnt == 1)
2228 mlx5_flow_destroy_mtr_drop_tbls(dev);