1 // SPDX-License-Identifier: BSD-3-Clause
3 * Copyright 2018 Mellanox Technologies, Ltd
8 #include <rte_malloc.h>
10 #include <rte_mtr_driver.h>
12 #include <mlx5_devx_cmds.h>
13 #include <mlx5_malloc.h>
16 #include "mlx5_flow.h"
19 * Create the meter action.
22 * Pointer to mlx5_priv.
24 * Pointer to flow meter to be converted.
27 * Pointer to the meter action on success, NULL otherwise.
30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
31 struct mlx5_flow_meter_info *fm)
33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
34 struct mlx5dv_dr_flow_meter_attr mtr_init;
35 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
36 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
37 &fm->profile->srtcm_prm;
38 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
39 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
41 enum mlx5_meter_domain domain =
42 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
43 fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
44 MLX5_MTR_DOMAIN_INGRESS;
45 struct mlx5_flow_meter_def_policy *def_policy =
46 priv->sh->mtrmng->def_policy[domain];
48 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
49 MLX5_SET(flow_meter_parameters, fmp, valid, 1);
50 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
51 MLX5_SET(flow_meter_parameters, fmp,
52 start_color, MLX5_FLOW_COLOR_GREEN);
53 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
54 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
55 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
56 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
57 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
58 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
59 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
60 val = (cbs_cir & ASO_DSEG_MAN_MASK);
61 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
62 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
63 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
64 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
65 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
66 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
67 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
68 mtr_init.flow_meter_parameter = fmp;
69 mtr_init.flow_meter_parameter_sz =
70 MLX5_ST_SZ_BYTES(flow_meter_parameters);
71 mtr_init.active = fm->active_state;
72 return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
81 * Find meter profile by id.
84 * Pointer to mlx5_priv.
85 * @param meter_profile_id
89 * Pointer to the profile found on success, NULL otherwise.
91 static struct mlx5_flow_meter_profile *
92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
94 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
95 struct mlx5_flow_meter_profile *fmp;
97 TAILQ_FOREACH(fmp, fmps, next)
98 if (meter_profile_id == fmp->id)
104 * Validate the MTR profile.
107 * Pointer to Ethernet device.
108 * @param[in] meter_profile_id
111 * Pointer to meter profile detail.
113 * Pointer to the error structure.
116 * 0 on success, a negative errno value otherwise and rte_errno is set.
119 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
120 uint32_t meter_profile_id,
121 struct rte_mtr_meter_profile *profile,
122 struct rte_mtr_error *error)
124 struct mlx5_priv *priv = dev->data->dev_private;
125 struct mlx5_flow_meter_profile *fmp;
127 /* Profile must not be NULL. */
129 return -rte_mtr_error_set(error, EINVAL,
130 RTE_MTR_ERROR_TYPE_METER_PROFILE,
131 NULL, "Meter profile is null.");
132 /* Meter profile ID must be valid. */
133 if (meter_profile_id == UINT32_MAX)
134 return -rte_mtr_error_set(error, EINVAL,
135 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
136 NULL, "Meter profile id not valid.");
137 /* Meter profile must not exist. */
138 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
140 return -rte_mtr_error_set(error, EEXIST,
141 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
143 "Meter profile already exists.");
144 if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
145 if (priv->config.hca_attr.qos.flow_meter_old) {
146 /* Verify support for flow meter parameters. */
147 if (priv->sh->meter_aso_en && profile->packet_mode) {
148 if (profile->srtcm_rfc2697.cir > 0 &&
149 (profile->srtcm_rfc2697.cir <<
150 MLX5_MTRS_PPS_MAP_BPS_SHIFT)
151 <= MLX5_SRTCM_CIR_MAX &&
152 profile->srtcm_rfc2697.cbs > 0 &&
153 (profile->srtcm_rfc2697.cbs <<
154 MLX5_MTRS_PPS_MAP_BPS_SHIFT)
155 <= MLX5_SRTCM_CBS_MAX &&
156 (profile->srtcm_rfc2697.ebs <<
157 MLX5_MTRS_PPS_MAP_BPS_SHIFT)
158 <= MLX5_SRTCM_EBS_MAX)
160 return -rte_mtr_error_set
162 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
164 profile->srtcm_rfc2697.ebs ?
165 "Metering value ebs must be 0." :
166 "Invalid metering parameters.");
168 if (profile->srtcm_rfc2697.cir > 0 &&
169 profile->srtcm_rfc2697.cir <=
170 MLX5_SRTCM_CIR_MAX &&
171 profile->srtcm_rfc2697.cbs > 0 &&
172 profile->srtcm_rfc2697.cbs <=
173 MLX5_SRTCM_CBS_MAX &&
174 profile->srtcm_rfc2697.ebs <=
177 return -rte_mtr_error_set(error, ENOTSUP,
178 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
180 profile->srtcm_rfc2697.ebs ?
181 "Metering value ebs must be 0." :
182 "Invalid metering parameters.");
185 return -rte_mtr_error_set(error, ENOTSUP,
186 RTE_MTR_ERROR_TYPE_METER_PROFILE,
187 NULL, "Metering algorithm not supported.");
191 * Calculate mantissa and exponent for cir.
194 * Value to be calculated.
196 * Pointer to the mantissa.
198 * Pointer to the exp.
201 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
204 int64_t delta = INT64_MAX;
209 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
210 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
211 _cir = (1000000000ULL * m) >> e;
212 if (llabs(cir - _cir) <= delta) {
213 delta = llabs(cir - _cir);
224 * Calculate mantissa and exponent for xbs.
227 * Value to be calculated.
229 * Pointer to the mantissa.
231 * Pointer to the exp.
234 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
239 /* Special case xbs == 0 ? both exp and matissa are 0. */
245 /* xbs = xbs_mantissa * 2^xbs_exponent */
246 _man = frexp(xbs, &_exp);
247 _man = _man * pow(2, MLX5_MAN_WIDTH);
248 _exp = _exp - MLX5_MAN_WIDTH;
249 *man = (uint8_t)ceil(_man);
254 * Fill the prm meter parameter.
257 * Pointer to meter profie to be converted.
259 * Pointer to the error structure.
262 * 0 on success, a negative errno value otherwise and rte_errno is set.
265 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
266 struct mlx5_priv *priv, struct rte_mtr_error *error)
268 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
270 uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
271 uint32_t ebs_exp, ebs_man;
272 uint64_t cir, cbs, ebs;
274 if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
275 return -rte_mtr_error_set(error, ENOTSUP,
276 RTE_MTR_ERROR_TYPE_METER_PROFILE,
277 NULL, "Metering algorithm not supported.");
278 if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
279 return -rte_mtr_error_set(error, ENOTSUP,
280 RTE_MTR_ERROR_TYPE_METER_PROFILE,
281 NULL, "Metering algorithm packet mode not supported.");
282 if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
283 cir = fmp->profile.srtcm_rfc2697.cir <<
284 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
285 cbs = fmp->profile.srtcm_rfc2697.cbs <<
286 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
287 ebs = fmp->profile.srtcm_rfc2697.ebs <<
288 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
290 cir = fmp->profile.srtcm_rfc2697.cir;
291 cbs = fmp->profile.srtcm_rfc2697.cbs;
292 ebs = fmp->profile.srtcm_rfc2697.ebs;
294 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
295 mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
296 /* Check if cir mantissa is too large. */
297 if (exp > ASO_DSEG_CIR_EXP_MASK)
298 return -rte_mtr_error_set(error, ENOTSUP,
299 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
300 "meter profile parameter cir is"
304 /* cbs = cbs_mantissa * 2^cbs_exponent */
305 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
306 /* Check if cbs mantissa is too large. */
307 if (exp > ASO_DSEG_EXP_MASK)
308 return -rte_mtr_error_set(error, ENOTSUP,
309 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
310 "meter profile parameter cbs is"
314 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
315 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
316 cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
318 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
319 /* Check if ebs mantissa is too large. */
320 if (exp > ASO_DSEG_EXP_MASK)
321 return -rte_mtr_error_set(error, ENOTSUP,
322 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
323 "meter profile parameter ebs is"
327 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
328 ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
333 * Callback to get MTR capabilities.
336 * Pointer to Ethernet device.
338 * Pointer to save MTR capabilities.
340 * Pointer to the error structure.
343 * 0 on success, a negative errno value otherwise and rte_errno is set.
346 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
347 struct rte_mtr_capabilities *cap,
348 struct rte_mtr_error *error __rte_unused)
350 struct mlx5_priv *priv = dev->data->dev_private;
351 struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
354 return -rte_mtr_error_set(error, ENOTSUP,
355 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
356 "Meter is not supported");
357 memset(cap, 0, sizeof(*cap));
358 if (priv->sh->meter_aso_en) {
359 /* 2 meters per one ASO cache line. */
360 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
361 cap->srtcm_rfc2697_packet_mode_supported = 1;
363 cap->n_max = 1 << qattr->log_max_flow_meter;
364 cap->srtcm_rfc2697_packet_mode_supported = 0;
366 cap->srtcm_rfc2697_byte_mode_supported = 1;
367 cap->n_shared_max = cap->n_max;
369 cap->shared_identical = 1;
370 cap->shared_n_flows_per_mtr_max = 4 << 20;
371 /* 2M flows can share the same meter. */
372 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
373 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
374 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
375 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
376 RTE_MTR_STATS_N_PKTS_DROPPED;
381 * Callback to add MTR profile.
384 * Pointer to Ethernet device.
385 * @param[in] meter_profile_id
388 * Pointer to meter profile detail.
390 * Pointer to the error structure.
393 * 0 on success, a negative errno value otherwise and rte_errno is set.
396 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
397 uint32_t meter_profile_id,
398 struct rte_mtr_meter_profile *profile,
399 struct rte_mtr_error *error)
401 struct mlx5_priv *priv = dev->data->dev_private;
402 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
403 struct mlx5_flow_meter_profile *fmp;
407 return -rte_mtr_error_set(error, ENOTSUP,
408 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
409 "Meter is not supported");
410 /* Check input params. */
411 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
415 /* Meter profile memory allocation. */
416 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
417 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
419 return -rte_mtr_error_set(error, ENOMEM,
420 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
421 NULL, "Meter profile memory "
423 /* Fill profile info. */
424 fmp->id = meter_profile_id;
425 fmp->profile = *profile;
426 /* Fill the flow meter parameters for the PRM. */
427 ret = mlx5_flow_meter_param_fill(fmp, priv, error);
431 TAILQ_INSERT_TAIL(fmps, fmp, next);
439 * Callback to delete MTR profile.
442 * Pointer to Ethernet device.
443 * @param[in] meter_profile_id
446 * Pointer to the error structure.
449 * 0 on success, a negative errno value otherwise and rte_errno is set.
452 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
453 uint32_t meter_profile_id,
454 struct rte_mtr_error *error)
456 struct mlx5_priv *priv = dev->data->dev_private;
457 struct mlx5_flow_meter_profile *fmp;
460 return -rte_mtr_error_set(error, ENOTSUP,
461 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
462 "Meter is not supported");
463 /* Meter profile must exist. */
464 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
466 return -rte_mtr_error_set(error, ENOENT,
467 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
469 "Meter profile id is invalid.");
470 /* Check profile is unused. */
472 return -rte_mtr_error_set(error, EBUSY,
473 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
474 NULL, "Meter profile is in use.");
475 /* Remove from list. */
476 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
485 * Pointer to Ethernet device.
490 * Pointer to the policy found on success, NULL otherwise.
492 struct mlx5_flow_meter_policy *
493 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
495 uint32_t *policy_idx)
497 struct mlx5_priv *priv = dev->data->dev_private;
498 struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
499 union mlx5_l3t_data data;
501 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM ||
502 !priv->sh->mtrmng->policy_idx_tbl)
504 if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl,
509 *policy_idx = data.dword;
510 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
512 /* Remove reference taken by the mlx5_l3t_get_entry. */
513 mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
516 if (sub_policy->main_policy_id)
517 return sub_policy->main_policy;
522 * Callback to check MTR policy action validate
525 * Pointer to Ethernet device.
527 * Pointer to meter policy action detail.
529 * Pointer to the error structure.
532 * 0 on success, a negative errno value otherwise and rte_errno is set.
535 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
536 struct rte_mtr_meter_policy_params *policy,
537 struct rte_mtr_error *error)
539 struct mlx5_priv *priv = dev->data->dev_private;
540 struct rte_flow_attr attr = { .transfer =
541 priv->config.dv_esw_en ? 1 : 0};
543 bool is_def_policy = false;
544 uint8_t domain_bitmap;
547 if (!priv->mtr_en || !priv->sh->meter_aso_en)
548 return -rte_mtr_error_set(error, ENOTSUP,
549 RTE_MTR_ERROR_TYPE_METER_POLICY,
550 NULL, "meter policy unsupported.");
551 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
552 &is_rss, &domain_bitmap, &is_def_policy, error);
559 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
561 struct mlx5_flow_meter_policy *mtr_policy,
562 struct rte_mtr_error *error)
564 struct mlx5_priv *priv = dev->data->dev_private;
565 struct mlx5_flow_meter_sub_policy *sub_policy;
567 uint16_t sub_policy_num;
569 rte_spinlock_lock(&mtr_policy->sl);
570 if (mtr_policy->ref_cnt) {
571 rte_spinlock_unlock(&mtr_policy->sl);
572 return -rte_mtr_error_set(error, EBUSY,
573 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
575 "Meter policy object is being used.");
577 mlx5_flow_destroy_policy_rules(dev, mtr_policy);
578 mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
579 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
580 sub_policy_num = (mtr_policy->sub_policy_num >>
581 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
582 MLX5_MTR_SUB_POLICY_NUM_MASK;
583 if (sub_policy_num) {
584 for (j = 0; j < sub_policy_num; j++) {
585 sub_policy = mtr_policy->sub_policys[i][j];
588 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
593 if (priv->sh->mtrmng->policy_idx_tbl) {
594 if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
596 rte_spinlock_unlock(&mtr_policy->sl);
597 return -rte_mtr_error_set(error, ENOTSUP,
598 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
599 "Fail to delete policy in index table.");
602 rte_spinlock_unlock(&mtr_policy->sl);
607 * Callback to add MTR policy.
610 * Pointer to Ethernet device.
611 * @param[out] policy_id
612 * Pointer to policy id
614 * Pointer to meter policy action detail.
616 * Pointer to the error structure.
619 * 0 on success, a negative errno value otherwise and rte_errno is set.
622 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
624 struct rte_mtr_meter_policy_params *policy,
625 struct rte_mtr_error *error)
627 struct mlx5_priv *priv = dev->data->dev_private;
628 struct rte_flow_attr attr = { .transfer =
629 priv->config.dv_esw_en ? 1 : 0};
630 uint32_t sub_policy_idx = 0;
631 uint32_t policy_idx = 0;
632 struct mlx5_flow_meter_policy *mtr_policy = NULL;
633 struct mlx5_flow_meter_sub_policy *sub_policy;
635 bool is_def_policy = false;
638 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
639 uint16_t sub_policy_num;
640 uint8_t domain_bitmap = 0;
641 union mlx5_l3t_data data;
644 return -rte_mtr_error_set(error, ENOTSUP,
645 RTE_MTR_ERROR_TYPE_METER_POLICY,
646 NULL, "meter policy unsupported.");
647 if (policy_id == MLX5_INVALID_POLICY_ID)
648 return -rte_mtr_error_set(error, ENOTSUP,
649 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
650 "policy ID is invalid. ");
651 if (policy_id == priv->sh->mtrmng->def_policy_id)
652 return -rte_mtr_error_set(error, EEXIST,
653 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
654 "policy ID exists. ");
655 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id,
658 return -rte_mtr_error_set(error, EEXIST,
659 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
660 "policy ID exists. ");
661 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
662 &is_rss, &domain_bitmap, &is_def_policy, error);
666 return -rte_mtr_error_set(error, ENOTSUP,
667 RTE_MTR_ERROR_TYPE_METER_POLICY,
668 NULL, "fail to find policy domain.");
670 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
671 return -rte_mtr_error_set(error, EEXIST,
672 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
673 NULL, "a policy with similar actions "
674 "is already configured");
675 if (mlx5_flow_create_def_policy(dev))
676 return -rte_mtr_error_set(error, ENOTSUP,
677 RTE_MTR_ERROR_TYPE_METER_POLICY,
679 "fail to create non-terminated policy.");
680 priv->sh->mtrmng->def_policy_id = policy_id;
683 if (!priv->sh->meter_aso_en)
684 return -rte_mtr_error_set(error, ENOTSUP,
685 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
686 "no ASO capability to support the policy ");
687 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
688 if (!(domain_bitmap & (1 << i)))
692 sizeof(struct mlx5_flow_meter_sub_policy *) *
693 MLX5_MTR_RSS_MAX_SUB_POLICY;
696 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
698 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
699 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
701 return -rte_mtr_error_set(error, ENOMEM,
702 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
703 "Memory alloc failed for meter policy.");
704 policy_size = sizeof(struct mlx5_flow_meter_policy);
705 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
706 if (!(domain_bitmap & (1 << i)))
708 if (i == MLX5_MTR_DOMAIN_INGRESS)
709 mtr_policy->ingress = 1;
710 if (i == MLX5_MTR_DOMAIN_EGRESS)
711 mtr_policy->egress = 1;
712 if (i == MLX5_MTR_DOMAIN_TRANSFER)
713 mtr_policy->transfer = 1;
714 sub_policy = mlx5_ipool_zmalloc
715 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
719 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
721 sub_policy->idx = sub_policy_idx;
722 sub_policy->main_policy = mtr_policy;
724 policy_idx = sub_policy_idx;
725 sub_policy->main_policy_id = 1;
727 mtr_policy->sub_policys[i] =
728 (struct mlx5_flow_meter_sub_policy **)
729 ((uint8_t *)mtr_policy + policy_size);
730 mtr_policy->sub_policys[i][0] = sub_policy;
731 sub_policy_num = (mtr_policy->sub_policy_num >>
732 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
733 MLX5_MTR_SUB_POLICY_NUM_MASK;
735 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
736 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
737 mtr_policy->sub_policy_num |=
738 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
739 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
741 mtr_policy->is_rss = 1;
744 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
746 rte_spinlock_init(&mtr_policy->sl);
747 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
748 policy->actions, error);
751 if (!is_rss && !mtr_policy->is_queue) {
752 /* Create policy rules in HW. */
753 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
757 data.dword = policy_idx;
758 if (!priv->sh->mtrmng->policy_idx_tbl) {
759 priv->sh->mtrmng->policy_idx_tbl =
760 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
761 if (!priv->sh->mtrmng->policy_idx_tbl)
764 if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl,
770 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
772 mlx5_free(mtr_policy);
776 return -rte_mtr_error_set(error, ENOTSUP,
777 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
778 NULL, "Failed to create devx policy.");
782 * Callback to delete MTR policy.
785 * Pointer to Ethernet device.
786 * @param[in] policy_id
789 * Pointer to the error structure.
792 * 0 on success, a negative errno value otherwise and rte_errno is set.
795 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
797 struct rte_mtr_error *error)
799 struct mlx5_priv *priv = dev->data->dev_private;
800 struct mlx5_flow_meter_policy *mtr_policy;
804 if (policy_id == priv->sh->mtrmng->def_policy_id) {
805 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
806 return -rte_mtr_error_set(error, ENOTSUP,
807 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
808 "Meter policy object is being used.");
809 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
812 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
814 return -rte_mtr_error_set(error, ENOTSUP,
815 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
816 "Meter policy id is invalid. ");
817 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
821 mlx5_free(mtr_policy);
826 * Check meter validation.
829 * Pointer to mlx5 private data structure.
830 * @param[in] meter_id
833 * Pointer to rte meter parameters.
835 * Pointer to rte meter error structure.
838 * 0 on success, a negative errno value otherwise and rte_errno is set.
841 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
842 struct rte_mtr_params *params,
843 struct rte_mtr_error *error)
845 /* Meter must use global drop action. */
846 if (!priv->sh->dr_drop_action)
847 return -rte_mtr_error_set(error, ENOTSUP,
848 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
850 "No drop action ready for meter.");
851 /* Meter params must not be NULL. */
853 return -rte_mtr_error_set(error, EINVAL,
854 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
855 NULL, "Meter object params null.");
856 /* Previous meter color is not supported. */
857 if (params->use_prev_mtr_color)
858 return -rte_mtr_error_set(error, ENOTSUP,
859 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
861 "Previous meter color "
863 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
864 return -rte_mtr_error_set(error, ENOENT,
865 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
866 NULL, "Meter policy id not valid.");
867 /* Validate meter id. */
868 if (mlx5_flow_meter_find(priv, meter_id, NULL))
869 return -rte_mtr_error_set(error, EEXIST,
870 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
871 "Meter object already exists.");
876 * Modify the flow meter action.
879 * Pointer to mlx5 private data structure.
881 * Pointer to flow meter to be modified.
883 * Pointer to meter srtcm description parameter.
884 * @param[in] modify_bits
885 * The bit in srtcm to be updated.
886 * @param[in] active_state
887 * The state to be updated.
889 * 0 on success, o negative value otherwise.
892 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
893 struct mlx5_flow_meter_info *fm,
894 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
895 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
897 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
898 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
900 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
902 struct mlx5_aso_mtr *aso_mtr = NULL;
903 uint32_t cbs_cir, ebs_eir, val;
905 if (priv->sh->meter_aso_en) {
906 fm->is_enable = !!is_enable;
907 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
908 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
911 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
915 /* Fill command parameters. */
916 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
917 mod_attr.flow_meter_parameter = in;
918 mod_attr.flow_meter_parameter_sz =
919 MLX5_ST_SZ_BYTES(flow_meter_parameters);
920 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
921 mod_attr.active = !!active_state;
925 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
926 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
927 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
928 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
930 MLX5_SET(flow_meter_parameters, attr,
932 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
934 MLX5_SET(flow_meter_parameters, attr,
937 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
938 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
940 MLX5_SET(flow_meter_parameters, attr,
942 val = cbs_cir & ASO_DSEG_MAN_MASK;
943 MLX5_SET(flow_meter_parameters, attr,
946 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
947 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
949 MLX5_SET(flow_meter_parameters, attr,
951 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
953 MLX5_SET(flow_meter_parameters, attr,
956 /* Apply modifications to meter only if it was created. */
957 if (fm->meter_action) {
958 ret = mlx5_glue->dv_modify_flow_action_meter
959 (fm->meter_action, &mod_attr,
960 rte_cpu_to_be_64(modify_bits));
964 /* Update succeedded modify meter parameters. */
965 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
966 fm->active_state = !!active_state;
981 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
982 struct mlx5_flow_meter_info *fm,
986 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
987 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
988 if (fm->bytes_dropped || fm->pkts_dropped) {
990 /* Alloc policer counters. */
991 fm->drop_cnt = mlx5_counter_alloc(dev);
997 mlx5_counter_free(dev, fm->drop_cnt);
1005 * Create meter rules.
1008 * Pointer to Ethernet device.
1009 * @param[in] meter_id
1012 * Pointer to rte meter parameters.
1014 * Meter shared with other flow or not.
1016 * Pointer to rte meter error structure.
1019 * 0 on success, a negative errno value otherwise and rte_errno is set.
1022 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1023 struct rte_mtr_params *params, int shared,
1024 struct rte_mtr_error *error)
1026 struct mlx5_priv *priv = dev->data->dev_private;
1027 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1028 struct mlx5_flow_meter_profile *fmp;
1029 struct mlx5_flow_meter_info *fm;
1030 struct mlx5_legacy_flow_meter *legacy_fm;
1031 struct mlx5_flow_meter_policy *mtr_policy = NULL;
1032 struct mlx5_indexed_pool_config flow_ipool_cfg = {
1036 .type = "mlx5_flow_mtr_flow_id_pool",
1038 struct mlx5_aso_mtr *aso_mtr;
1039 uint32_t mtr_idx, policy_idx;
1040 union mlx5_l3t_data data;
1042 uint8_t domain_bitmap;
1043 uint8_t mtr_id_bits;
1044 uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1045 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1048 return -rte_mtr_error_set(error, ENOTSUP,
1049 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1050 "Meter is not supported");
1051 /* Validate the parameters. */
1052 ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1055 /* Meter profile must exist. */
1056 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1058 return -rte_mtr_error_set(error, ENOENT,
1059 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1060 NULL, "Meter profile id not valid.");
1061 /* Meter policy must exist. */
1062 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1064 (&priv->sh->mtrmng->def_policy_ref_cnt,
1065 1, __ATOMIC_RELAXED);
1066 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1067 if (!priv->config.dv_esw_en)
1068 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1070 mtr_policy = mlx5_flow_meter_policy_find(dev,
1071 params->meter_policy_id, &policy_idx);
1072 if (!priv->sh->meter_aso_en)
1073 return -rte_mtr_error_set(error, ENOTSUP,
1074 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1075 "Part of the policies cannot be "
1076 "supported without ASO ");
1078 return -rte_mtr_error_set(error, ENOENT,
1079 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1080 NULL, "Meter policy id not valid.");
1081 domain_bitmap = (mtr_policy->ingress ?
1082 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1083 (mtr_policy->egress ?
1084 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1085 (mtr_policy->transfer ?
1086 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1088 /* Allocate the flow meter memory. */
1089 if (priv->sh->meter_aso_en) {
1090 mtr_idx = mlx5_flow_mtr_alloc(dev);
1092 return -rte_mtr_error_set(error, ENOMEM,
1093 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1094 "Memory alloc failed for meter.");
1095 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1098 legacy_fm = mlx5_ipool_zmalloc
1099 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1100 if (legacy_fm == NULL)
1101 return -rte_mtr_error_set(error, ENOMEM,
1102 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1103 "Memory alloc failed for meter.");
1104 legacy_fm->idx = mtr_idx;
1105 fm = &legacy_fm->fm;
1107 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1108 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1110 DRV_LOG(ERR, "Meter number exceeds max limit.");
1113 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1114 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1115 /* Fill the flow meter parameters. */
1116 fm->meter_id = meter_id;
1117 fm->policy_id = params->meter_policy_id;
1119 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1121 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1123 /* Add to the flow meter list. */
1124 if (!priv->sh->meter_aso_en)
1125 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1126 /* Add to the flow meter list. */
1127 fm->active_state = 1; /* Config meter starts as active. */
1129 fm->shared = !!shared;
1130 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1131 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1133 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1134 if (!fm->flow_ipool)
1137 rte_spinlock_init(&fm->sl);
1138 /* If ASO meter supported, update ASO flow meter by wqe. */
1139 if (priv->sh->meter_aso_en) {
1140 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1141 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1144 if (!priv->mtr_idx_tbl) {
1146 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1147 if (!priv->mtr_idx_tbl)
1150 data.dword = mtr_idx;
1151 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1155 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1158 mlx5_flow_destroy_mtr_tbls(dev, fm);
1159 /* Free policer counters. */
1161 mlx5_counter_free(dev, fm->drop_cnt);
1162 if (priv->sh->meter_aso_en)
1163 mlx5_flow_mtr_free(dev, mtr_idx);
1165 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1166 return -rte_mtr_error_set(error, ENOTSUP,
1167 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1168 NULL, "Failed to create devx meter.");
1172 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1173 struct mlx5_flow_meter_info *fm,
1176 struct mlx5_priv *priv = dev->data->dev_private;
1177 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1178 struct mlx5_flow_meter_profile *fmp;
1179 struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1180 struct mlx5_flow_meter_policy *mtr_policy;
1182 /* Meter object must not have any owner. */
1183 MLX5_ASSERT(!fm->ref_cnt);
1184 /* Get meter profile. */
1188 /* Update dependencies. */
1189 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1191 /* Remove from list. */
1192 if (!priv->sh->meter_aso_en) {
1193 legacy_fm = container_of(fm,
1194 struct mlx5_legacy_flow_meter, fm);
1195 TAILQ_REMOVE(fms, legacy_fm, next);
1197 /* Free drop counters. */
1199 mlx5_counter_free(dev, fm->drop_cnt);
1200 /* Free meter flow table. */
1201 if (fm->flow_ipool) {
1202 mlx5_ipool_destroy(fm->flow_ipool);
1205 mlx5_flow_destroy_mtr_tbls(dev, fm);
1207 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1208 1, __ATOMIC_RELAXED);
1209 if (priv->sh->meter_aso_en) {
1210 if (!fm->def_policy) {
1211 mtr_policy = mlx5_flow_meter_policy_find(dev,
1212 fm->policy_id, NULL);
1214 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1215 1, __ATOMIC_RELAXED);
1219 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1221 mlx5_flow_mtr_free(dev, mtr_idx);
1223 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1230 * Destroy meter rules.
1233 * Pointer to Ethernet device.
1234 * @param[in] meter_id
1237 * Pointer to rte meter error structure.
1240 * 0 on success, a negative errno value otherwise and rte_errno is set.
1243 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1244 struct rte_mtr_error *error)
1246 struct mlx5_priv *priv = dev->data->dev_private;
1247 struct mlx5_flow_meter_info *fm;
1248 uint32_t mtr_idx = 0;
1251 return -rte_mtr_error_set(error, ENOTSUP,
1252 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1254 "Meter is not supported");
1255 /* Meter object must exist. */
1256 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1258 return -rte_mtr_error_set(error, ENOENT,
1259 RTE_MTR_ERROR_TYPE_MTR_ID,
1261 "Meter object id not valid.");
1262 /* Meter object must not have any owner. */
1263 if (fm->ref_cnt > 0)
1264 return -rte_mtr_error_set(error, EBUSY,
1265 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1267 "Meter object is being used.");
1268 /* Destroy the meter profile. */
1269 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1270 return -rte_mtr_error_set(error, EINVAL,
1271 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1273 "MTR object meter profile invalid.");
1278 * Modify meter state.
1281 * Pointer to mlx5 private data structure.
1283 * Pointer to flow meter.
1284 * @param[in] new_state
1285 * New state to update.
1287 * Pointer to rte meter error structure.
1290 * 0 on success, a negative errno value otherwise and rte_errno is set.
1293 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1294 struct mlx5_flow_meter_info *fm,
1296 struct rte_mtr_error *error)
1298 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1299 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1302 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1303 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1306 if (new_state == MLX5_FLOW_METER_DISABLE)
1307 ret = mlx5_flow_meter_action_modify(priv, fm,
1308 &srtcm, modify_bits, 0, 0);
1310 ret = mlx5_flow_meter_action_modify(priv, fm,
1311 &fm->profile->srtcm_prm,
1314 return -rte_mtr_error_set(error, -ret,
1315 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1318 "Failed to enable meter." :
1319 "Failed to disable meter.");
1324 * Callback to enable flow meter.
1327 * Pointer to Ethernet device.
1328 * @param[in] meter_id
1331 * Pointer to rte meter error structure.
1334 * 0 on success, a negative errno value otherwise and rte_errno is set.
1337 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1339 struct rte_mtr_error *error)
1341 struct mlx5_priv *priv = dev->data->dev_private;
1342 struct mlx5_flow_meter_info *fm;
1346 return -rte_mtr_error_set(error, ENOTSUP,
1347 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1348 "Meter is not supported");
1349 /* Meter object must exist. */
1350 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1352 return -rte_mtr_error_set(error, ENOENT,
1353 RTE_MTR_ERROR_TYPE_MTR_ID,
1354 NULL, "Meter not found.");
1355 if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1357 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1360 fm->active_state = MLX5_FLOW_METER_ENABLE;
1365 * Callback to disable flow meter.
1368 * Pointer to Ethernet device.
1369 * @param[in] meter_id
1372 * Pointer to rte meter error structure.
1375 * 0 on success, a negative errno value otherwise and rte_errno is set.
1378 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1380 struct rte_mtr_error *error)
1382 struct mlx5_priv *priv = dev->data->dev_private;
1383 struct mlx5_flow_meter_info *fm;
1387 return -rte_mtr_error_set(error, ENOTSUP,
1388 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1389 "Meter is not supported");
1390 /* Meter object must exist. */
1391 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1393 return -rte_mtr_error_set(error, ENOENT,
1394 RTE_MTR_ERROR_TYPE_MTR_ID,
1395 NULL, "Meter not found.");
1396 if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1398 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1401 fm->active_state = MLX5_FLOW_METER_DISABLE;
1406 * Callback to update meter profile.
1409 * Pointer to Ethernet device.
1410 * @param[in] meter_id
1412 * @param[in] meter_profile_id
1413 * To be updated meter profile id.
1415 * Pointer to rte meter error structure.
1418 * 0 on success, a negative errno value otherwise and rte_errno is set.
1421 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1423 uint32_t meter_profile_id,
1424 struct rte_mtr_error *error)
1426 struct mlx5_priv *priv = dev->data->dev_private;
1427 struct mlx5_flow_meter_profile *fmp;
1428 struct mlx5_flow_meter_profile *old_fmp;
1429 struct mlx5_flow_meter_info *fm;
1430 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1431 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1435 return -rte_mtr_error_set(error, ENOTSUP,
1436 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1437 "Meter is not supported");
1438 /* Meter profile must exist. */
1439 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1441 return -rte_mtr_error_set(error, ENOENT,
1442 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1443 NULL, "Meter profile not found.");
1444 /* Meter object must exist. */
1445 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1447 return -rte_mtr_error_set(error, ENOENT,
1448 RTE_MTR_ERROR_TYPE_MTR_ID,
1449 NULL, "Meter not found.");
1450 /* MTR object already set to meter profile id. */
1451 old_fmp = fm->profile;
1454 /* Update the profile. */
1456 /* Update meter params in HW (if not disabled). */
1457 if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1459 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1460 modify_bits, fm->active_state, 1);
1462 fm->profile = old_fmp;
1463 return -rte_mtr_error_set(error, -ret,
1464 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1465 NULL, "Failed to update meter"
1466 " parmeters in hardware.");
1474 * Callback to update meter stats mask.
1477 * Pointer to Ethernet device.
1478 * @param[in] meter_id
1480 * @param[in] stats_mask
1481 * To be updated stats_mask.
1483 * Pointer to rte meter error structure.
1486 * 0 on success, a negative errno value otherwise and rte_errno is set.
1489 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1491 uint64_t stats_mask,
1492 struct rte_mtr_error *error)
1494 struct mlx5_priv *priv = dev->data->dev_private;
1495 struct mlx5_flow_meter_info *fm;
1498 return -rte_mtr_error_set(error, ENOTSUP,
1499 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1500 "Meter is not supported");
1501 /* Meter object must exist. */
1502 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1504 return -rte_mtr_error_set(error, ENOENT,
1505 RTE_MTR_ERROR_TYPE_MTR_ID,
1506 NULL, "Meter object id not valid.");
1507 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1508 return -rte_mtr_error_set(error, ENOENT,
1509 RTE_MTR_ERROR_TYPE_MTR_ID,
1510 NULL, "Fail to allocate "
1511 "counter for meter.");
1516 * Callback to read meter statistics.
1519 * Pointer to Ethernet device.
1520 * @param[in] meter_id
1523 * Pointer to store the statistics.
1524 * @param[out] stats_mask
1525 * Pointer to store the stats_mask.
1527 * Statistic to be cleared after read or not.
1529 * Pointer to rte meter error structure.
1532 * 0 on success, a negative errno value otherwise and rte_errno is set.
1535 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1537 struct rte_mtr_stats *stats,
1538 uint64_t *stats_mask,
1540 struct rte_mtr_error *error)
1542 struct mlx5_priv *priv = dev->data->dev_private;
1543 struct mlx5_flow_meter_info *fm;
1549 return -rte_mtr_error_set(error, ENOTSUP,
1550 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1551 "Meter is not supported");
1552 /* Meter object must exist. */
1553 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1555 return -rte_mtr_error_set(error, ENOENT,
1556 RTE_MTR_ERROR_TYPE_MTR_ID,
1557 NULL, "Meter object id not valid.");
1559 if (fm->bytes_dropped)
1560 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1561 if (fm->pkts_dropped)
1562 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1563 memset(stats, 0, sizeof(*stats));
1565 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1569 /* If need to read the packets, set it. */
1570 if (fm->pkts_dropped)
1571 stats->n_pkts_dropped = pkts;
1572 /* If need to read the bytes, set it. */
1573 if (fm->bytes_dropped)
1574 stats->n_bytes_dropped = bytes;
1578 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1579 "Failed to read meter drop counters.");
1582 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1583 .capabilities_get = mlx5_flow_mtr_cap_get,
1584 .meter_profile_add = mlx5_flow_meter_profile_add,
1585 .meter_profile_delete = mlx5_flow_meter_profile_delete,
1586 .meter_policy_validate = mlx5_flow_meter_policy_validate,
1587 .meter_policy_add = mlx5_flow_meter_policy_add,
1588 .meter_policy_delete = mlx5_flow_meter_policy_delete,
1589 .create = mlx5_flow_meter_create,
1590 .destroy = mlx5_flow_meter_destroy,
1591 .meter_enable = mlx5_flow_meter_enable,
1592 .meter_disable = mlx5_flow_meter_disable,
1593 .meter_profile_update = mlx5_flow_meter_profile_update,
1594 .meter_dscp_table_update = NULL,
1595 .stats_update = mlx5_flow_meter_stats_update,
1596 .stats_read = mlx5_flow_meter_stats_read,
1600 * Get meter operations.
1603 * Pointer to Ethernet device structure.
1605 * Pointer to set the mtr operations.
1611 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1613 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1621 * Pointer to mlx5_priv.
1625 * Pointer to Meter index.
1628 * Pointer to the meter info found on success, NULL otherwise.
1630 struct mlx5_flow_meter_info *
1631 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1634 struct mlx5_legacy_flow_meter *legacy_fm;
1635 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1636 struct mlx5_aso_mtr *aso_mtr;
1637 struct mlx5_aso_mtr_pools_mng *pools_mng =
1638 &priv->sh->mtrmng->pools_mng;
1639 union mlx5_l3t_data data;
1641 if (priv->sh->meter_aso_en) {
1642 rte_spinlock_lock(&pools_mng->mtrsl);
1643 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1644 rte_spinlock_unlock(&pools_mng->mtrsl);
1647 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1649 rte_spinlock_unlock(&pools_mng->mtrsl);
1653 *mtr_idx = data.dword;
1654 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1655 /* Remove reference taken by the mlx5_l3t_get_entry. */
1656 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1657 rte_spinlock_unlock(&pools_mng->mtrsl);
1658 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1660 return &aso_mtr->fm;
1662 TAILQ_FOREACH(legacy_fm, fms, next)
1663 if (meter_id == legacy_fm->fm.meter_id) {
1665 *mtr_idx = legacy_fm->idx;
1666 return &legacy_fm->fm;
1672 * Find meter by index.
1675 * Pointer to mlx5_priv.
1680 * Pointer to the meter info found on success, NULL otherwise.
1682 struct mlx5_flow_meter_info *
1683 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1685 struct mlx5_aso_mtr *aso_mtr;
1687 if (priv->sh->meter_aso_en) {
1688 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1691 return &aso_mtr->fm;
1693 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1698 * Attach meter to flow.
1699 * Unidirectional Meter creation can only be done
1700 * when flow direction is known, i.e. when calling meter_attach.
1703 * Pointer to mlx5 private data.
1705 * Pointer to flow meter.
1707 * Pointer to flow attributes.
1708 * @param [out] error
1709 * Pointer to error structure.
1712 * 0 on success, a negative errno value otherwise and rte_errno is set.
1715 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1716 struct mlx5_flow_meter_info *fm,
1717 const struct rte_flow_attr *attr,
1718 struct rte_flow_error *error)
1722 if (priv->sh->meter_aso_en) {
1723 struct mlx5_aso_mtr *aso_mtr;
1725 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1726 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1727 return rte_flow_error_set(error, ENOENT,
1728 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1730 "Timeout in meter configuration");
1732 rte_spinlock_lock(&fm->sl);
1733 if (fm->shared || !fm->ref_cnt) {
1736 rte_flow_error_set(error, EINVAL,
1737 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1738 "Meter cannot be shared");
1741 rte_spinlock_unlock(&fm->sl);
1743 rte_spinlock_lock(&fm->sl);
1744 if (fm->meter_action) {
1746 attr->transfer == fm->transfer &&
1747 attr->ingress == fm->ingress &&
1748 attr->egress == fm->egress) {
1751 rte_flow_error_set(error, EINVAL,
1752 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1754 "Meter attr not match." :
1755 "Meter cannot be shared.");
1759 fm->ingress = attr->ingress;
1760 fm->egress = attr->egress;
1761 fm->transfer = attr->transfer;
1763 /* This also creates the meter object. */
1764 fm->meter_action = mlx5_flow_meter_action_create(priv,
1766 if (!fm->meter_action) {
1771 rte_flow_error_set(error, EINVAL,
1772 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1773 "Meter action create failed.");
1777 rte_spinlock_unlock(&fm->sl);
1779 return ret ? -rte_errno : 0;
1783 * Detach meter from flow.
1786 * Pointer to mlx5 private data.
1788 * Pointer to flow meter.
1791 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1792 struct mlx5_flow_meter_info *fm)
1794 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1795 rte_spinlock_lock(&fm->sl);
1796 MLX5_ASSERT(fm->ref_cnt);
1797 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1798 mlx5_glue->destroy_flow_action(fm->meter_action);
1799 fm->meter_action = NULL;
1804 rte_spinlock_unlock(&fm->sl);
1812 * Flush meter with Rx queue configuration.
1815 * Pointer to Ethernet device.
1818 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1820 struct mlx5_priv *priv = dev->data->dev_private;
1821 struct mlx5_flow_meter_sub_policy *sub_policy;
1822 struct mlx5_flow_meter_policy *mtr_policy;
1824 uint32_t i, policy_idx;
1828 if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1829 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1831 policy_idx = *(uint32_t *)entry;
1832 sub_policy = mlx5_ipool_get
1833 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1835 if (!sub_policy || !sub_policy->main_policy)
1837 mtr_policy = sub_policy->main_policy;
1838 if (mtr_policy->is_queue || mtr_policy->is_rss)
1839 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1846 * Flush meter configuration.
1849 * Pointer to Ethernet device.
1851 * Pointer to rte meter error structure.
1854 * 0 on success, a negative errno value otherwise and rte_errno is set.
1857 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1859 struct mlx5_priv *priv = dev->data->dev_private;
1860 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1861 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
1862 struct mlx5_flow_meter_profile *fmp;
1863 struct mlx5_legacy_flow_meter *legacy_fm;
1864 struct mlx5_flow_meter_info *fm;
1865 struct mlx5_flow_meter_sub_policy *sub_policy;
1867 uint32_t i, mtr_idx, policy_idx;
1869 struct mlx5_aso_mtr *aso_mtr;
1873 if (priv->sh->meter_aso_en) {
1874 if (priv->mtr_idx_tbl) {
1875 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
1876 mtr_idx = *(uint32_t *)entry;
1879 mlx5_aso_meter_by_idx(priv, mtr_idx);
1881 (void)mlx5_flow_meter_params_flush(dev,
1885 mlx5_l3t_destroy(priv->mtr_idx_tbl);
1886 priv->mtr_idx_tbl = NULL;
1889 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1890 fm = &legacy_fm->fm;
1891 if (mlx5_flow_meter_params_flush(dev, fm, 0))
1892 return -rte_mtr_error_set(error, EINVAL,
1893 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1894 NULL, "MTR object meter profile invalid.");
1897 if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1898 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1900 policy_idx = *(uint32_t *)entry;
1901 sub_policy = mlx5_ipool_get
1902 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1905 return -rte_mtr_error_set(error,
1907 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1909 "meter policy invalid.");
1910 if (__mlx5_flow_meter_policy_delete(dev, i,
1911 sub_policy->main_policy,
1913 return -rte_mtr_error_set(error,
1915 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1917 "meter policy invalid.");
1918 mlx5_free(sub_policy->main_policy);
1920 mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl);
1921 priv->sh->mtrmng->policy_idx_tbl = NULL;
1923 TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) {
1925 MLX5_ASSERT(!fmp->ref_cnt);
1926 /* Remove from list. */
1927 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
1930 /* Delete default policy table. */
1931 mlx5_flow_destroy_def_policy(dev);
1932 if (priv->sh->refcnt == 1)
1933 mlx5_flow_destroy_mtr_drop_tbls(dev);