1 // SPDX-License-Identifier: BSD-3-Clause
3 * Copyright 2018 Mellanox Technologies, Ltd
8 #include <rte_malloc.h>
10 #include <rte_mtr_driver.h>
12 #include <mlx5_devx_cmds.h>
13 #include <mlx5_malloc.h>
16 #include "mlx5_flow.h"
19 * Create the meter action.
22 * Pointer to mlx5_priv.
24 * Pointer to flow meter to be converted.
27 * Pointer to the meter action on success, NULL otherwise.
30 mlx5_flow_meter_action_create(struct mlx5_priv *priv,
31 struct mlx5_flow_meter_info *fm)
33 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
34 struct mlx5dv_dr_flow_meter_attr mtr_init;
35 uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
36 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm =
37 &fm->profile->srtcm_prm;
38 uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
39 uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
41 enum mlx5_meter_domain domain =
42 fm->transfer ? MLX5_MTR_DOMAIN_TRANSFER :
43 fm->egress ? MLX5_MTR_DOMAIN_EGRESS :
44 MLX5_MTR_DOMAIN_INGRESS;
45 struct mlx5_flow_meter_def_policy *def_policy =
46 priv->sh->mtrmng->def_policy[domain];
48 memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters));
49 MLX5_SET(flow_meter_parameters, fmp, valid, 1);
50 MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1);
51 MLX5_SET(flow_meter_parameters, fmp,
52 start_color, MLX5_FLOW_COLOR_GREEN);
53 MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0);
54 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
55 MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val);
56 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
57 MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val);
58 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
59 MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val);
60 val = (cbs_cir & ASO_DSEG_MAN_MASK);
61 MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val);
62 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK;
63 MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val);
64 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK;
65 MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val);
66 mtr_init.next_table = def_policy->sub_policy.tbl_rsc->obj;
67 mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0;
68 mtr_init.flow_meter_parameter = fmp;
69 mtr_init.flow_meter_parameter_sz =
70 MLX5_ST_SZ_BYTES(flow_meter_parameters);
71 mtr_init.active = fm->active_state;
72 return mlx5_glue->dv_create_flow_action_meter(&mtr_init);
81 * Find meter profile by id.
84 * Pointer to mlx5_priv.
85 * @param meter_profile_id
89 * Pointer to the profile found on success, NULL otherwise.
91 static struct mlx5_flow_meter_profile *
92 mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id)
94 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
95 struct mlx5_flow_meter_profile *fmp;
97 TAILQ_FOREACH(fmp, fmps, next)
98 if (meter_profile_id == fmp->id)
104 * Validate the MTR profile.
107 * Pointer to Ethernet device.
108 * @param[in] meter_profile_id
111 * Pointer to meter profile detail.
113 * Pointer to the error structure.
116 * 0 on success, a negative errno value otherwise and rte_errno is set.
119 mlx5_flow_meter_profile_validate(struct rte_eth_dev *dev,
120 uint32_t meter_profile_id,
121 struct rte_mtr_meter_profile *profile,
122 struct rte_mtr_error *error)
124 struct mlx5_priv *priv = dev->data->dev_private;
125 struct mlx5_flow_meter_profile *fmp;
127 /* Profile must not be NULL. */
129 return -rte_mtr_error_set(error, EINVAL,
130 RTE_MTR_ERROR_TYPE_METER_PROFILE,
131 NULL, "Meter profile is null.");
132 /* Meter profile ID must be valid. */
133 if (meter_profile_id == UINT32_MAX)
134 return -rte_mtr_error_set(error, EINVAL,
135 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
136 NULL, "Meter profile id not valid.");
137 /* Meter profile must not exist. */
138 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
140 return -rte_mtr_error_set(error, EEXIST,
141 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
143 "Meter profile already exists.");
144 if (profile->alg == RTE_MTR_SRTCM_RFC2697) {
145 if (priv->config.hca_attr.qos.flow_meter_old) {
146 /* Verify support for flow meter parameters. */
147 if (priv->sh->meter_aso_en && profile->packet_mode) {
148 if (profile->srtcm_rfc2697.cir > 0 &&
149 (profile->srtcm_rfc2697.cir <<
150 MLX5_MTRS_PPS_MAP_BPS_SHIFT)
151 <= MLX5_SRTCM_CIR_MAX &&
152 profile->srtcm_rfc2697.cbs > 0 &&
153 (profile->srtcm_rfc2697.cbs <<
154 MLX5_MTRS_PPS_MAP_BPS_SHIFT)
155 <= MLX5_SRTCM_CBS_MAX &&
156 (profile->srtcm_rfc2697.ebs <<
157 MLX5_MTRS_PPS_MAP_BPS_SHIFT)
158 <= MLX5_SRTCM_EBS_MAX)
160 return -rte_mtr_error_set
162 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
164 profile->srtcm_rfc2697.ebs ?
165 "Metering value ebs must be 0." :
166 "Invalid metering parameters.");
168 if (profile->srtcm_rfc2697.cir > 0 &&
169 profile->srtcm_rfc2697.cir <=
170 MLX5_SRTCM_CIR_MAX &&
171 profile->srtcm_rfc2697.cbs > 0 &&
172 profile->srtcm_rfc2697.cbs <=
173 MLX5_SRTCM_CBS_MAX &&
174 profile->srtcm_rfc2697.ebs <=
177 return -rte_mtr_error_set(error, ENOTSUP,
178 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
180 profile->srtcm_rfc2697.ebs ?
181 "Metering value ebs must be 0." :
182 "Invalid metering parameters.");
185 return -rte_mtr_error_set(error, ENOTSUP,
186 RTE_MTR_ERROR_TYPE_METER_PROFILE,
187 NULL, "Metering algorithm not supported.");
191 * Calculate mantissa and exponent for cir.
194 * Value to be calculated.
196 * Pointer to the mantissa.
198 * Pointer to the exp.
201 mlx5_flow_meter_cir_man_exp_calc(int64_t cir, uint8_t *man, uint8_t *exp)
204 int64_t delta = INT64_MAX;
209 for (m = 0; m <= 0xFF; m++) { /* man width 8 bit */
210 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
211 _cir = (1000000000ULL * m) >> e;
212 if (llabs(cir - _cir) <= delta) {
213 delta = llabs(cir - _cir);
224 * Calculate mantissa and exponent for xbs.
227 * Value to be calculated.
229 * Pointer to the mantissa.
231 * Pointer to the exp.
234 mlx5_flow_meter_xbs_man_exp_calc(uint64_t xbs, uint8_t *man, uint8_t *exp)
239 /* Special case xbs == 0 ? both exp and matissa are 0. */
245 /* xbs = xbs_mantissa * 2^xbs_exponent */
246 _man = frexp(xbs, &_exp);
247 _man = _man * pow(2, MLX5_MAN_WIDTH);
248 _exp = _exp - MLX5_MAN_WIDTH;
249 *man = (uint8_t)ceil(_man);
254 * Fill the prm meter parameter.
257 * Pointer to meter profie to be converted.
259 * Pointer to the error structure.
262 * 0 on success, a negative errno value otherwise and rte_errno is set.
265 mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
266 struct mlx5_priv *priv, struct rte_mtr_error *error)
268 struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm;
270 uint32_t cbs_exp, cbs_man, cir_exp, cir_man;
271 uint32_t ebs_exp, ebs_man;
272 uint64_t cir, cbs, ebs;
274 if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697)
275 return -rte_mtr_error_set(error, ENOTSUP,
276 RTE_MTR_ERROR_TYPE_METER_PROFILE,
277 NULL, "Metering algorithm not supported.");
278 if (!priv->sh->meter_aso_en && fmp->profile.packet_mode)
279 return -rte_mtr_error_set(error, ENOTSUP,
280 RTE_MTR_ERROR_TYPE_METER_PROFILE,
281 NULL, "Metering algorithm packet mode not supported.");
282 if (priv->sh->meter_aso_en && fmp->profile.packet_mode) {
283 cir = fmp->profile.srtcm_rfc2697.cir <<
284 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
285 cbs = fmp->profile.srtcm_rfc2697.cbs <<
286 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
287 ebs = fmp->profile.srtcm_rfc2697.ebs <<
288 MLX5_MTRS_PPS_MAP_BPS_SHIFT;
290 cir = fmp->profile.srtcm_rfc2697.cir;
291 cbs = fmp->profile.srtcm_rfc2697.cbs;
292 ebs = fmp->profile.srtcm_rfc2697.ebs;
294 /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */
295 mlx5_flow_meter_cir_man_exp_calc(cir, &man, &exp);
296 /* Check if cir mantissa is too large. */
297 if (exp > ASO_DSEG_CIR_EXP_MASK)
298 return -rte_mtr_error_set(error, ENOTSUP,
299 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
300 "meter profile parameter cir is"
304 /* cbs = cbs_mantissa * 2^cbs_exponent */
305 mlx5_flow_meter_xbs_man_exp_calc(cbs, &man, &exp);
306 /* Check if cbs mantissa is too large. */
307 if (exp > ASO_DSEG_EXP_MASK)
308 return -rte_mtr_error_set(error, ENOTSUP,
309 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
310 "meter profile parameter cbs is"
314 srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET |
315 cbs_man << ASO_DSEG_CBS_MAN_OFFSET |
316 cir_exp << ASO_DSEG_CIR_EXP_OFFSET |
318 mlx5_flow_meter_xbs_man_exp_calc(ebs, &man, &exp);
319 /* Check if ebs mantissa is too large. */
320 if (exp > ASO_DSEG_EXP_MASK)
321 return -rte_mtr_error_set(error, ENOTSUP,
322 RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL,
323 "meter profile parameter ebs is"
327 srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET |
328 ebs_man << ASO_DSEG_EBS_MAN_OFFSET);
333 * Callback to get MTR capabilities.
336 * Pointer to Ethernet device.
338 * Pointer to save MTR capabilities.
340 * Pointer to the error structure.
343 * 0 on success, a negative errno value otherwise and rte_errno is set.
346 mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
347 struct rte_mtr_capabilities *cap,
348 struct rte_mtr_error *error __rte_unused)
350 struct mlx5_priv *priv = dev->data->dev_private;
351 struct mlx5_hca_qos_attr *qattr = &priv->config.hca_attr.qos;
354 return -rte_mtr_error_set(error, ENOTSUP,
355 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
356 "Meter is not supported");
357 memset(cap, 0, sizeof(*cap));
358 if (priv->sh->meter_aso_en) {
359 /* 2 meters per one ASO cache line. */
360 cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
361 cap->srtcm_rfc2697_packet_mode_supported = 1;
363 cap->n_max = 1 << qattr->log_max_flow_meter;
364 cap->srtcm_rfc2697_packet_mode_supported = 0;
366 cap->srtcm_rfc2697_byte_mode_supported = 1;
367 cap->n_shared_max = cap->n_max;
369 cap->shared_identical = 1;
370 cap->shared_n_flows_per_mtr_max = 4 << 20;
371 /* 2M flows can share the same meter. */
372 cap->chaining_n_mtrs_per_flow_max = 1; /* Chaining is not supported. */
373 cap->meter_srtcm_rfc2697_n_max = qattr->flow_meter_old ? cap->n_max : 0;
374 cap->meter_rate_max = 1ULL << 40; /* 1 Tera tokens per sec. */
375 cap->stats_mask = RTE_MTR_STATS_N_BYTES_DROPPED |
376 RTE_MTR_STATS_N_PKTS_DROPPED;
381 * Callback to add MTR profile.
384 * Pointer to Ethernet device.
385 * @param[in] meter_profile_id
388 * Pointer to meter profile detail.
390 * Pointer to the error structure.
393 * 0 on success, a negative errno value otherwise and rte_errno is set.
396 mlx5_flow_meter_profile_add(struct rte_eth_dev *dev,
397 uint32_t meter_profile_id,
398 struct rte_mtr_meter_profile *profile,
399 struct rte_mtr_error *error)
401 struct mlx5_priv *priv = dev->data->dev_private;
402 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
403 struct mlx5_flow_meter_profile *fmp;
407 return -rte_mtr_error_set(error, ENOTSUP,
408 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
409 "Meter is not supported");
410 /* Check input params. */
411 ret = mlx5_flow_meter_profile_validate(dev, meter_profile_id,
415 /* Meter profile memory allocation. */
416 fmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flow_meter_profile),
417 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
419 return -rte_mtr_error_set(error, ENOMEM,
420 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
421 NULL, "Meter profile memory "
423 /* Fill profile info. */
424 fmp->id = meter_profile_id;
425 fmp->profile = *profile;
426 /* Fill the flow meter parameters for the PRM. */
427 ret = mlx5_flow_meter_param_fill(fmp, priv, error);
431 TAILQ_INSERT_TAIL(fmps, fmp, next);
439 * Callback to delete MTR profile.
442 * Pointer to Ethernet device.
443 * @param[in] meter_profile_id
446 * Pointer to the error structure.
449 * 0 on success, a negative errno value otherwise and rte_errno is set.
452 mlx5_flow_meter_profile_delete(struct rte_eth_dev *dev,
453 uint32_t meter_profile_id,
454 struct rte_mtr_error *error)
456 struct mlx5_priv *priv = dev->data->dev_private;
457 struct mlx5_flow_meter_profile *fmp;
460 return -rte_mtr_error_set(error, ENOTSUP,
461 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
462 "Meter is not supported");
463 /* Meter profile must exist. */
464 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
466 return -rte_mtr_error_set(error, ENOENT,
467 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
469 "Meter profile id is invalid.");
470 /* Check profile is unused. */
472 return -rte_mtr_error_set(error, EBUSY,
473 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
474 NULL, "Meter profile is in use.");
475 /* Remove from list. */
476 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
485 * Pointer to Ethernet device.
490 * Pointer to the policy found on success, NULL otherwise.
492 struct mlx5_flow_meter_policy *
493 mlx5_flow_meter_policy_find(struct rte_eth_dev *dev,
495 uint32_t *policy_idx)
497 struct mlx5_priv *priv = dev->data->dev_private;
498 struct mlx5_flow_meter_sub_policy *sub_policy = NULL;
499 union mlx5_l3t_data data;
501 if (policy_id > MLX5_MAX_SUB_POLICY_TBL_NUM ||
502 !priv->sh->mtrmng->policy_idx_tbl)
504 if (mlx5_l3t_get_entry(priv->sh->mtrmng->policy_idx_tbl,
509 *policy_idx = data.dword;
510 sub_policy = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
512 /* Remove reference taken by the mlx5_l3t_get_entry. */
513 mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
516 if (sub_policy->main_policy_id)
517 return sub_policy->main_policy;
522 * Callback to check MTR policy action validate
525 * Pointer to Ethernet device.
527 * Pointer to meter policy action detail.
529 * Pointer to the error structure.
532 * 0 on success, a negative errno value otherwise and rte_errno is set.
535 mlx5_flow_meter_policy_validate(struct rte_eth_dev *dev,
536 struct rte_mtr_meter_policy_params *policy,
537 struct rte_mtr_error *error)
539 struct mlx5_priv *priv = dev->data->dev_private;
540 struct rte_flow_attr attr = { .transfer =
541 priv->config.dv_esw_en ? 1 : 0};
543 bool is_def_policy = false;
544 uint8_t domain_bitmap;
547 if (!priv->mtr_en || !priv->sh->meter_aso_en)
548 return -rte_mtr_error_set(error, ENOTSUP,
549 RTE_MTR_ERROR_TYPE_METER_POLICY,
550 NULL, "meter policy unsupported.");
551 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
552 &is_rss, &domain_bitmap, &is_def_policy, error);
559 __mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
561 struct mlx5_flow_meter_policy *mtr_policy,
562 struct rte_mtr_error *error,
565 struct mlx5_priv *priv = dev->data->dev_private;
566 struct mlx5_flow_meter_sub_policy *sub_policy;
568 uint16_t sub_policy_num;
570 rte_spinlock_lock(&mtr_policy->sl);
571 if (mtr_policy->ref_cnt) {
572 rte_spinlock_unlock(&mtr_policy->sl);
573 return -rte_mtr_error_set(error, EBUSY,
574 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
576 "Meter policy object is being used.");
578 mlx5_flow_destroy_policy_rules(dev, mtr_policy);
579 mlx5_flow_destroy_mtr_acts(dev, mtr_policy);
580 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
581 sub_policy_num = (mtr_policy->sub_policy_num >>
582 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
583 MLX5_MTR_SUB_POLICY_NUM_MASK;
584 if (sub_policy_num) {
585 for (j = 0; j < sub_policy_num; j++) {
586 sub_policy = mtr_policy->sub_policys[i][j];
589 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
594 if (priv->sh->mtrmng->policy_idx_tbl && clear_l3t) {
595 if (mlx5_l3t_clear_entry(priv->sh->mtrmng->policy_idx_tbl,
597 rte_spinlock_unlock(&mtr_policy->sl);
598 return -rte_mtr_error_set(error, ENOTSUP,
599 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
600 "Fail to delete policy in index table.");
603 rte_spinlock_unlock(&mtr_policy->sl);
608 * Callback to add MTR policy.
611 * Pointer to Ethernet device.
612 * @param[out] policy_id
613 * Pointer to policy id
615 * Pointer to meter policy action detail.
617 * Pointer to the error structure.
620 * 0 on success, a negative errno value otherwise and rte_errno is set.
623 mlx5_flow_meter_policy_add(struct rte_eth_dev *dev,
625 struct rte_mtr_meter_policy_params *policy,
626 struct rte_mtr_error *error)
628 struct mlx5_priv *priv = dev->data->dev_private;
629 struct rte_flow_attr attr = { .transfer =
630 priv->config.dv_esw_en ? 1 : 0};
631 uint32_t sub_policy_idx = 0;
632 uint32_t policy_idx = 0;
633 struct mlx5_flow_meter_policy *mtr_policy = NULL;
634 struct mlx5_flow_meter_sub_policy *sub_policy;
636 bool is_def_policy = false;
639 uint32_t policy_size = sizeof(struct mlx5_flow_meter_policy);
640 uint16_t sub_policy_num;
641 uint8_t domain_bitmap = 0;
642 union mlx5_l3t_data data;
645 return -rte_mtr_error_set(error, ENOTSUP,
646 RTE_MTR_ERROR_TYPE_METER_POLICY,
647 NULL, "meter policy unsupported.");
648 if (policy_id == MLX5_INVALID_POLICY_ID)
649 return -rte_mtr_error_set(error, ENOTSUP,
650 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
651 "policy ID is invalid. ");
652 if (policy_id == priv->sh->mtrmng->def_policy_id)
653 return -rte_mtr_error_set(error, EEXIST,
654 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
655 "policy ID exists. ");
656 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id,
659 return -rte_mtr_error_set(error, EEXIST,
660 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
661 "policy ID exists. ");
662 ret = mlx5_flow_validate_mtr_acts(dev, policy->actions, &attr,
663 &is_rss, &domain_bitmap, &is_def_policy, error);
667 return -rte_mtr_error_set(error, ENOTSUP,
668 RTE_MTR_ERROR_TYPE_METER_POLICY,
669 NULL, "fail to find policy domain.");
671 if (priv->sh->mtrmng->def_policy_id != MLX5_INVALID_POLICY_ID)
672 return -rte_mtr_error_set(error, EEXIST,
673 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
674 NULL, "a policy with similar actions "
675 "is already configured");
676 if (mlx5_flow_create_def_policy(dev))
677 return -rte_mtr_error_set(error, ENOTSUP,
678 RTE_MTR_ERROR_TYPE_METER_POLICY,
680 "fail to create non-terminated policy.");
681 priv->sh->mtrmng->def_policy_id = policy_id;
684 if (!priv->sh->meter_aso_en)
685 return -rte_mtr_error_set(error, ENOTSUP,
686 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
687 "no ASO capability to support the policy ");
688 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
689 if (!(domain_bitmap & (1 << i)))
693 sizeof(struct mlx5_flow_meter_sub_policy *) *
694 MLX5_MTR_RSS_MAX_SUB_POLICY;
697 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
699 mtr_policy = mlx5_malloc(MLX5_MEM_ZERO, policy_size,
700 RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
702 return -rte_mtr_error_set(error, ENOMEM,
703 RTE_MTR_ERROR_TYPE_METER_POLICY, NULL,
704 "Memory alloc failed for meter policy.");
705 policy_size = sizeof(struct mlx5_flow_meter_policy);
706 for (i = 0; i < MLX5_MTR_DOMAIN_MAX; i++) {
707 if (!(domain_bitmap & (1 << i)))
709 if (i == MLX5_MTR_DOMAIN_INGRESS)
710 mtr_policy->ingress = 1;
711 if (i == MLX5_MTR_DOMAIN_EGRESS)
712 mtr_policy->egress = 1;
713 if (i == MLX5_MTR_DOMAIN_TRANSFER)
714 mtr_policy->transfer = 1;
715 sub_policy = mlx5_ipool_zmalloc
716 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
720 if (sub_policy_idx > MLX5_MAX_SUB_POLICY_TBL_NUM)
722 sub_policy->idx = sub_policy_idx;
723 sub_policy->main_policy = mtr_policy;
725 policy_idx = sub_policy_idx;
726 sub_policy->main_policy_id = 1;
728 mtr_policy->sub_policys[i] =
729 (struct mlx5_flow_meter_sub_policy **)
730 ((uint8_t *)mtr_policy + policy_size);
731 mtr_policy->sub_policys[i][0] = sub_policy;
732 sub_policy_num = (mtr_policy->sub_policy_num >>
733 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i)) &
734 MLX5_MTR_SUB_POLICY_NUM_MASK;
736 mtr_policy->sub_policy_num &= ~(MLX5_MTR_SUB_POLICY_NUM_MASK <<
737 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i));
738 mtr_policy->sub_policy_num |=
739 (sub_policy_num & MLX5_MTR_SUB_POLICY_NUM_MASK) <<
740 (MLX5_MTR_SUB_POLICY_NUM_SHIFT * i);
742 mtr_policy->is_rss = 1;
745 policy_size += sizeof(struct mlx5_flow_meter_sub_policy *);
747 rte_spinlock_init(&mtr_policy->sl);
748 ret = mlx5_flow_create_mtr_acts(dev, mtr_policy,
749 policy->actions, error);
752 if (!is_rss && !mtr_policy->is_queue) {
753 /* Create policy rules in HW. */
754 ret = mlx5_flow_create_policy_rules(dev, mtr_policy);
758 data.dword = policy_idx;
759 if (!priv->sh->mtrmng->policy_idx_tbl) {
760 priv->sh->mtrmng->policy_idx_tbl =
761 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
762 if (!priv->sh->mtrmng->policy_idx_tbl)
765 if (mlx5_l3t_set_entry(priv->sh->mtrmng->policy_idx_tbl,
771 ret = __mlx5_flow_meter_policy_delete(dev, policy_id,
772 mtr_policy, error, false);
773 mlx5_free(mtr_policy);
777 return -rte_mtr_error_set(error, ENOTSUP,
778 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
779 NULL, "Failed to create devx policy.");
783 * Callback to delete MTR policy.
786 * Pointer to Ethernet device.
787 * @param[in] policy_id
790 * Pointer to the error structure.
793 * 0 on success, a negative errno value otherwise and rte_errno is set.
796 mlx5_flow_meter_policy_delete(struct rte_eth_dev *dev,
798 struct rte_mtr_error *error)
800 struct mlx5_priv *priv = dev->data->dev_private;
801 struct mlx5_flow_meter_policy *mtr_policy;
805 if (policy_id == priv->sh->mtrmng->def_policy_id) {
806 if (priv->sh->mtrmng->def_policy_ref_cnt > 0)
807 return -rte_mtr_error_set(error, ENOTSUP,
808 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
809 "Meter policy object is being used.");
810 priv->sh->mtrmng->def_policy_id = MLX5_INVALID_POLICY_ID;
813 mtr_policy = mlx5_flow_meter_policy_find(dev, policy_id, &policy_idx);
815 return -rte_mtr_error_set(error, ENOTSUP,
816 RTE_MTR_ERROR_TYPE_METER_POLICY_ID, NULL,
817 "Meter policy id is invalid. ");
818 ret = __mlx5_flow_meter_policy_delete(dev, policy_id, mtr_policy,
822 mlx5_free(mtr_policy);
827 * Check meter validation.
830 * Pointer to mlx5 private data structure.
831 * @param[in] meter_id
834 * Pointer to rte meter parameters.
836 * Pointer to rte meter error structure.
839 * 0 on success, a negative errno value otherwise and rte_errno is set.
842 mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id,
843 struct rte_mtr_params *params,
844 struct rte_mtr_error *error)
846 /* Meter must use global drop action. */
847 if (!priv->sh->dr_drop_action)
848 return -rte_mtr_error_set(error, ENOTSUP,
849 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
851 "No drop action ready for meter.");
852 /* Meter params must not be NULL. */
854 return -rte_mtr_error_set(error, EINVAL,
855 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
856 NULL, "Meter object params null.");
857 /* Previous meter color is not supported. */
858 if (params->use_prev_mtr_color)
859 return -rte_mtr_error_set(error, ENOTSUP,
860 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
862 "Previous meter color "
864 if (params->meter_policy_id == MLX5_INVALID_POLICY_ID)
865 return -rte_mtr_error_set(error, ENOENT,
866 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
867 NULL, "Meter policy id not valid.");
868 /* Validate meter id. */
869 if (mlx5_flow_meter_find(priv, meter_id, NULL))
870 return -rte_mtr_error_set(error, EEXIST,
871 RTE_MTR_ERROR_TYPE_MTR_ID, NULL,
872 "Meter object already exists.");
877 * Modify the flow meter action.
880 * Pointer to mlx5 private data structure.
882 * Pointer to flow meter to be modified.
884 * Pointer to meter srtcm description parameter.
885 * @param[in] modify_bits
886 * The bit in srtcm to be updated.
887 * @param[in] active_state
888 * The state to be updated.
890 * 0 on success, o negative value otherwise.
893 mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
894 struct mlx5_flow_meter_info *fm,
895 const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm,
896 uint64_t modify_bits, uint32_t active_state, uint32_t is_enable)
898 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
899 uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 };
901 struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 };
903 struct mlx5_aso_mtr *aso_mtr = NULL;
904 uint32_t cbs_cir, ebs_eir, val;
906 if (priv->sh->meter_aso_en) {
907 fm->is_enable = !!is_enable;
908 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
909 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
912 ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr);
916 /* Fill command parameters. */
917 mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0;
918 mod_attr.flow_meter_parameter = in;
919 mod_attr.flow_meter_parameter_sz =
920 MLX5_ST_SZ_BYTES(flow_meter_parameters);
921 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
922 mod_attr.active = !!active_state;
926 cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir);
927 ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir);
928 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) {
929 val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) &
931 MLX5_SET(flow_meter_parameters, attr,
933 val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) &
935 MLX5_SET(flow_meter_parameters, attr,
938 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) {
939 val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) &
941 MLX5_SET(flow_meter_parameters, attr,
943 val = cbs_cir & ASO_DSEG_MAN_MASK;
944 MLX5_SET(flow_meter_parameters, attr,
947 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) {
948 val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) &
950 MLX5_SET(flow_meter_parameters, attr,
952 val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) &
954 MLX5_SET(flow_meter_parameters, attr,
957 /* Apply modifications to meter only if it was created. */
958 if (fm->meter_action) {
959 ret = mlx5_glue->dv_modify_flow_action_meter
960 (fm->meter_action, &mod_attr,
961 rte_cpu_to_be_64(modify_bits));
965 /* Update succeedded modify meter parameters. */
966 if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE)
967 fm->active_state = !!active_state;
982 mlx5_flow_meter_stats_enable_update(struct rte_eth_dev *dev,
983 struct mlx5_flow_meter_info *fm,
987 (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0;
988 fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0;
989 if (fm->bytes_dropped || fm->pkts_dropped) {
991 /* Alloc policer counters. */
992 fm->drop_cnt = mlx5_counter_alloc(dev);
998 mlx5_counter_free(dev, fm->drop_cnt);
1006 * Create meter rules.
1009 * Pointer to Ethernet device.
1010 * @param[in] meter_id
1013 * Pointer to rte meter parameters.
1015 * Meter shared with other flow or not.
1017 * Pointer to rte meter error structure.
1020 * 0 on success, a negative errno value otherwise and rte_errno is set.
1023 mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id,
1024 struct rte_mtr_params *params, int shared,
1025 struct rte_mtr_error *error)
1027 struct mlx5_priv *priv = dev->data->dev_private;
1028 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1029 struct mlx5_flow_meter_profile *fmp;
1030 struct mlx5_flow_meter_info *fm;
1031 struct mlx5_legacy_flow_meter *legacy_fm;
1032 struct mlx5_flow_meter_policy *mtr_policy = NULL;
1033 struct mlx5_indexed_pool_config flow_ipool_cfg = {
1037 .type = "mlx5_flow_mtr_flow_id_pool",
1039 struct mlx5_aso_mtr *aso_mtr;
1040 uint32_t mtr_idx, policy_idx;
1041 union mlx5_l3t_data data;
1043 uint8_t domain_bitmap;
1044 uint8_t mtr_id_bits;
1045 uint8_t mtr_reg_bits = priv->mtr_reg_share ?
1046 MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
1049 return -rte_mtr_error_set(error, ENOTSUP,
1050 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1051 "Meter is not supported");
1052 /* Validate the parameters. */
1053 ret = mlx5_flow_meter_validate(priv, meter_id, params, error);
1056 /* Meter profile must exist. */
1057 fmp = mlx5_flow_meter_profile_find(priv, params->meter_profile_id);
1059 return -rte_mtr_error_set(error, ENOENT,
1060 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1061 NULL, "Meter profile id not valid.");
1062 /* Meter policy must exist. */
1063 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1065 (&priv->sh->mtrmng->def_policy_ref_cnt,
1066 1, __ATOMIC_RELAXED);
1067 domain_bitmap = MLX5_MTR_ALL_DOMAIN_BIT;
1068 if (!priv->config.dv_esw_en)
1069 domain_bitmap &= ~MLX5_MTR_DOMAIN_TRANSFER_BIT;
1071 mtr_policy = mlx5_flow_meter_policy_find(dev,
1072 params->meter_policy_id, &policy_idx);
1073 if (!priv->sh->meter_aso_en)
1074 return -rte_mtr_error_set(error, ENOTSUP,
1075 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1076 "Part of the policies cannot be "
1077 "supported without ASO ");
1079 return -rte_mtr_error_set(error, ENOENT,
1080 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1081 NULL, "Meter policy id not valid.");
1082 domain_bitmap = (mtr_policy->ingress ?
1083 MLX5_MTR_DOMAIN_INGRESS_BIT : 0) |
1084 (mtr_policy->egress ?
1085 MLX5_MTR_DOMAIN_EGRESS_BIT : 0) |
1086 (mtr_policy->transfer ?
1087 MLX5_MTR_DOMAIN_TRANSFER_BIT : 0);
1089 /* Allocate the flow meter memory. */
1090 if (priv->sh->meter_aso_en) {
1091 mtr_idx = mlx5_flow_mtr_alloc(dev);
1093 return -rte_mtr_error_set(error, ENOMEM,
1094 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1095 "Memory alloc failed for meter.");
1096 aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx);
1099 legacy_fm = mlx5_ipool_zmalloc
1100 (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx);
1101 if (legacy_fm == NULL)
1102 return -rte_mtr_error_set(error, ENOMEM,
1103 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1104 "Memory alloc failed for meter.");
1105 legacy_fm->idx = mtr_idx;
1106 fm = &legacy_fm->fm;
1108 mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx);
1109 if ((mtr_id_bits + priv->sh->mtrmng->max_mtr_flow_bits) >
1111 DRV_LOG(ERR, "Meter number exceeds max limit.");
1114 if (mtr_id_bits > priv->sh->mtrmng->max_mtr_bits)
1115 priv->sh->mtrmng->max_mtr_bits = mtr_id_bits;
1116 /* Fill the flow meter parameters. */
1117 fm->meter_id = meter_id;
1118 fm->policy_id = params->meter_policy_id;
1120 if (mlx5_flow_meter_stats_enable_update(dev, fm, params->stats_mask))
1122 if (mlx5_flow_create_mtr_tbls(dev, fm, mtr_idx, domain_bitmap))
1124 /* Add to the flow meter list. */
1125 if (!priv->sh->meter_aso_en)
1126 TAILQ_INSERT_TAIL(fms, legacy_fm, next);
1127 /* Add to the flow meter list. */
1128 fm->active_state = 1; /* Config meter starts as active. */
1130 fm->shared = !!shared;
1131 __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED);
1132 if (params->meter_policy_id == priv->sh->mtrmng->def_policy_id) {
1134 fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
1135 if (!fm->flow_ipool)
1138 rte_spinlock_init(&fm->sl);
1139 /* If ASO meter supported, update ASO flow meter by wqe. */
1140 if (priv->sh->meter_aso_en) {
1141 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1142 ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr);
1145 if (!priv->mtr_idx_tbl) {
1147 mlx5_l3t_create(MLX5_L3T_TYPE_DWORD);
1148 if (!priv->mtr_idx_tbl)
1151 data.dword = mtr_idx;
1152 if (mlx5_l3t_set_entry(priv->mtr_idx_tbl, meter_id, &data))
1156 __atomic_add_fetch(&mtr_policy->ref_cnt, 1, __ATOMIC_RELAXED);
1159 mlx5_flow_destroy_mtr_tbls(dev, fm);
1160 /* Free policer counters. */
1162 mlx5_counter_free(dev, fm->drop_cnt);
1163 if (priv->sh->meter_aso_en)
1164 mlx5_flow_mtr_free(dev, mtr_idx);
1166 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx);
1167 return -rte_mtr_error_set(error, ENOTSUP,
1168 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1169 NULL, "Failed to create devx meter.");
1173 mlx5_flow_meter_params_flush(struct rte_eth_dev *dev,
1174 struct mlx5_flow_meter_info *fm,
1177 struct mlx5_priv *priv = dev->data->dev_private;
1178 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1179 struct mlx5_flow_meter_profile *fmp;
1180 struct mlx5_legacy_flow_meter *legacy_fm = NULL;
1181 struct mlx5_flow_meter_policy *mtr_policy;
1183 /* Meter object must not have any owner. */
1184 MLX5_ASSERT(!fm->ref_cnt);
1185 /* Get meter profile. */
1189 /* Update dependencies. */
1190 __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED);
1192 /* Remove from list. */
1193 if (!priv->sh->meter_aso_en) {
1194 legacy_fm = container_of(fm,
1195 struct mlx5_legacy_flow_meter, fm);
1196 TAILQ_REMOVE(fms, legacy_fm, next);
1198 /* Free drop counters. */
1200 mlx5_counter_free(dev, fm->drop_cnt);
1201 /* Free meter flow table. */
1202 if (fm->flow_ipool) {
1203 mlx5_ipool_destroy(fm->flow_ipool);
1206 mlx5_flow_destroy_mtr_tbls(dev, fm);
1208 __atomic_sub_fetch(&priv->sh->mtrmng->def_policy_ref_cnt,
1209 1, __ATOMIC_RELAXED);
1210 if (priv->sh->meter_aso_en) {
1211 if (!fm->def_policy) {
1212 mtr_policy = mlx5_flow_meter_policy_find(dev,
1213 fm->policy_id, NULL);
1215 __atomic_sub_fetch(&mtr_policy->ref_cnt,
1216 1, __ATOMIC_RELAXED);
1220 if (mlx5_l3t_clear_entry(priv->mtr_idx_tbl, fm->meter_id))
1222 mlx5_flow_mtr_free(dev, mtr_idx);
1224 mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR],
1231 * Destroy meter rules.
1234 * Pointer to Ethernet device.
1235 * @param[in] meter_id
1238 * Pointer to rte meter error structure.
1241 * 0 on success, a negative errno value otherwise and rte_errno is set.
1244 mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id,
1245 struct rte_mtr_error *error)
1247 struct mlx5_priv *priv = dev->data->dev_private;
1248 struct mlx5_flow_meter_info *fm;
1249 uint32_t mtr_idx = 0;
1252 return -rte_mtr_error_set(error, ENOTSUP,
1253 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1255 "Meter is not supported");
1256 /* Meter object must exist. */
1257 fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
1259 return -rte_mtr_error_set(error, ENOENT,
1260 RTE_MTR_ERROR_TYPE_MTR_ID,
1262 "Meter object id not valid.");
1263 /* Meter object must not have any owner. */
1264 if (fm->ref_cnt > 0)
1265 return -rte_mtr_error_set(error, EBUSY,
1266 RTE_MTR_ERROR_TYPE_UNSPECIFIED,
1268 "Meter object is being used.");
1269 /* Destroy the meter profile. */
1270 if (mlx5_flow_meter_params_flush(dev, fm, mtr_idx))
1271 return -rte_mtr_error_set(error, EINVAL,
1272 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1274 "MTR object meter profile invalid.");
1279 * Modify meter state.
1282 * Pointer to mlx5 private data structure.
1284 * Pointer to flow meter.
1285 * @param[in] new_state
1286 * New state to update.
1288 * Pointer to rte meter error structure.
1291 * 0 on success, a negative errno value otherwise and rte_errno is set.
1294 mlx5_flow_meter_modify_state(struct mlx5_priv *priv,
1295 struct mlx5_flow_meter_info *fm,
1297 struct rte_mtr_error *error)
1299 static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = {
1300 .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL),
1303 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1304 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1307 if (new_state == MLX5_FLOW_METER_DISABLE)
1308 ret = mlx5_flow_meter_action_modify(priv, fm,
1309 &srtcm, modify_bits, 0, 0);
1311 ret = mlx5_flow_meter_action_modify(priv, fm,
1312 &fm->profile->srtcm_prm,
1315 return -rte_mtr_error_set(error, -ret,
1316 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1319 "Failed to enable meter." :
1320 "Failed to disable meter.");
1325 * Callback to enable flow meter.
1328 * Pointer to Ethernet device.
1329 * @param[in] meter_id
1332 * Pointer to rte meter error structure.
1335 * 0 on success, a negative errno value otherwise and rte_errno is set.
1338 mlx5_flow_meter_enable(struct rte_eth_dev *dev,
1340 struct rte_mtr_error *error)
1342 struct mlx5_priv *priv = dev->data->dev_private;
1343 struct mlx5_flow_meter_info *fm;
1347 return -rte_mtr_error_set(error, ENOTSUP,
1348 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1349 "Meter is not supported");
1350 /* Meter object must exist. */
1351 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1353 return -rte_mtr_error_set(error, ENOENT,
1354 RTE_MTR_ERROR_TYPE_MTR_ID,
1355 NULL, "Meter not found.");
1356 if (fm->active_state == MLX5_FLOW_METER_ENABLE)
1358 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_ENABLE,
1361 fm->active_state = MLX5_FLOW_METER_ENABLE;
1366 * Callback to disable flow meter.
1369 * Pointer to Ethernet device.
1370 * @param[in] meter_id
1373 * Pointer to rte meter error structure.
1376 * 0 on success, a negative errno value otherwise and rte_errno is set.
1379 mlx5_flow_meter_disable(struct rte_eth_dev *dev,
1381 struct rte_mtr_error *error)
1383 struct mlx5_priv *priv = dev->data->dev_private;
1384 struct mlx5_flow_meter_info *fm;
1388 return -rte_mtr_error_set(error, ENOTSUP,
1389 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1390 "Meter is not supported");
1391 /* Meter object must exist. */
1392 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1394 return -rte_mtr_error_set(error, ENOENT,
1395 RTE_MTR_ERROR_TYPE_MTR_ID,
1396 NULL, "Meter not found.");
1397 if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1399 ret = mlx5_flow_meter_modify_state(priv, fm, MLX5_FLOW_METER_DISABLE,
1402 fm->active_state = MLX5_FLOW_METER_DISABLE;
1407 * Callback to update meter profile.
1410 * Pointer to Ethernet device.
1411 * @param[in] meter_id
1413 * @param[in] meter_profile_id
1414 * To be updated meter profile id.
1416 * Pointer to rte meter error structure.
1419 * 0 on success, a negative errno value otherwise and rte_errno is set.
1422 mlx5_flow_meter_profile_update(struct rte_eth_dev *dev,
1424 uint32_t meter_profile_id,
1425 struct rte_mtr_error *error)
1427 struct mlx5_priv *priv = dev->data->dev_private;
1428 struct mlx5_flow_meter_profile *fmp;
1429 struct mlx5_flow_meter_profile *old_fmp;
1430 struct mlx5_flow_meter_info *fm;
1431 uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS |
1432 MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR;
1436 return -rte_mtr_error_set(error, ENOTSUP,
1437 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1438 "Meter is not supported");
1439 /* Meter profile must exist. */
1440 fmp = mlx5_flow_meter_profile_find(priv, meter_profile_id);
1442 return -rte_mtr_error_set(error, ENOENT,
1443 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1444 NULL, "Meter profile not found.");
1445 /* Meter object must exist. */
1446 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1448 return -rte_mtr_error_set(error, ENOENT,
1449 RTE_MTR_ERROR_TYPE_MTR_ID,
1450 NULL, "Meter not found.");
1451 /* MTR object already set to meter profile id. */
1452 old_fmp = fm->profile;
1455 /* Update the profile. */
1457 /* Update meter params in HW (if not disabled). */
1458 if (fm->active_state == MLX5_FLOW_METER_DISABLE)
1460 ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm,
1461 modify_bits, fm->active_state, 1);
1463 fm->profile = old_fmp;
1464 return -rte_mtr_error_set(error, -ret,
1465 RTE_MTR_ERROR_TYPE_MTR_PARAMS,
1466 NULL, "Failed to update meter"
1467 " parmeters in hardware.");
1475 * Callback to update meter stats mask.
1478 * Pointer to Ethernet device.
1479 * @param[in] meter_id
1481 * @param[in] stats_mask
1482 * To be updated stats_mask.
1484 * Pointer to rte meter error structure.
1487 * 0 on success, a negative errno value otherwise and rte_errno is set.
1490 mlx5_flow_meter_stats_update(struct rte_eth_dev *dev,
1492 uint64_t stats_mask,
1493 struct rte_mtr_error *error)
1495 struct mlx5_priv *priv = dev->data->dev_private;
1496 struct mlx5_flow_meter_info *fm;
1499 return -rte_mtr_error_set(error, ENOTSUP,
1500 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1501 "Meter is not supported");
1502 /* Meter object must exist. */
1503 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1505 return -rte_mtr_error_set(error, ENOENT,
1506 RTE_MTR_ERROR_TYPE_MTR_ID,
1507 NULL, "Meter object id not valid.");
1508 if (mlx5_flow_meter_stats_enable_update(dev, fm, stats_mask))
1509 return -rte_mtr_error_set(error, ENOENT,
1510 RTE_MTR_ERROR_TYPE_MTR_ID,
1511 NULL, "Fail to allocate "
1512 "counter for meter.");
1517 * Callback to read meter statistics.
1520 * Pointer to Ethernet device.
1521 * @param[in] meter_id
1524 * Pointer to store the statistics.
1525 * @param[out] stats_mask
1526 * Pointer to store the stats_mask.
1528 * Statistic to be cleared after read or not.
1530 * Pointer to rte meter error structure.
1533 * 0 on success, a negative errno value otherwise and rte_errno is set.
1536 mlx5_flow_meter_stats_read(struct rte_eth_dev *dev,
1538 struct rte_mtr_stats *stats,
1539 uint64_t *stats_mask,
1541 struct rte_mtr_error *error)
1543 struct mlx5_priv *priv = dev->data->dev_private;
1544 struct mlx5_flow_meter_info *fm;
1550 return -rte_mtr_error_set(error, ENOTSUP,
1551 RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
1552 "Meter is not supported");
1553 /* Meter object must exist. */
1554 fm = mlx5_flow_meter_find(priv, meter_id, NULL);
1556 return -rte_mtr_error_set(error, ENOENT,
1557 RTE_MTR_ERROR_TYPE_MTR_ID,
1558 NULL, "Meter object id not valid.");
1560 if (fm->bytes_dropped)
1561 *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED;
1562 if (fm->pkts_dropped)
1563 *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED;
1564 memset(stats, 0, sizeof(*stats));
1566 ret = mlx5_counter_query(dev, fm->drop_cnt, clear, &pkts,
1570 /* If need to read the packets, set it. */
1571 if (fm->pkts_dropped)
1572 stats->n_pkts_dropped = pkts;
1573 /* If need to read the bytes, set it. */
1574 if (fm->bytes_dropped)
1575 stats->n_bytes_dropped = bytes;
1579 return -rte_mtr_error_set(error, ret, RTE_MTR_ERROR_TYPE_STATS, NULL,
1580 "Failed to read meter drop counters.");
1583 static const struct rte_mtr_ops mlx5_flow_mtr_ops = {
1584 .capabilities_get = mlx5_flow_mtr_cap_get,
1585 .meter_profile_add = mlx5_flow_meter_profile_add,
1586 .meter_profile_delete = mlx5_flow_meter_profile_delete,
1587 .meter_policy_validate = mlx5_flow_meter_policy_validate,
1588 .meter_policy_add = mlx5_flow_meter_policy_add,
1589 .meter_policy_delete = mlx5_flow_meter_policy_delete,
1590 .create = mlx5_flow_meter_create,
1591 .destroy = mlx5_flow_meter_destroy,
1592 .meter_enable = mlx5_flow_meter_enable,
1593 .meter_disable = mlx5_flow_meter_disable,
1594 .meter_profile_update = mlx5_flow_meter_profile_update,
1595 .meter_dscp_table_update = NULL,
1596 .stats_update = mlx5_flow_meter_stats_update,
1597 .stats_read = mlx5_flow_meter_stats_read,
1601 * Get meter operations.
1604 * Pointer to Ethernet device structure.
1606 * Pointer to set the mtr operations.
1612 mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg)
1614 *(const struct rte_mtr_ops **)arg = &mlx5_flow_mtr_ops;
1622 * Pointer to mlx5_priv.
1626 * Pointer to Meter index.
1629 * Pointer to the meter info found on success, NULL otherwise.
1631 struct mlx5_flow_meter_info *
1632 mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
1635 struct mlx5_legacy_flow_meter *legacy_fm;
1636 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1637 struct mlx5_aso_mtr *aso_mtr;
1638 struct mlx5_aso_mtr_pools_mng *pools_mng =
1639 &priv->sh->mtrmng->pools_mng;
1640 union mlx5_l3t_data data;
1642 if (priv->sh->meter_aso_en) {
1643 rte_spinlock_lock(&pools_mng->mtrsl);
1644 if (!pools_mng->n_valid || !priv->mtr_idx_tbl) {
1645 rte_spinlock_unlock(&pools_mng->mtrsl);
1648 if (mlx5_l3t_get_entry(priv->mtr_idx_tbl, meter_id, &data) ||
1650 rte_spinlock_unlock(&pools_mng->mtrsl);
1654 *mtr_idx = data.dword;
1655 aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword);
1656 /* Remove reference taken by the mlx5_l3t_get_entry. */
1657 mlx5_l3t_clear_entry(priv->mtr_idx_tbl, meter_id);
1658 rte_spinlock_unlock(&pools_mng->mtrsl);
1659 if (!aso_mtr || aso_mtr->state == ASO_METER_FREE)
1661 return &aso_mtr->fm;
1663 TAILQ_FOREACH(legacy_fm, fms, next)
1664 if (meter_id == legacy_fm->fm.meter_id) {
1666 *mtr_idx = legacy_fm->idx;
1667 return &legacy_fm->fm;
1673 * Find meter by index.
1676 * Pointer to mlx5_priv.
1681 * Pointer to the meter info found on success, NULL otherwise.
1683 struct mlx5_flow_meter_info *
1684 flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx)
1686 struct mlx5_aso_mtr *aso_mtr;
1688 if (priv->sh->meter_aso_en) {
1689 aso_mtr = mlx5_aso_meter_by_idx(priv, idx);
1692 return &aso_mtr->fm;
1694 return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx);
1699 * Attach meter to flow.
1700 * Unidirectional Meter creation can only be done
1701 * when flow direction is known, i.e. when calling meter_attach.
1704 * Pointer to mlx5 private data.
1706 * Pointer to flow meter.
1708 * Pointer to flow attributes.
1709 * @param [out] error
1710 * Pointer to error structure.
1713 * 0 on success, a negative errno value otherwise and rte_errno is set.
1716 mlx5_flow_meter_attach(struct mlx5_priv *priv,
1717 struct mlx5_flow_meter_info *fm,
1718 const struct rte_flow_attr *attr,
1719 struct rte_flow_error *error)
1723 if (priv->sh->meter_aso_en) {
1724 struct mlx5_aso_mtr *aso_mtr;
1726 aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
1727 if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
1728 return rte_flow_error_set(error, ENOENT,
1729 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1731 "Timeout in meter configuration");
1733 rte_spinlock_lock(&fm->sl);
1734 if (fm->shared || !fm->ref_cnt) {
1737 rte_flow_error_set(error, EINVAL,
1738 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1739 "Meter cannot be shared");
1742 rte_spinlock_unlock(&fm->sl);
1744 rte_spinlock_lock(&fm->sl);
1745 if (fm->meter_action) {
1747 attr->transfer == fm->transfer &&
1748 attr->ingress == fm->ingress &&
1749 attr->egress == fm->egress) {
1752 rte_flow_error_set(error, EINVAL,
1753 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1755 "Meter attr not match." :
1756 "Meter cannot be shared.");
1760 fm->ingress = attr->ingress;
1761 fm->egress = attr->egress;
1762 fm->transfer = attr->transfer;
1764 /* This also creates the meter object. */
1765 fm->meter_action = mlx5_flow_meter_action_create(priv,
1767 if (!fm->meter_action) {
1772 rte_flow_error_set(error, EINVAL,
1773 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1774 "Meter action create failed.");
1778 rte_spinlock_unlock(&fm->sl);
1780 return ret ? -rte_errno : 0;
1784 * Detach meter from flow.
1787 * Pointer to mlx5 private data.
1789 * Pointer to flow meter.
1792 mlx5_flow_meter_detach(struct mlx5_priv *priv,
1793 struct mlx5_flow_meter_info *fm)
1795 #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
1796 rte_spinlock_lock(&fm->sl);
1797 MLX5_ASSERT(fm->ref_cnt);
1798 if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
1799 mlx5_glue->destroy_flow_action(fm->meter_action);
1800 fm->meter_action = NULL;
1805 rte_spinlock_unlock(&fm->sl);
1813 * Flush meter with Rx queue configuration.
1816 * Pointer to Ethernet device.
1819 mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev)
1821 struct mlx5_priv *priv = dev->data->dev_private;
1822 struct mlx5_flow_meter_sub_policy *sub_policy;
1823 struct mlx5_flow_meter_policy *mtr_policy;
1825 uint32_t i, policy_idx;
1829 if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1830 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1832 policy_idx = *(uint32_t *)entry;
1833 sub_policy = mlx5_ipool_get
1834 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1836 if (!sub_policy || !sub_policy->main_policy)
1838 mtr_policy = sub_policy->main_policy;
1839 if (mtr_policy->is_queue || mtr_policy->is_rss)
1840 mlx5_flow_destroy_sub_policy_with_rxq(dev,
1847 * Flush meter configuration.
1850 * Pointer to Ethernet device.
1852 * Pointer to rte meter error structure.
1855 * 0 on success, a negative errno value otherwise and rte_errno is set.
1858 mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error)
1860 struct mlx5_priv *priv = dev->data->dev_private;
1861 struct mlx5_legacy_flow_meters *fms = &priv->flow_meters;
1862 struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles;
1863 struct mlx5_flow_meter_profile *fmp;
1864 struct mlx5_legacy_flow_meter *legacy_fm;
1865 struct mlx5_flow_meter_info *fm;
1866 struct mlx5_flow_meter_sub_policy *sub_policy;
1868 uint32_t i, mtr_idx, policy_idx;
1870 struct mlx5_aso_mtr *aso_mtr;
1874 if (priv->sh->meter_aso_en) {
1875 if (priv->mtr_idx_tbl) {
1876 MLX5_L3T_FOREACH(priv->mtr_idx_tbl, i, entry) {
1877 mtr_idx = *(uint32_t *)entry;
1880 mlx5_aso_meter_by_idx(priv, mtr_idx);
1882 (void)mlx5_flow_meter_params_flush(dev,
1886 mlx5_l3t_destroy(priv->mtr_idx_tbl);
1887 priv->mtr_idx_tbl = NULL;
1890 TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) {
1891 fm = &legacy_fm->fm;
1892 if (mlx5_flow_meter_params_flush(dev, fm, 0))
1893 return -rte_mtr_error_set(error, EINVAL,
1894 RTE_MTR_ERROR_TYPE_METER_PROFILE_ID,
1895 NULL, "MTR object meter profile invalid.");
1898 if (priv->sh->mtrmng->policy_idx_tbl && priv->sh->refcnt == 1) {
1899 MLX5_L3T_FOREACH(priv->sh->mtrmng->policy_idx_tbl,
1901 policy_idx = *(uint32_t *)entry;
1902 sub_policy = mlx5_ipool_get
1903 (priv->sh->ipool[MLX5_IPOOL_MTR_POLICY],
1906 return -rte_mtr_error_set(error,
1908 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1910 "meter policy invalid.");
1911 if (__mlx5_flow_meter_policy_delete(dev, i,
1912 sub_policy->main_policy,
1914 return -rte_mtr_error_set(error,
1916 RTE_MTR_ERROR_TYPE_METER_POLICY_ID,
1918 "meter policy invalid.");
1919 mlx5_free(sub_policy->main_policy);
1921 mlx5_l3t_destroy(priv->sh->mtrmng->policy_idx_tbl);
1922 priv->sh->mtrmng->policy_idx_tbl = NULL;
1924 TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) {
1926 MLX5_ASSERT(!fmp->ref_cnt);
1927 /* Remove from list. */
1928 TAILQ_REMOVE(&priv->flow_meter_profiles, fmp, next);
1931 /* Delete default policy table. */
1932 mlx5_flow_destroy_def_policy(dev);
1933 if (priv->sh->refcnt == 1)
1934 mlx5_flow_destroy_mtr_drop_tbls(dev);