1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
5 #include <rte_malloc.h>
7 #include "otx2_ethdev.h"
10 /* Use last LVL_CNT nodes as default nodes */
11 #define NIX_DEFAULT_NODE_ID_START (RTE_TM_NODE_ID_NULL - NIX_TXSCH_LVL_CNT)
13 enum otx2_tm_node_level {
24 uint64_t shaper2regval(struct shaper_params *shaper)
26 return (shaper->burst_exponent << 37) | (shaper->burst_mantissa << 29) |
27 (shaper->div_exp << 13) | (shaper->exponent << 9) |
28 (shaper->mantissa << 1);
32 otx2_nix_get_link(struct otx2_eth_dev *dev)
34 int link = 13 /* SDP */;
38 lmac_chan = dev->tx_chan_base;
41 if (lmac_chan >= 0x800) {
42 map = lmac_chan & 0x7FF;
43 link = 4 * ((map >> 8) & 0xF) + ((map >> 4) & 0xF);
44 } else if (lmac_chan < 0x700) {
53 nix_get_relchan(struct otx2_eth_dev *dev)
55 return dev->tx_chan_base & 0xff;
59 nix_tm_have_tl1_access(struct otx2_eth_dev *dev)
61 bool is_lbk = otx2_dev_is_lbk(dev);
62 return otx2_dev_is_pf(dev) && !otx2_dev_is_Ax(dev) && !is_lbk;
66 nix_tm_is_leaf(struct otx2_eth_dev *dev, int lvl)
68 if (nix_tm_have_tl1_access(dev))
69 return (lvl == OTX2_TM_LVL_QUEUE);
71 return (lvl == OTX2_TM_LVL_SCH4);
75 find_prio_anchor(struct otx2_eth_dev *dev, uint32_t node_id)
77 struct otx2_nix_tm_node *child_node;
79 TAILQ_FOREACH(child_node, &dev->node_list, node) {
80 if (!child_node->parent)
82 if (!(child_node->parent->id == node_id))
84 if (child_node->priority == child_node->parent->rr_prio)
86 return child_node->hw_id - child_node->priority;
92 static struct otx2_nix_tm_shaper_profile *
93 nix_tm_shaper_profile_search(struct otx2_eth_dev *dev, uint32_t shaper_id)
95 struct otx2_nix_tm_shaper_profile *tm_shaper_profile;
97 TAILQ_FOREACH(tm_shaper_profile, &dev->shaper_profile_list, shaper) {
98 if (tm_shaper_profile->shaper_profile_id == shaper_id)
99 return tm_shaper_profile;
104 static inline uint64_t
105 shaper_rate_to_nix(uint64_t value, uint64_t *exponent_p,
106 uint64_t *mantissa_p, uint64_t *div_exp_p)
108 uint64_t div_exp, exponent, mantissa;
110 /* Boundary checks */
111 if (value < MIN_SHAPER_RATE ||
112 value > MAX_SHAPER_RATE)
115 if (value <= SHAPER_RATE(0, 0, 0)) {
116 /* Calculate rate div_exp and mantissa using
117 * the following formula:
119 * value = (2E6 * (256 + mantissa)
120 * / ((1 << div_exp) * 256))
124 mantissa = MAX_RATE_MANTISSA;
126 while (value < (NIX_SHAPER_RATE_CONST / (1 << div_exp)))
130 ((NIX_SHAPER_RATE_CONST * (256 + mantissa)) /
131 ((1 << div_exp) * 256)))
134 /* Calculate rate exponent and mantissa using
135 * the following formula:
137 * value = (2E6 * ((256 + mantissa) << exponent)) / 256
141 exponent = MAX_RATE_EXPONENT;
142 mantissa = MAX_RATE_MANTISSA;
144 while (value < (NIX_SHAPER_RATE_CONST * (1 << exponent)))
147 while (value < ((NIX_SHAPER_RATE_CONST *
148 ((256 + mantissa) << exponent)) / 256))
152 if (div_exp > MAX_RATE_DIV_EXP ||
153 exponent > MAX_RATE_EXPONENT || mantissa > MAX_RATE_MANTISSA)
157 *div_exp_p = div_exp;
159 *exponent_p = exponent;
161 *mantissa_p = mantissa;
163 /* Calculate real rate value */
164 return SHAPER_RATE(exponent, mantissa, div_exp);
167 static inline uint64_t
168 shaper_burst_to_nix(uint64_t value, uint64_t *exponent_p,
169 uint64_t *mantissa_p)
171 uint64_t exponent, mantissa;
173 if (value < MIN_SHAPER_BURST || value > MAX_SHAPER_BURST)
176 /* Calculate burst exponent and mantissa using
177 * the following formula:
179 * value = (((256 + mantissa) << (exponent + 1)
183 exponent = MAX_BURST_EXPONENT;
184 mantissa = MAX_BURST_MANTISSA;
186 while (value < (1ull << (exponent + 1)))
189 while (value < ((256 + mantissa) << (exponent + 1)) / 256)
192 if (exponent > MAX_BURST_EXPONENT || mantissa > MAX_BURST_MANTISSA)
196 *exponent_p = exponent;
198 *mantissa_p = mantissa;
200 return SHAPER_BURST(exponent, mantissa);
204 shaper_config_to_nix(struct otx2_nix_tm_shaper_profile *profile,
205 struct shaper_params *cir,
206 struct shaper_params *pir)
208 struct rte_tm_shaper_params *param = &profile->params;
213 /* Calculate CIR exponent and mantissa */
214 if (param->committed.rate)
215 cir->rate = shaper_rate_to_nix(param->committed.rate,
220 /* Calculate PIR exponent and mantissa */
221 if (param->peak.rate)
222 pir->rate = shaper_rate_to_nix(param->peak.rate,
227 /* Calculate CIR burst exponent and mantissa */
228 if (param->committed.size)
229 cir->burst = shaper_burst_to_nix(param->committed.size,
230 &cir->burst_exponent,
231 &cir->burst_mantissa);
233 /* Calculate PIR burst exponent and mantissa */
234 if (param->peak.size)
235 pir->burst = shaper_burst_to_nix(param->peak.size,
236 &pir->burst_exponent,
237 &pir->burst_mantissa);
241 shaper_default_red_algo(struct otx2_eth_dev *dev,
242 struct otx2_nix_tm_node *tm_node,
243 struct otx2_nix_tm_shaper_profile *profile)
245 struct shaper_params cir, pir;
247 /* C0 doesn't support STALL when both PIR & CIR are enabled */
248 if (profile && otx2_dev_is_96xx_Cx(dev)) {
249 memset(&cir, 0, sizeof(cir));
250 memset(&pir, 0, sizeof(pir));
251 shaper_config_to_nix(profile, &cir, &pir);
253 if (pir.rate && cir.rate) {
254 tm_node->red_algo = NIX_REDALG_DISCARD;
255 tm_node->flags |= NIX_TM_NODE_RED_DISCARD;
260 tm_node->red_algo = NIX_REDALG_STD;
261 tm_node->flags &= ~NIX_TM_NODE_RED_DISCARD;
265 populate_tm_tl1_default(struct otx2_eth_dev *dev, uint32_t schq)
267 struct otx2_mbox *mbox = dev->mbox;
268 struct nix_txschq_config *req;
271 * Default config for TL1.
272 * For VF this is always ignored.
275 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
276 req->lvl = NIX_TXSCH_LVL_TL1;
278 /* Set DWRR quantum */
279 req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq);
280 req->regval[0] = TXSCH_TL1_DFLT_RR_QTM;
283 req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq);
284 req->regval[1] = (TXSCH_TL1_DFLT_RR_PRIO << 1);
287 req->reg[2] = NIX_AF_TL1X_CIR(schq);
291 return otx2_mbox_process(mbox);
295 prepare_tm_sched_reg(struct otx2_eth_dev *dev,
296 struct otx2_nix_tm_node *tm_node,
297 volatile uint64_t *reg, volatile uint64_t *regval)
299 uint64_t strict_prio = tm_node->priority;
300 uint32_t hw_lvl = tm_node->hw_lvl;
301 uint32_t schq = tm_node->hw_id;
305 rr_quantum = NIX_TM_WEIGHT_TO_RR_QUANTUM(tm_node->weight);
307 /* For children to root, strict prio is default if either
308 * device root is TL2 or TL1 Static Priority is disabled.
310 if (hw_lvl == NIX_TXSCH_LVL_TL2 &&
311 (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 ||
312 dev->tm_flags & NIX_TM_TL1_NO_SP))
313 strict_prio = TXSCH_TL1_DFLT_RR_PRIO;
315 otx2_tm_dbg("Schedule config node %s(%u) lvl %u id %u, "
316 "prio 0x%" PRIx64 ", rr_quantum 0x%" PRIx64 " (%p)",
317 nix_hwlvl2str(tm_node->hw_lvl), schq, tm_node->lvl,
318 tm_node->id, strict_prio, rr_quantum, tm_node);
321 case NIX_TXSCH_LVL_SMQ:
322 reg[k] = NIX_AF_MDQX_SCHEDULE(schq);
323 regval[k] = (strict_prio << 24) | rr_quantum;
327 case NIX_TXSCH_LVL_TL4:
328 reg[k] = NIX_AF_TL4X_SCHEDULE(schq);
329 regval[k] = (strict_prio << 24) | rr_quantum;
333 case NIX_TXSCH_LVL_TL3:
334 reg[k] = NIX_AF_TL3X_SCHEDULE(schq);
335 regval[k] = (strict_prio << 24) | rr_quantum;
339 case NIX_TXSCH_LVL_TL2:
340 reg[k] = NIX_AF_TL2X_SCHEDULE(schq);
341 regval[k] = (strict_prio << 24) | rr_quantum;
345 case NIX_TXSCH_LVL_TL1:
346 reg[k] = NIX_AF_TL1X_SCHEDULE(schq);
347 regval[k] = rr_quantum;
357 prepare_tm_shaper_reg(struct otx2_nix_tm_node *tm_node,
358 struct otx2_nix_tm_shaper_profile *profile,
359 volatile uint64_t *reg, volatile uint64_t *regval)
361 struct shaper_params cir, pir;
362 uint32_t schq = tm_node->hw_id;
366 memset(&cir, 0, sizeof(cir));
367 memset(&pir, 0, sizeof(pir));
368 shaper_config_to_nix(profile, &cir, &pir);
370 /* Packet length adjust */
371 if (tm_node->pkt_mode)
374 adjust = profile->params.pkt_length_adjust & 0x1FF;
376 otx2_tm_dbg("Shaper config node %s(%u) lvl %u id %u, pir %" PRIu64
377 "(%" PRIu64 "B), cir %" PRIu64 "(%" PRIu64 "B)"
378 "adjust 0x%" PRIx64 "(pktmode %u) (%p)",
379 nix_hwlvl2str(tm_node->hw_lvl), schq, tm_node->lvl,
380 tm_node->id, pir.rate, pir.burst, cir.rate, cir.burst,
381 adjust, tm_node->pkt_mode, tm_node);
383 switch (tm_node->hw_lvl) {
384 case NIX_TXSCH_LVL_SMQ:
385 /* Configure PIR, CIR */
386 reg[k] = NIX_AF_MDQX_PIR(schq);
387 regval[k] = (pir.rate && pir.burst) ?
388 (shaper2regval(&pir) | 1) : 0;
391 reg[k] = NIX_AF_MDQX_CIR(schq);
392 regval[k] = (cir.rate && cir.burst) ?
393 (shaper2regval(&cir) | 1) : 0;
396 /* Configure RED ALG */
397 reg[k] = NIX_AF_MDQX_SHAPE(schq);
398 regval[k] = (adjust |
399 (uint64_t)tm_node->red_algo << 9 |
400 (uint64_t)tm_node->pkt_mode << 24);
403 case NIX_TXSCH_LVL_TL4:
404 /* Configure PIR, CIR */
405 reg[k] = NIX_AF_TL4X_PIR(schq);
406 regval[k] = (pir.rate && pir.burst) ?
407 (shaper2regval(&pir) | 1) : 0;
410 reg[k] = NIX_AF_TL4X_CIR(schq);
411 regval[k] = (cir.rate && cir.burst) ?
412 (shaper2regval(&cir) | 1) : 0;
415 /* Configure RED algo */
416 reg[k] = NIX_AF_TL4X_SHAPE(schq);
417 regval[k] = (adjust |
418 (uint64_t)tm_node->red_algo << 9 |
419 (uint64_t)tm_node->pkt_mode << 24);
422 case NIX_TXSCH_LVL_TL3:
423 /* Configure PIR, CIR */
424 reg[k] = NIX_AF_TL3X_PIR(schq);
425 regval[k] = (pir.rate && pir.burst) ?
426 (shaper2regval(&pir) | 1) : 0;
429 reg[k] = NIX_AF_TL3X_CIR(schq);
430 regval[k] = (cir.rate && cir.burst) ?
431 (shaper2regval(&cir) | 1) : 0;
434 /* Configure RED algo */
435 reg[k] = NIX_AF_TL3X_SHAPE(schq);
436 regval[k] = (adjust |
437 (uint64_t)tm_node->red_algo << 9 |
438 (uint64_t)tm_node->pkt_mode << 24);
442 case NIX_TXSCH_LVL_TL2:
443 /* Configure PIR, CIR */
444 reg[k] = NIX_AF_TL2X_PIR(schq);
445 regval[k] = (pir.rate && pir.burst) ?
446 (shaper2regval(&pir) | 1) : 0;
449 reg[k] = NIX_AF_TL2X_CIR(schq);
450 regval[k] = (cir.rate && cir.burst) ?
451 (shaper2regval(&cir) | 1) : 0;
454 /* Configure RED algo */
455 reg[k] = NIX_AF_TL2X_SHAPE(schq);
456 regval[k] = (adjust |
457 (uint64_t)tm_node->red_algo << 9 |
458 (uint64_t)tm_node->pkt_mode << 24);
462 case NIX_TXSCH_LVL_TL1:
464 reg[k] = NIX_AF_TL1X_CIR(schq);
465 regval[k] = (cir.rate && cir.burst) ?
466 (shaper2regval(&cir) | 1) : 0;
469 /* Configure length disable and adjust */
470 reg[k] = NIX_AF_TL1X_SHAPE(schq);
471 regval[k] = (adjust |
472 (uint64_t)tm_node->pkt_mode << 24);
481 prepare_tm_sw_xoff(struct otx2_nix_tm_node *tm_node, bool enable,
482 volatile uint64_t *reg, volatile uint64_t *regval)
484 uint32_t hw_lvl = tm_node->hw_lvl;
485 uint32_t schq = tm_node->hw_id;
488 otx2_tm_dbg("sw xoff config node %s(%u) lvl %u id %u, enable %u (%p)",
489 nix_hwlvl2str(hw_lvl), schq, tm_node->lvl,
490 tm_node->id, enable, tm_node);
495 case NIX_TXSCH_LVL_MDQ:
496 reg[k] = NIX_AF_MDQX_SW_XOFF(schq);
499 case NIX_TXSCH_LVL_TL4:
500 reg[k] = NIX_AF_TL4X_SW_XOFF(schq);
503 case NIX_TXSCH_LVL_TL3:
504 reg[k] = NIX_AF_TL3X_SW_XOFF(schq);
507 case NIX_TXSCH_LVL_TL2:
508 reg[k] = NIX_AF_TL2X_SW_XOFF(schq);
511 case NIX_TXSCH_LVL_TL1:
512 reg[k] = NIX_AF_TL1X_SW_XOFF(schq);
523 populate_tm_reg(struct otx2_eth_dev *dev,
524 struct otx2_nix_tm_node *tm_node)
526 struct otx2_nix_tm_shaper_profile *profile;
527 uint64_t regval_mask[MAX_REGS_PER_MBOX_MSG];
528 uint64_t regval[MAX_REGS_PER_MBOX_MSG];
529 uint64_t reg[MAX_REGS_PER_MBOX_MSG];
530 struct otx2_mbox *mbox = dev->mbox;
531 uint64_t parent = 0, child = 0;
532 uint32_t hw_lvl, rr_prio, schq;
533 struct nix_txschq_config *req;
537 memset(regval_mask, 0, sizeof(regval_mask));
538 profile = nix_tm_shaper_profile_search(dev,
539 tm_node->params.shaper_profile_id);
540 rr_prio = tm_node->rr_prio;
541 hw_lvl = tm_node->hw_lvl;
542 schq = tm_node->hw_id;
544 /* Root node will not have a parent node */
545 if (hw_lvl == dev->otx2_tm_root_lvl)
546 parent = tm_node->parent_hw_id;
548 parent = tm_node->parent->hw_id;
550 /* Do we need this trigger to configure TL1 */
551 if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
552 hw_lvl == dev->otx2_tm_root_lvl) {
553 rc = populate_tm_tl1_default(dev, parent);
558 if (hw_lvl != NIX_TXSCH_LVL_SMQ)
559 child = find_prio_anchor(dev, tm_node->id);
561 /* Override default rr_prio when TL1
562 * Static Priority is disabled
564 if (hw_lvl == NIX_TXSCH_LVL_TL1 &&
565 dev->tm_flags & NIX_TM_TL1_NO_SP) {
566 rr_prio = TXSCH_TL1_DFLT_RR_PRIO;
570 otx2_tm_dbg("Topology config node %s(%u)->%s(%"PRIu64") lvl %u, id %u"
571 " prio_anchor %"PRIu64" rr_prio %u (%p)",
572 nix_hwlvl2str(hw_lvl), schq, nix_hwlvl2str(hw_lvl + 1),
573 parent, tm_node->lvl, tm_node->id, child, rr_prio, tm_node);
575 /* Prepare Topology and Link config */
577 case NIX_TXSCH_LVL_SMQ:
579 /* Set xoff which will be cleared later and minimum length
580 * which will be used for zero padding if packet length is
583 reg[k] = NIX_AF_SMQX_CFG(schq);
584 regval[k] = BIT_ULL(50) | NIX_MIN_HW_FRS;
585 regval_mask[k] = ~(BIT_ULL(50) | 0x7f);
588 /* Parent and schedule conf */
589 reg[k] = NIX_AF_MDQX_PARENT(schq);
590 regval[k] = parent << 16;
594 case NIX_TXSCH_LVL_TL4:
595 /* Parent and schedule conf */
596 reg[k] = NIX_AF_TL4X_PARENT(schq);
597 regval[k] = parent << 16;
600 reg[k] = NIX_AF_TL4X_TOPOLOGY(schq);
601 regval[k] = (child << 32) | (rr_prio << 1);
604 /* Configure TL4 to send to SDP channel instead of CGX/LBK */
605 if (otx2_dev_is_sdp(dev)) {
606 reg[k] = NIX_AF_TL4X_SDP_LINK_CFG(schq);
607 regval[k] = BIT_ULL(12);
611 case NIX_TXSCH_LVL_TL3:
612 /* Parent and schedule conf */
613 reg[k] = NIX_AF_TL3X_PARENT(schq);
614 regval[k] = parent << 16;
617 reg[k] = NIX_AF_TL3X_TOPOLOGY(schq);
618 regval[k] = (child << 32) | (rr_prio << 1);
621 /* Link configuration */
622 if (!otx2_dev_is_sdp(dev) &&
623 dev->link_cfg_lvl == NIX_TXSCH_LVL_TL3) {
624 reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(schq,
625 otx2_nix_get_link(dev));
626 regval[k] = BIT_ULL(12) | nix_get_relchan(dev);
631 case NIX_TXSCH_LVL_TL2:
632 /* Parent and schedule conf */
633 reg[k] = NIX_AF_TL2X_PARENT(schq);
634 regval[k] = parent << 16;
637 reg[k] = NIX_AF_TL2X_TOPOLOGY(schq);
638 regval[k] = (child << 32) | (rr_prio << 1);
641 /* Link configuration */
642 if (!otx2_dev_is_sdp(dev) &&
643 dev->link_cfg_lvl == NIX_TXSCH_LVL_TL2) {
644 reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(schq,
645 otx2_nix_get_link(dev));
646 regval[k] = BIT_ULL(12) | nix_get_relchan(dev);
651 case NIX_TXSCH_LVL_TL1:
652 reg[k] = NIX_AF_TL1X_TOPOLOGY(schq);
653 regval[k] = (child << 32) | (rr_prio << 1 /*RR_PRIO*/);
659 /* Prepare schedule config */
660 k += prepare_tm_sched_reg(dev, tm_node, ®[k], ®val[k]);
662 /* Prepare shaping config */
663 k += prepare_tm_shaper_reg(tm_node, profile, ®[k], ®val[k]);
668 /* Copy and send config mbox */
669 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
673 otx2_mbox_memcpy(req->reg, reg, sizeof(uint64_t) * k);
674 otx2_mbox_memcpy(req->regval, regval, sizeof(uint64_t) * k);
675 otx2_mbox_memcpy(req->regval_mask, regval_mask, sizeof(uint64_t) * k);
677 rc = otx2_mbox_process(mbox);
683 otx2_err("Txschq cfg request failed for node %p, rc=%d", tm_node, rc);
689 nix_tm_txsch_reg_config(struct otx2_eth_dev *dev)
691 struct otx2_nix_tm_node *tm_node;
695 for (hw_lvl = 0; hw_lvl <= dev->otx2_tm_root_lvl; hw_lvl++) {
696 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
697 if (tm_node->hw_lvl == hw_lvl &&
698 tm_node->hw_lvl != NIX_TXSCH_LVL_CNT) {
699 rc = populate_tm_reg(dev, tm_node);
709 static struct otx2_nix_tm_node *
710 nix_tm_node_search(struct otx2_eth_dev *dev,
711 uint32_t node_id, bool user)
713 struct otx2_nix_tm_node *tm_node;
715 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
716 if (tm_node->id == node_id &&
717 (user == !!(tm_node->flags & NIX_TM_NODE_USER)))
724 check_rr(struct otx2_eth_dev *dev, uint32_t priority, uint32_t parent_id)
726 struct otx2_nix_tm_node *tm_node;
729 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
730 if (!tm_node->parent)
733 if (!(tm_node->parent->id == parent_id))
736 if (tm_node->priority == priority)
743 nix_tm_update_parent_info(struct otx2_eth_dev *dev)
745 struct otx2_nix_tm_node *tm_node_child;
746 struct otx2_nix_tm_node *tm_node;
747 struct otx2_nix_tm_node *parent;
751 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
752 if (!tm_node->parent)
754 /* Count group of children of same priority i.e are RR */
755 parent = tm_node->parent;
756 priority = tm_node->priority;
757 rr_num = check_rr(dev, priority, parent->id);
759 /* Assuming that multiple RR groups are
760 * not configured based on capability.
763 parent->rr_prio = priority;
764 parent->rr_num = rr_num;
767 /* Find out static priority children that are not in RR */
768 TAILQ_FOREACH(tm_node_child, &dev->node_list, node) {
769 if (!tm_node_child->parent)
771 if (parent->id != tm_node_child->parent->id)
773 if (parent->max_prio == UINT32_MAX &&
774 tm_node_child->priority != parent->rr_prio)
775 parent->max_prio = 0;
777 if (parent->max_prio < tm_node_child->priority &&
778 parent->rr_prio != tm_node_child->priority)
779 parent->max_prio = tm_node_child->priority;
787 nix_tm_node_add_to_list(struct otx2_eth_dev *dev, uint32_t node_id,
788 uint32_t parent_node_id, uint32_t priority,
789 uint32_t weight, uint16_t hw_lvl,
790 uint16_t lvl, bool user,
791 struct rte_tm_node_params *params)
793 struct otx2_nix_tm_shaper_profile *profile;
794 struct otx2_nix_tm_node *tm_node, *parent_node;
797 profile_id = params->shaper_profile_id;
798 profile = nix_tm_shaper_profile_search(dev, profile_id);
800 parent_node = nix_tm_node_search(dev, parent_node_id, user);
802 tm_node = rte_zmalloc("otx2_nix_tm_node",
803 sizeof(struct otx2_nix_tm_node), 0);
808 tm_node->hw_lvl = hw_lvl;
810 /* Maintain minimum weight */
814 tm_node->id = node_id;
815 tm_node->priority = priority;
816 tm_node->weight = weight;
817 tm_node->rr_prio = 0xf;
818 tm_node->max_prio = UINT32_MAX;
819 tm_node->hw_id = UINT32_MAX;
822 tm_node->flags = NIX_TM_NODE_USER;
825 if (!nix_tm_is_leaf(dev, lvl) &&
826 ((profile && profile->params.packet_mode) ||
827 (params->nonleaf.wfq_weight_mode &&
828 params->nonleaf.n_sp_priorities &&
829 !params->nonleaf.wfq_weight_mode[0])))
830 tm_node->pkt_mode = 1;
832 rte_memcpy(&tm_node->params, params, sizeof(struct rte_tm_node_params));
835 profile->reference_count++;
837 tm_node->parent = parent_node;
838 tm_node->parent_hw_id = UINT32_MAX;
839 shaper_default_red_algo(dev, tm_node, profile);
841 TAILQ_INSERT_TAIL(&dev->node_list, tm_node, node);
847 nix_tm_clear_shaper_profiles(struct otx2_eth_dev *dev)
849 struct otx2_nix_tm_shaper_profile *shaper_profile;
851 while ((shaper_profile = TAILQ_FIRST(&dev->shaper_profile_list))) {
852 if (shaper_profile->reference_count)
853 otx2_tm_dbg("Shaper profile %u has non zero references",
854 shaper_profile->shaper_profile_id);
855 TAILQ_REMOVE(&dev->shaper_profile_list, shaper_profile, shaper);
856 rte_free(shaper_profile);
863 nix_clear_path_xoff(struct otx2_eth_dev *dev,
864 struct otx2_nix_tm_node *tm_node)
866 struct nix_txschq_config *req;
867 struct otx2_nix_tm_node *p;
870 /* Manipulating SW_XOFF not supported on Ax */
871 if (otx2_dev_is_Ax(dev))
874 /* Enable nodes in path for flush to succeed */
875 if (!nix_tm_is_leaf(dev, tm_node->lvl))
880 if (!(p->flags & NIX_TM_NODE_ENABLED) &&
881 (p->flags & NIX_TM_NODE_HWRES)) {
882 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
883 req->lvl = p->hw_lvl;
884 req->num_regs = prepare_tm_sw_xoff(p, false, req->reg,
886 rc = otx2_mbox_process(dev->mbox);
890 p->flags |= NIX_TM_NODE_ENABLED;
899 nix_smq_xoff(struct otx2_eth_dev *dev,
900 struct otx2_nix_tm_node *tm_node,
903 struct otx2_mbox *mbox = dev->mbox;
904 struct nix_txschq_config *req;
908 smq = tm_node->hw_id;
909 otx2_tm_dbg("Setting SMQ %u XOFF/FLUSH to %s", smq,
910 enable ? "enable" : "disable");
912 rc = nix_clear_path_xoff(dev, tm_node);
916 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
917 req->lvl = NIX_TXSCH_LVL_SMQ;
920 req->reg[0] = NIX_AF_SMQX_CFG(smq);
921 req->regval[0] = enable ? (BIT_ULL(50) | BIT_ULL(49)) : 0;
922 req->regval_mask[0] = enable ?
923 ~(BIT_ULL(50) | BIT_ULL(49)) : ~BIT_ULL(50);
925 return otx2_mbox_process(mbox);
929 otx2_nix_sq_sqb_aura_fc(void *__txq, bool enable)
931 struct otx2_eth_txq *txq = __txq;
932 struct npa_aq_enq_req *req;
933 struct npa_aq_enq_rsp *rsp;
934 struct otx2_npa_lf *lf;
935 struct otx2_mbox *mbox;
936 uint64_t aura_handle;
939 otx2_tm_dbg("Setting SQ %u SQB aura FC to %s", txq->sq,
940 enable ? "enable" : "disable");
942 lf = otx2_npa_lf_obj_get();
946 /* Set/clear sqb aura fc_ena */
947 aura_handle = txq->sqb_pool->pool_id;
948 req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
950 req->aura_id = npa_lf_aura_handle_to_aura(aura_handle);
951 req->ctype = NPA_AQ_CTYPE_AURA;
952 req->op = NPA_AQ_INSTOP_WRITE;
953 /* Below is not needed for aura writes but AF driver needs it */
954 /* AF will translate to associated poolctx */
955 req->aura.pool_addr = req->aura_id;
957 req->aura.fc_ena = enable;
958 req->aura_mask.fc_ena = 1;
960 rc = otx2_mbox_process(mbox);
964 /* Read back npa aura ctx */
965 req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
967 req->aura_id = npa_lf_aura_handle_to_aura(aura_handle);
968 req->ctype = NPA_AQ_CTYPE_AURA;
969 req->op = NPA_AQ_INSTOP_READ;
971 rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
975 /* Init when enabled as there might be no triggers */
977 *(volatile uint64_t *)txq->fc_mem = rsp->aura.count;
979 *(volatile uint64_t *)txq->fc_mem = txq->nb_sqb_bufs;
980 /* Sync write barrier */
987 nix_txq_flush_sq_spin(struct otx2_eth_txq *txq)
989 uint16_t sqb_cnt, head_off, tail_off;
990 struct otx2_eth_dev *dev = txq->dev;
991 uint64_t wdata, val, prev;
992 uint16_t sq = txq->sq;
994 uint64_t timeout;/* 10's of usec */
996 /* Wait for enough time based on shaper min rate */
997 timeout = (txq->qconf.nb_desc * NIX_MAX_HW_FRS * 8 * 1E5);
998 timeout = timeout / dev->tm_rate_min;
1002 wdata = ((uint64_t)sq << 32);
1003 regaddr = (int64_t *)(dev->base + NIX_LF_SQ_OP_STATUS);
1004 val = otx2_atomic64_add_nosync(wdata, regaddr);
1006 /* Spin multiple iterations as "txq->fc_cache_pkts" can still
1007 * have space to send pkts even though fc_mem is disabled
1013 val = otx2_atomic64_add_nosync(wdata, regaddr);
1014 /* Continue on error */
1015 if (val & BIT_ULL(63))
1021 sqb_cnt = val & 0xFFFF;
1022 head_off = (val >> 20) & 0x3F;
1023 tail_off = (val >> 28) & 0x3F;
1025 /* SQ reached quiescent state */
1026 if (sqb_cnt <= 1 && head_off == tail_off &&
1027 (*txq->fc_mem == txq->nb_sqb_bufs)) {
1039 otx2_nix_tm_dump(dev);
1043 /* Flush and disable tx queue and its parent SMQ */
1044 int otx2_nix_sq_flush_pre(void *_txq, bool dev_started)
1046 struct otx2_nix_tm_node *tm_node, *sibling;
1047 struct otx2_eth_txq *txq;
1048 struct otx2_eth_dev *dev;
1057 user = !!(dev->tm_flags & NIX_TM_COMMITTED);
1059 /* Find the node for this SQ */
1060 tm_node = nix_tm_node_search(dev, sq, user);
1061 if (!tm_node || !(tm_node->flags & NIX_TM_NODE_ENABLED)) {
1062 otx2_err("Invalid node/state for sq %u", sq);
1066 /* Enable CGX RXTX to drain pkts */
1068 /* Though it enables both RX MCAM Entries and CGX Link
1069 * we assume all the rx queues are stopped way back.
1071 otx2_mbox_alloc_msg_nix_lf_start_rx(dev->mbox);
1072 rc = otx2_mbox_process(dev->mbox);
1074 otx2_err("cgx start failed, rc=%d", rc);
1079 /* Disable smq xoff for case it was enabled earlier */
1080 rc = nix_smq_xoff(dev, tm_node->parent, false);
1082 otx2_err("Failed to enable smq %u, rc=%d",
1083 tm_node->parent->hw_id, rc);
1087 /* As per HRM, to disable an SQ, all other SQ's
1088 * that feed to same SMQ must be paused before SMQ flush.
1090 TAILQ_FOREACH(sibling, &dev->node_list, node) {
1091 if (sibling->parent != tm_node->parent)
1093 if (!(sibling->flags & NIX_TM_NODE_ENABLED))
1097 txq = dev->eth_dev->data->tx_queues[sq];
1101 rc = otx2_nix_sq_sqb_aura_fc(txq, false);
1103 otx2_err("Failed to disable sqb aura fc, rc=%d", rc);
1107 /* Wait for sq entries to be flushed */
1108 rc = nix_txq_flush_sq_spin(txq);
1110 otx2_err("Failed to drain sq %u, rc=%d\n", txq->sq, rc);
1115 tm_node->flags &= ~NIX_TM_NODE_ENABLED;
1117 /* Disable and flush */
1118 rc = nix_smq_xoff(dev, tm_node->parent, true);
1120 otx2_err("Failed to disable smq %u, rc=%d",
1121 tm_node->parent->hw_id, rc);
1125 /* Restore cgx state */
1127 otx2_mbox_alloc_msg_nix_lf_stop_rx(dev->mbox);
1128 rc |= otx2_mbox_process(dev->mbox);
1134 int otx2_nix_sq_flush_post(void *_txq)
1136 struct otx2_nix_tm_node *tm_node, *sibling;
1137 struct otx2_eth_txq *txq = _txq;
1138 struct otx2_eth_txq *s_txq;
1139 struct otx2_eth_dev *dev;
1147 user = !!(dev->tm_flags & NIX_TM_COMMITTED);
1149 /* Find the node for this SQ */
1150 tm_node = nix_tm_node_search(dev, sq, user);
1152 otx2_err("Invalid node for sq %u", sq);
1156 /* Enable all the siblings back */
1157 TAILQ_FOREACH(sibling, &dev->node_list, node) {
1158 if (sibling->parent != tm_node->parent)
1161 if (sibling->id == sq)
1164 if (!(sibling->flags & NIX_TM_NODE_ENABLED))
1168 s_txq = dev->eth_dev->data->tx_queues[s_sq];
1173 /* Enable back if any SQ is still present */
1174 rc = nix_smq_xoff(dev, tm_node->parent, false);
1176 otx2_err("Failed to enable smq %u, rc=%d",
1177 tm_node->parent->hw_id, rc);
1183 rc = otx2_nix_sq_sqb_aura_fc(s_txq, true);
1185 otx2_err("Failed to enable sqb aura fc, rc=%d", rc);
1194 nix_sq_sched_data(struct otx2_eth_dev *dev,
1195 struct otx2_nix_tm_node *tm_node,
1196 bool rr_quantum_only)
1198 struct rte_eth_dev *eth_dev = dev->eth_dev;
1199 struct otx2_mbox *mbox = dev->mbox;
1200 uint16_t sq = tm_node->id, smq;
1201 struct nix_aq_enq_req *req;
1202 uint64_t rr_quantum;
1205 smq = tm_node->parent->hw_id;
1206 rr_quantum = NIX_TM_WEIGHT_TO_RR_QUANTUM(tm_node->weight);
1208 if (rr_quantum_only)
1209 otx2_tm_dbg("Update sq(%u) rr_quantum 0x%"PRIx64, sq, rr_quantum);
1211 otx2_tm_dbg("Enabling sq(%u)->smq(%u), rr_quantum 0x%"PRIx64,
1212 sq, smq, rr_quantum);
1214 if (sq > eth_dev->data->nb_tx_queues)
1217 req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
1219 req->ctype = NIX_AQ_CTYPE_SQ;
1220 req->op = NIX_AQ_INSTOP_WRITE;
1222 /* smq update only when needed */
1223 if (!rr_quantum_only) {
1225 req->sq_mask.smq = ~req->sq_mask.smq;
1227 req->sq.smq_rr_quantum = rr_quantum;
1228 req->sq_mask.smq_rr_quantum = ~req->sq_mask.smq_rr_quantum;
1230 rc = otx2_mbox_process(mbox);
1232 otx2_err("Failed to set smq, rc=%d", rc);
1236 int otx2_nix_sq_enable(void *_txq)
1238 struct otx2_eth_txq *txq = _txq;
1241 /* Enable sqb_aura fc */
1242 rc = otx2_nix_sq_sqb_aura_fc(txq, true);
1244 otx2_err("Failed to enable sqb aura fc, rc=%d", rc);
1252 nix_tm_free_resources(struct otx2_eth_dev *dev, uint32_t flags_mask,
1253 uint32_t flags, bool hw_only)
1255 struct otx2_nix_tm_shaper_profile *profile;
1256 struct otx2_nix_tm_node *tm_node, *next_node;
1257 struct otx2_mbox *mbox = dev->mbox;
1258 struct nix_txsch_free_req *req;
1259 uint32_t profile_id;
1262 next_node = TAILQ_FIRST(&dev->node_list);
1264 tm_node = next_node;
1265 next_node = TAILQ_NEXT(tm_node, node);
1267 /* Check for only requested nodes */
1268 if ((tm_node->flags & flags_mask) != flags)
1271 if (!nix_tm_is_leaf(dev, tm_node->lvl) &&
1272 tm_node->hw_lvl != NIX_TXSCH_LVL_TL1 &&
1273 tm_node->flags & NIX_TM_NODE_HWRES) {
1274 /* Free specific HW resource */
1275 otx2_tm_dbg("Free hwres %s(%u) lvl %u id %u (%p)",
1276 nix_hwlvl2str(tm_node->hw_lvl),
1277 tm_node->hw_id, tm_node->lvl,
1278 tm_node->id, tm_node);
1280 rc = nix_clear_path_xoff(dev, tm_node);
1284 req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
1286 req->schq_lvl = tm_node->hw_lvl;
1287 req->schq = tm_node->hw_id;
1288 rc = otx2_mbox_process(mbox);
1291 tm_node->flags &= ~NIX_TM_NODE_HWRES;
1294 /* Leave software elements if needed */
1298 otx2_tm_dbg("Free node lvl %u id %u (%p)",
1299 tm_node->lvl, tm_node->id, tm_node);
1301 profile_id = tm_node->params.shaper_profile_id;
1302 profile = nix_tm_shaper_profile_search(dev, profile_id);
1304 profile->reference_count--;
1306 TAILQ_REMOVE(&dev->node_list, tm_node, node);
1311 /* Free all hw resources */
1312 req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
1313 req->flags = TXSCHQ_FREE_ALL;
1315 return otx2_mbox_process(mbox);
1322 nix_tm_copy_rsp_to_dev(struct otx2_eth_dev *dev,
1323 struct nix_txsch_alloc_rsp *rsp)
1328 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1329 for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) {
1330 dev->txschq_list[lvl][schq] = rsp->schq_list[lvl][schq];
1331 dev->txschq_contig_list[lvl][schq] =
1332 rsp->schq_contig_list[lvl][schq];
1335 dev->txschq[lvl] = rsp->schq[lvl];
1336 dev->txschq_contig[lvl] = rsp->schq_contig[lvl];
1342 nix_tm_assign_id_to_node(struct otx2_eth_dev *dev,
1343 struct otx2_nix_tm_node *child,
1344 struct otx2_nix_tm_node *parent)
1346 uint32_t hw_id, schq_con_index, prio_offset;
1347 uint32_t l_id, schq_index;
1349 otx2_tm_dbg("Assign hw id for child node %s lvl %u id %u (%p)",
1350 nix_hwlvl2str(child->hw_lvl), child->lvl, child->id, child);
1352 child->flags |= NIX_TM_NODE_HWRES;
1354 /* Process root nodes */
1355 if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
1356 child->hw_lvl == dev->otx2_tm_root_lvl && !parent) {
1358 uint32_t tschq_con_index;
1360 l_id = child->hw_lvl;
1361 tschq_con_index = dev->txschq_contig_index[l_id];
1362 hw_id = dev->txschq_contig_list[l_id][tschq_con_index];
1363 child->hw_id = hw_id;
1364 dev->txschq_contig_index[l_id]++;
1365 /* Update TL1 hw_id for its parent for config purpose */
1366 idx = dev->txschq_index[NIX_TXSCH_LVL_TL1]++;
1367 hw_id = dev->txschq_list[NIX_TXSCH_LVL_TL1][idx];
1368 child->parent_hw_id = hw_id;
1371 if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL1 &&
1372 child->hw_lvl == dev->otx2_tm_root_lvl && !parent) {
1373 uint32_t tschq_con_index;
1375 l_id = child->hw_lvl;
1376 tschq_con_index = dev->txschq_index[l_id];
1377 hw_id = dev->txschq_list[l_id][tschq_con_index];
1378 child->hw_id = hw_id;
1379 dev->txschq_index[l_id]++;
1383 /* Process children with parents */
1384 l_id = child->hw_lvl;
1385 schq_index = dev->txschq_index[l_id];
1386 schq_con_index = dev->txschq_contig_index[l_id];
1388 if (child->priority == parent->rr_prio) {
1389 hw_id = dev->txschq_list[l_id][schq_index];
1390 child->hw_id = hw_id;
1391 child->parent_hw_id = parent->hw_id;
1392 dev->txschq_index[l_id]++;
1394 prio_offset = schq_con_index + child->priority;
1395 hw_id = dev->txschq_contig_list[l_id][prio_offset];
1396 child->hw_id = hw_id;
1402 nix_tm_assign_hw_id(struct otx2_eth_dev *dev)
1404 struct otx2_nix_tm_node *parent, *child;
1405 uint32_t child_hw_lvl, con_index_inc, i;
1407 for (i = NIX_TXSCH_LVL_TL1; i > 0; i--) {
1408 TAILQ_FOREACH(parent, &dev->node_list, node) {
1409 child_hw_lvl = parent->hw_lvl - 1;
1410 if (parent->hw_lvl != i)
1412 TAILQ_FOREACH(child, &dev->node_list, node) {
1415 if (child->parent->id != parent->id)
1417 nix_tm_assign_id_to_node(dev, child, parent);
1420 con_index_inc = parent->max_prio + 1;
1421 dev->txschq_contig_index[child_hw_lvl] += con_index_inc;
1424 * Explicitly assign id to parent node if it
1425 * doesn't have a parent
1427 if (parent->hw_lvl == dev->otx2_tm_root_lvl)
1428 nix_tm_assign_id_to_node(dev, parent, NULL);
1435 nix_tm_count_req_schq(struct otx2_eth_dev *dev,
1436 struct nix_txsch_alloc_req *req, uint8_t lvl)
1438 struct otx2_nix_tm_node *tm_node;
1439 uint8_t contig_count;
1441 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1442 if (lvl == tm_node->hw_lvl) {
1443 req->schq[lvl - 1] += tm_node->rr_num;
1444 if (tm_node->max_prio != UINT32_MAX) {
1445 contig_count = tm_node->max_prio + 1;
1446 req->schq_contig[lvl - 1] += contig_count;
1449 if (lvl == dev->otx2_tm_root_lvl &&
1450 dev->otx2_tm_root_lvl && lvl == NIX_TXSCH_LVL_TL2 &&
1451 tm_node->hw_lvl == dev->otx2_tm_root_lvl) {
1452 req->schq_contig[dev->otx2_tm_root_lvl]++;
1456 req->schq[NIX_TXSCH_LVL_TL1] = 1;
1457 req->schq_contig[NIX_TXSCH_LVL_TL1] = 0;
1463 nix_tm_prepare_txschq_req(struct otx2_eth_dev *dev,
1464 struct nix_txsch_alloc_req *req)
1468 for (i = NIX_TXSCH_LVL_TL1; i > 0; i--)
1469 nix_tm_count_req_schq(dev, req, i);
1471 for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
1472 dev->txschq_index[i] = 0;
1473 dev->txschq_contig_index[i] = 0;
1479 nix_tm_send_txsch_alloc_msg(struct otx2_eth_dev *dev)
1481 struct otx2_mbox *mbox = dev->mbox;
1482 struct nix_txsch_alloc_req *req;
1483 struct nix_txsch_alloc_rsp *rsp;
1486 req = otx2_mbox_alloc_msg_nix_txsch_alloc(mbox);
1488 rc = nix_tm_prepare_txschq_req(dev, req);
1492 rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
1496 nix_tm_copy_rsp_to_dev(dev, rsp);
1497 dev->link_cfg_lvl = rsp->link_cfg_lvl;
1499 nix_tm_assign_hw_id(dev);
1504 nix_tm_alloc_resources(struct rte_eth_dev *eth_dev, bool xmit_enable)
1506 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1507 struct otx2_nix_tm_node *tm_node;
1508 struct otx2_eth_txq *txq;
1512 nix_tm_update_parent_info(dev);
1514 rc = nix_tm_send_txsch_alloc_msg(dev);
1516 otx2_err("TM failed to alloc tm resources=%d", rc);
1520 rc = nix_tm_txsch_reg_config(dev);
1522 otx2_err("TM failed to configure sched registers=%d", rc);
1526 /* Trigger MTU recalculate as SMQ needs MTU conf */
1527 if (eth_dev->data->dev_started && eth_dev->data->nb_rx_queues) {
1528 rc = otx2_nix_recalc_mtu(eth_dev);
1530 otx2_err("TM MTU update failed, rc=%d", rc);
1535 /* Mark all non-leaf's as enabled */
1536 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1537 if (!nix_tm_is_leaf(dev, tm_node->lvl))
1538 tm_node->flags |= NIX_TM_NODE_ENABLED;
1544 /* Update SQ Sched Data while SQ is idle */
1545 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1546 if (!nix_tm_is_leaf(dev, tm_node->lvl))
1549 rc = nix_sq_sched_data(dev, tm_node, false);
1551 otx2_err("SQ %u sched update failed, rc=%d",
1557 /* Finally XON all SMQ's */
1558 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1559 if (tm_node->hw_lvl != NIX_TXSCH_LVL_SMQ)
1562 rc = nix_smq_xoff(dev, tm_node, false);
1564 otx2_err("Failed to enable smq %u, rc=%d",
1565 tm_node->hw_id, rc);
1570 /* Enable xmit as all the topology is ready */
1571 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1572 if (!nix_tm_is_leaf(dev, tm_node->lvl))
1576 txq = eth_dev->data->tx_queues[sq];
1578 rc = otx2_nix_sq_enable(txq);
1580 otx2_err("TM sw xon failed on SQ %u, rc=%d",
1584 tm_node->flags |= NIX_TM_NODE_ENABLED;
1591 send_tm_reqval(struct otx2_mbox *mbox,
1592 struct nix_txschq_config *req,
1593 struct rte_tm_error *error)
1597 if (!req->num_regs ||
1598 req->num_regs > MAX_REGS_PER_MBOX_MSG) {
1599 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
1600 error->message = "invalid config";
1604 rc = otx2_mbox_process(mbox);
1606 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
1607 error->message = "unexpected fatal error";
1613 nix_tm_lvl2nix(struct otx2_eth_dev *dev, uint32_t lvl)
1615 if (nix_tm_have_tl1_access(dev)) {
1617 case OTX2_TM_LVL_ROOT:
1618 return NIX_TXSCH_LVL_TL1;
1619 case OTX2_TM_LVL_SCH1:
1620 return NIX_TXSCH_LVL_TL2;
1621 case OTX2_TM_LVL_SCH2:
1622 return NIX_TXSCH_LVL_TL3;
1623 case OTX2_TM_LVL_SCH3:
1624 return NIX_TXSCH_LVL_TL4;
1625 case OTX2_TM_LVL_SCH4:
1626 return NIX_TXSCH_LVL_SMQ;
1628 return NIX_TXSCH_LVL_CNT;
1632 case OTX2_TM_LVL_ROOT:
1633 return NIX_TXSCH_LVL_TL2;
1634 case OTX2_TM_LVL_SCH1:
1635 return NIX_TXSCH_LVL_TL3;
1636 case OTX2_TM_LVL_SCH2:
1637 return NIX_TXSCH_LVL_TL4;
1638 case OTX2_TM_LVL_SCH3:
1639 return NIX_TXSCH_LVL_SMQ;
1641 return NIX_TXSCH_LVL_CNT;
1647 nix_max_prio(struct otx2_eth_dev *dev, uint16_t hw_lvl)
1649 if (hw_lvl >= NIX_TXSCH_LVL_CNT)
1652 /* MDQ doesn't support SP */
1653 if (hw_lvl == NIX_TXSCH_LVL_MDQ)
1656 /* PF's TL1 with VF's enabled doesn't support SP */
1657 if (hw_lvl == NIX_TXSCH_LVL_TL1 &&
1658 (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 ||
1659 (dev->tm_flags & NIX_TM_TL1_NO_SP)))
1662 return TXSCH_TLX_SP_PRIO_MAX - 1;
1667 validate_prio(struct otx2_eth_dev *dev, uint32_t lvl,
1668 uint32_t parent_id, uint32_t priority,
1669 struct rte_tm_error *error)
1671 uint8_t priorities[TXSCH_TLX_SP_PRIO_MAX];
1672 struct otx2_nix_tm_node *tm_node;
1673 uint32_t rr_num = 0;
1676 /* Validate priority against max */
1677 if (priority > nix_max_prio(dev, nix_tm_lvl2nix(dev, lvl - 1))) {
1678 error->type = RTE_TM_ERROR_TYPE_CAPABILITIES;
1679 error->message = "unsupported priority value";
1683 if (parent_id == RTE_TM_NODE_ID_NULL)
1686 memset(priorities, 0, TXSCH_TLX_SP_PRIO_MAX);
1687 priorities[priority] = 1;
1689 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1690 if (!tm_node->parent)
1693 if (!(tm_node->flags & NIX_TM_NODE_USER))
1696 if (tm_node->parent->id != parent_id)
1699 priorities[tm_node->priority]++;
1702 for (i = 0; i < TXSCH_TLX_SP_PRIO_MAX; i++)
1703 if (priorities[i] > 1)
1706 /* At max, one rr groups per parent */
1708 error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY;
1709 error->message = "multiple DWRR node priority";
1713 /* Check for previous priority to avoid holes in priorities */
1714 if (priority && !priorities[priority - 1]) {
1715 error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY;
1716 error->message = "priority not in order";
1724 read_tm_reg(struct otx2_mbox *mbox, uint64_t reg,
1725 uint64_t *regval, uint32_t hw_lvl)
1727 volatile struct nix_txschq_config *req;
1728 struct nix_txschq_config *rsp;
1731 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
1737 rc = otx2_mbox_process_msg(mbox, (void **)&rsp);
1740 *regval = rsp->regval[0];
1744 /* Search for min rate in topology */
1746 nix_tm_shaper_profile_update_min(struct otx2_eth_dev *dev)
1748 struct otx2_nix_tm_shaper_profile *profile;
1749 uint64_t rate_min = 1E9; /* 1 Gbps */
1751 TAILQ_FOREACH(profile, &dev->shaper_profile_list, shaper) {
1752 if (profile->params.peak.rate &&
1753 profile->params.peak.rate < rate_min)
1754 rate_min = profile->params.peak.rate;
1756 if (profile->params.committed.rate &&
1757 profile->params.committed.rate < rate_min)
1758 rate_min = profile->params.committed.rate;
1761 dev->tm_rate_min = rate_min;
1765 nix_xmit_disable(struct rte_eth_dev *eth_dev)
1767 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1768 uint16_t sq_cnt = eth_dev->data->nb_tx_queues;
1769 uint16_t sqb_cnt, head_off, tail_off;
1770 struct otx2_nix_tm_node *tm_node;
1771 struct otx2_eth_txq *txq;
1772 uint64_t wdata, val;
1775 otx2_tm_dbg("Disabling xmit on %s", eth_dev->data->name);
1777 /* Enable CGX RXTX to drain pkts */
1778 if (!eth_dev->data->dev_started) {
1779 otx2_mbox_alloc_msg_nix_lf_start_rx(dev->mbox);
1780 rc = otx2_mbox_process(dev->mbox);
1786 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1787 if (tm_node->hw_lvl != NIX_TXSCH_LVL_SMQ)
1789 if (!(tm_node->flags & NIX_TM_NODE_HWRES))
1792 rc = nix_smq_xoff(dev, tm_node, false);
1794 otx2_err("Failed to enable smq %u, rc=%d",
1795 tm_node->hw_id, rc);
1800 /* Flush all tx queues */
1801 for (i = 0; i < sq_cnt; i++) {
1802 txq = eth_dev->data->tx_queues[i];
1804 rc = otx2_nix_sq_sqb_aura_fc(txq, false);
1806 otx2_err("Failed to disable sqb aura fc, rc=%d", rc);
1810 /* Wait for sq entries to be flushed */
1811 rc = nix_txq_flush_sq_spin(txq);
1813 otx2_err("Failed to drain sq, rc=%d\n", rc);
1818 /* XOFF & Flush all SMQ's. HRM mandates
1819 * all SQ's empty before SMQ flush is issued.
1821 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1822 if (tm_node->hw_lvl != NIX_TXSCH_LVL_SMQ)
1824 if (!(tm_node->flags & NIX_TM_NODE_HWRES))
1827 rc = nix_smq_xoff(dev, tm_node, true);
1829 otx2_err("Failed to enable smq %u, rc=%d",
1830 tm_node->hw_id, rc);
1835 /* Verify sanity of all tx queues */
1836 for (i = 0; i < sq_cnt; i++) {
1837 txq = eth_dev->data->tx_queues[i];
1839 wdata = ((uint64_t)txq->sq << 32);
1840 val = otx2_atomic64_add_nosync(wdata,
1841 (int64_t *)(dev->base + NIX_LF_SQ_OP_STATUS));
1843 sqb_cnt = val & 0xFFFF;
1844 head_off = (val >> 20) & 0x3F;
1845 tail_off = (val >> 28) & 0x3F;
1847 if (sqb_cnt > 1 || head_off != tail_off ||
1848 (*txq->fc_mem != txq->nb_sqb_bufs))
1849 otx2_err("Failed to gracefully flush sq %u", txq->sq);
1853 /* restore cgx state */
1854 if (!eth_dev->data->dev_started) {
1855 otx2_mbox_alloc_msg_nix_lf_stop_rx(dev->mbox);
1856 rc |= otx2_mbox_process(dev->mbox);
1863 otx2_nix_tm_node_type_get(struct rte_eth_dev *eth_dev, uint32_t node_id,
1864 int *is_leaf, struct rte_tm_error *error)
1866 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1867 struct otx2_nix_tm_node *tm_node;
1869 if (is_leaf == NULL) {
1870 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
1874 tm_node = nix_tm_node_search(dev, node_id, true);
1875 if (node_id == RTE_TM_NODE_ID_NULL || !tm_node) {
1876 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
1879 if (nix_tm_is_leaf(dev, tm_node->lvl))
1887 otx2_nix_tm_capa_get(struct rte_eth_dev *eth_dev,
1888 struct rte_tm_capabilities *cap,
1889 struct rte_tm_error *error)
1891 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1892 struct otx2_mbox *mbox = dev->mbox;
1893 int rc, max_nr_nodes = 0, i;
1894 struct free_rsrcs_rsp *rsp;
1896 memset(cap, 0, sizeof(*cap));
1898 otx2_mbox_alloc_msg_free_rsrc_cnt(mbox);
1899 rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
1901 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
1902 error->message = "unexpected fatal error";
1906 for (i = 0; i < NIX_TXSCH_LVL_TL1; i++)
1907 max_nr_nodes += rsp->schq[i];
1909 cap->n_nodes_max = max_nr_nodes + dev->tm_leaf_cnt;
1910 /* TL1 level is reserved for PF */
1911 cap->n_levels_max = nix_tm_have_tl1_access(dev) ?
1912 OTX2_TM_LVL_MAX : OTX2_TM_LVL_MAX - 1;
1913 cap->non_leaf_nodes_identical = 1;
1914 cap->leaf_nodes_identical = 1;
1916 /* Shaper Capabilities */
1917 cap->shaper_private_n_max = max_nr_nodes;
1918 cap->shaper_n_max = max_nr_nodes;
1919 cap->shaper_private_dual_rate_n_max = max_nr_nodes;
1920 cap->shaper_private_rate_min = MIN_SHAPER_RATE / 8;
1921 cap->shaper_private_rate_max = MAX_SHAPER_RATE / 8;
1922 cap->shaper_private_packet_mode_supported = 1;
1923 cap->shaper_private_byte_mode_supported = 1;
1924 cap->shaper_pkt_length_adjust_min = NIX_LENGTH_ADJUST_MIN;
1925 cap->shaper_pkt_length_adjust_max = NIX_LENGTH_ADJUST_MAX;
1927 /* Schedule Capabilities */
1928 cap->sched_n_children_max = rsp->schq[NIX_TXSCH_LVL_MDQ];
1929 cap->sched_sp_n_priorities_max = TXSCH_TLX_SP_PRIO_MAX;
1930 cap->sched_wfq_n_children_per_group_max = cap->sched_n_children_max;
1931 cap->sched_wfq_n_groups_max = 1;
1932 cap->sched_wfq_weight_max = MAX_SCHED_WEIGHT;
1933 cap->sched_wfq_packet_mode_supported = 1;
1934 cap->sched_wfq_byte_mode_supported = 1;
1936 cap->dynamic_update_mask =
1937 RTE_TM_UPDATE_NODE_PARENT_KEEP_LEVEL |
1938 RTE_TM_UPDATE_NODE_SUSPEND_RESUME;
1940 RTE_TM_STATS_N_PKTS |
1941 RTE_TM_STATS_N_BYTES |
1942 RTE_TM_STATS_N_PKTS_RED_DROPPED |
1943 RTE_TM_STATS_N_BYTES_RED_DROPPED;
1945 for (i = 0; i < RTE_COLORS; i++) {
1946 cap->mark_vlan_dei_supported[i] = false;
1947 cap->mark_ip_ecn_tcp_supported[i] = false;
1948 cap->mark_ip_dscp_supported[i] = false;
1955 otx2_nix_tm_level_capa_get(struct rte_eth_dev *eth_dev, uint32_t lvl,
1956 struct rte_tm_level_capabilities *cap,
1957 struct rte_tm_error *error)
1959 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1960 struct otx2_mbox *mbox = dev->mbox;
1961 struct free_rsrcs_rsp *rsp;
1965 memset(cap, 0, sizeof(*cap));
1967 otx2_mbox_alloc_msg_free_rsrc_cnt(mbox);
1968 rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
1970 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
1971 error->message = "unexpected fatal error";
1975 hw_lvl = nix_tm_lvl2nix(dev, lvl);
1977 if (nix_tm_is_leaf(dev, lvl)) {
1979 cap->n_nodes_max = dev->tm_leaf_cnt;
1980 cap->n_nodes_leaf_max = dev->tm_leaf_cnt;
1981 cap->leaf_nodes_identical = 1;
1982 cap->leaf.stats_mask =
1983 RTE_TM_STATS_N_PKTS |
1984 RTE_TM_STATS_N_BYTES;
1986 } else if (lvl == OTX2_TM_LVL_ROOT) {
1987 /* Root node, aka TL2(vf)/TL1(pf) */
1988 cap->n_nodes_max = 1;
1989 cap->n_nodes_nonleaf_max = 1;
1990 cap->non_leaf_nodes_identical = 1;
1992 cap->nonleaf.shaper_private_supported = true;
1993 cap->nonleaf.shaper_private_dual_rate_supported =
1994 nix_tm_have_tl1_access(dev) ? false : true;
1995 cap->nonleaf.shaper_private_rate_min = MIN_SHAPER_RATE / 8;
1996 cap->nonleaf.shaper_private_rate_max = MAX_SHAPER_RATE / 8;
1997 cap->nonleaf.shaper_private_packet_mode_supported = 1;
1998 cap->nonleaf.shaper_private_byte_mode_supported = 1;
2000 cap->nonleaf.sched_n_children_max = rsp->schq[hw_lvl - 1];
2001 cap->nonleaf.sched_sp_n_priorities_max =
2002 nix_max_prio(dev, hw_lvl) + 1;
2003 cap->nonleaf.sched_wfq_n_groups_max = 1;
2004 cap->nonleaf.sched_wfq_weight_max = MAX_SCHED_WEIGHT;
2005 cap->nonleaf.sched_wfq_packet_mode_supported = 1;
2006 cap->nonleaf.sched_wfq_byte_mode_supported = 1;
2008 if (nix_tm_have_tl1_access(dev))
2009 cap->nonleaf.stats_mask =
2010 RTE_TM_STATS_N_PKTS_RED_DROPPED |
2011 RTE_TM_STATS_N_BYTES_RED_DROPPED;
2012 } else if ((lvl < OTX2_TM_LVL_MAX) &&
2013 (hw_lvl < NIX_TXSCH_LVL_CNT)) {
2014 /* TL2, TL3, TL4, MDQ */
2015 cap->n_nodes_max = rsp->schq[hw_lvl];
2016 cap->n_nodes_nonleaf_max = cap->n_nodes_max;
2017 cap->non_leaf_nodes_identical = 1;
2019 cap->nonleaf.shaper_private_supported = true;
2020 cap->nonleaf.shaper_private_dual_rate_supported = true;
2021 cap->nonleaf.shaper_private_rate_min = MIN_SHAPER_RATE / 8;
2022 cap->nonleaf.shaper_private_rate_max = MAX_SHAPER_RATE / 8;
2023 cap->nonleaf.shaper_private_packet_mode_supported = 1;
2024 cap->nonleaf.shaper_private_byte_mode_supported = 1;
2026 /* MDQ doesn't support Strict Priority */
2027 if (hw_lvl == NIX_TXSCH_LVL_MDQ)
2028 cap->nonleaf.sched_n_children_max = dev->tm_leaf_cnt;
2030 cap->nonleaf.sched_n_children_max =
2031 rsp->schq[hw_lvl - 1];
2032 cap->nonleaf.sched_sp_n_priorities_max =
2033 nix_max_prio(dev, hw_lvl) + 1;
2034 cap->nonleaf.sched_wfq_n_groups_max = 1;
2035 cap->nonleaf.sched_wfq_weight_max = MAX_SCHED_WEIGHT;
2036 cap->nonleaf.sched_wfq_packet_mode_supported = 1;
2037 cap->nonleaf.sched_wfq_byte_mode_supported = 1;
2039 /* unsupported level */
2040 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2047 otx2_nix_tm_node_capa_get(struct rte_eth_dev *eth_dev, uint32_t node_id,
2048 struct rte_tm_node_capabilities *cap,
2049 struct rte_tm_error *error)
2051 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2052 struct otx2_mbox *mbox = dev->mbox;
2053 struct otx2_nix_tm_node *tm_node;
2054 struct free_rsrcs_rsp *rsp;
2055 int rc, hw_lvl, lvl;
2057 memset(cap, 0, sizeof(*cap));
2059 tm_node = nix_tm_node_search(dev, node_id, true);
2061 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2062 error->message = "no such node";
2066 hw_lvl = tm_node->hw_lvl;
2070 if (nix_tm_is_leaf(dev, lvl)) {
2071 cap->stats_mask = RTE_TM_STATS_N_PKTS |
2072 RTE_TM_STATS_N_BYTES;
2076 otx2_mbox_alloc_msg_free_rsrc_cnt(mbox);
2077 rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
2079 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2080 error->message = "unexpected fatal error";
2084 /* Non Leaf Shaper */
2085 cap->shaper_private_supported = true;
2086 cap->shaper_private_dual_rate_supported =
2087 (hw_lvl == NIX_TXSCH_LVL_TL1) ? false : true;
2088 cap->shaper_private_rate_min = MIN_SHAPER_RATE / 8;
2089 cap->shaper_private_rate_max = MAX_SHAPER_RATE / 8;
2090 cap->shaper_private_packet_mode_supported = 1;
2091 cap->shaper_private_byte_mode_supported = 1;
2093 /* Non Leaf Scheduler */
2094 if (hw_lvl == NIX_TXSCH_LVL_MDQ)
2095 cap->nonleaf.sched_n_children_max = dev->tm_leaf_cnt;
2097 cap->nonleaf.sched_n_children_max = rsp->schq[hw_lvl - 1];
2099 cap->nonleaf.sched_sp_n_priorities_max = nix_max_prio(dev, hw_lvl) + 1;
2100 cap->nonleaf.sched_wfq_n_children_per_group_max =
2101 cap->nonleaf.sched_n_children_max;
2102 cap->nonleaf.sched_wfq_n_groups_max = 1;
2103 cap->nonleaf.sched_wfq_weight_max = MAX_SCHED_WEIGHT;
2104 cap->nonleaf.sched_wfq_packet_mode_supported = 1;
2105 cap->nonleaf.sched_wfq_byte_mode_supported = 1;
2107 if (hw_lvl == NIX_TXSCH_LVL_TL1)
2108 cap->stats_mask = RTE_TM_STATS_N_PKTS_RED_DROPPED |
2109 RTE_TM_STATS_N_BYTES_RED_DROPPED;
2114 otx2_nix_tm_shaper_profile_add(struct rte_eth_dev *eth_dev,
2115 uint32_t profile_id,
2116 struct rte_tm_shaper_params *params,
2117 struct rte_tm_error *error)
2119 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2120 struct otx2_nix_tm_shaper_profile *profile;
2122 profile = nix_tm_shaper_profile_search(dev, profile_id);
2124 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
2125 error->message = "shaper profile ID exist";
2129 /* Committed rate and burst size can be enabled/disabled */
2130 if (params->committed.size || params->committed.rate) {
2131 if (params->committed.size < MIN_SHAPER_BURST ||
2132 params->committed.size > MAX_SHAPER_BURST) {
2134 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE;
2136 } else if (!shaper_rate_to_nix(params->committed.rate * 8,
2137 NULL, NULL, NULL)) {
2139 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE;
2140 error->message = "shaper committed rate invalid";
2145 /* Peak rate and burst size can be enabled/disabled */
2146 if (params->peak.size || params->peak.rate) {
2147 if (params->peak.size < MIN_SHAPER_BURST ||
2148 params->peak.size > MAX_SHAPER_BURST) {
2150 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE;
2152 } else if (!shaper_rate_to_nix(params->peak.rate * 8,
2153 NULL, NULL, NULL)) {
2155 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE;
2156 error->message = "shaper peak rate invalid";
2161 if (params->pkt_length_adjust < NIX_LENGTH_ADJUST_MIN ||
2162 params->pkt_length_adjust > NIX_LENGTH_ADJUST_MAX) {
2163 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN;
2164 error->message = "length adjust invalid";
2168 profile = rte_zmalloc("otx2_nix_tm_shaper_profile",
2169 sizeof(struct otx2_nix_tm_shaper_profile), 0);
2173 profile->shaper_profile_id = profile_id;
2174 rte_memcpy(&profile->params, params,
2175 sizeof(struct rte_tm_shaper_params));
2176 TAILQ_INSERT_TAIL(&dev->shaper_profile_list, profile, shaper);
2178 otx2_tm_dbg("Added TM shaper profile %u, "
2179 " pir %" PRIu64 " , pbs %" PRIu64 ", cir %" PRIu64
2180 ", cbs %" PRIu64 " , adj %u, pkt mode %d",
2182 params->peak.rate * 8,
2184 params->committed.rate * 8,
2185 params->committed.size,
2186 params->pkt_length_adjust,
2187 params->packet_mode);
2189 /* Translate rate as bits per second */
2190 profile->params.peak.rate = profile->params.peak.rate * 8;
2191 profile->params.committed.rate = profile->params.committed.rate * 8;
2192 /* Always use PIR for single rate shaping */
2193 if (!params->peak.rate && params->committed.rate) {
2194 profile->params.peak = profile->params.committed;
2195 memset(&profile->params.committed, 0,
2196 sizeof(profile->params.committed));
2199 /* update min rate */
2200 nix_tm_shaper_profile_update_min(dev);
2205 otx2_nix_tm_shaper_profile_delete(struct rte_eth_dev *eth_dev,
2206 uint32_t profile_id,
2207 struct rte_tm_error *error)
2209 struct otx2_nix_tm_shaper_profile *profile;
2210 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2212 profile = nix_tm_shaper_profile_search(dev, profile_id);
2215 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
2216 error->message = "shaper profile ID not exist";
2220 if (profile->reference_count) {
2221 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE;
2222 error->message = "shaper profile in use";
2226 otx2_tm_dbg("Removing TM shaper profile %u", profile_id);
2227 TAILQ_REMOVE(&dev->shaper_profile_list, profile, shaper);
2230 /* update min rate */
2231 nix_tm_shaper_profile_update_min(dev);
2236 otx2_nix_tm_node_add(struct rte_eth_dev *eth_dev, uint32_t node_id,
2237 uint32_t parent_node_id, uint32_t priority,
2238 uint32_t weight, uint32_t lvl,
2239 struct rte_tm_node_params *params,
2240 struct rte_tm_error *error)
2242 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2243 struct otx2_nix_tm_shaper_profile *profile = NULL;
2244 struct otx2_nix_tm_node *parent_node;
2245 int rc, pkt_mode, clear_on_fail = 0;
2246 uint32_t exp_next_lvl, i;
2247 uint32_t profile_id;
2250 /* we don't support dynamic updates */
2251 if (dev->tm_flags & NIX_TM_COMMITTED) {
2252 error->type = RTE_TM_ERROR_TYPE_CAPABILITIES;
2253 error->message = "dynamic update not supported";
2257 /* Leaf nodes have to be same priority */
2258 if (nix_tm_is_leaf(dev, lvl) && priority != 0) {
2259 error->type = RTE_TM_ERROR_TYPE_CAPABILITIES;
2260 error->message = "queue shapers must be priority 0";
2264 parent_node = nix_tm_node_search(dev, parent_node_id, true);
2266 /* find the right level */
2267 if (lvl == RTE_TM_NODE_LEVEL_ID_ANY) {
2268 if (parent_node_id == RTE_TM_NODE_ID_NULL) {
2269 lvl = OTX2_TM_LVL_ROOT;
2270 } else if (parent_node) {
2271 lvl = parent_node->lvl + 1;
2273 /* Neigher proper parent nor proper level id given */
2274 error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
2275 error->message = "invalid parent node id";
2280 /* Translate rte_tm level id's to nix hw level id's */
2281 hw_lvl = nix_tm_lvl2nix(dev, lvl);
2282 if (hw_lvl == NIX_TXSCH_LVL_CNT &&
2283 !nix_tm_is_leaf(dev, lvl)) {
2284 error->type = RTE_TM_ERROR_TYPE_LEVEL_ID;
2285 error->message = "invalid level id";
2289 if (node_id < dev->tm_leaf_cnt)
2290 exp_next_lvl = NIX_TXSCH_LVL_SMQ;
2292 exp_next_lvl = hw_lvl + 1;
2294 /* Check if there is no parent node yet */
2295 if (hw_lvl != dev->otx2_tm_root_lvl &&
2296 (!parent_node || parent_node->hw_lvl != exp_next_lvl)) {
2297 error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
2298 error->message = "invalid parent node id";
2302 /* Check if a node already exists */
2303 if (nix_tm_node_search(dev, node_id, true)) {
2304 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2305 error->message = "node already exists";
2309 if (!nix_tm_is_leaf(dev, lvl)) {
2310 /* Check if shaper profile exists for non leaf node */
2311 profile_id = params->shaper_profile_id;
2312 profile = nix_tm_shaper_profile_search(dev, profile_id);
2313 if (profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE && !profile) {
2314 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
2315 error->message = "invalid shaper profile";
2319 /* Minimum static priority count is 1 */
2320 if (!params->nonleaf.n_sp_priorities ||
2321 params->nonleaf.n_sp_priorities > TXSCH_TLX_SP_PRIO_MAX) {
2323 RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES;
2324 error->message = "invalid sp priorities";
2329 /* Validate weight mode */
2330 for (i = 0; i < params->nonleaf.n_sp_priorities &&
2331 params->nonleaf.wfq_weight_mode; i++) {
2332 pkt_mode = !params->nonleaf.wfq_weight_mode[i];
2333 if (pkt_mode == !params->nonleaf.wfq_weight_mode[0])
2337 RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE;
2338 error->message = "unsupported weight mode";
2342 if (profile && params->nonleaf.n_sp_priorities &&
2343 pkt_mode != profile->params.packet_mode) {
2344 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE;
2345 error->message = "shaper wfq packet mode mismatch";
2350 /* Check if there is second DWRR already in siblings or holes in prio */
2351 if (validate_prio(dev, lvl, parent_node_id, priority, error))
2354 if (weight > MAX_SCHED_WEIGHT) {
2355 error->type = RTE_TM_ERROR_TYPE_NODE_WEIGHT;
2356 error->message = "max weight exceeded";
2360 rc = nix_tm_node_add_to_list(dev, node_id, parent_node_id,
2361 priority, weight, hw_lvl,
2364 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2365 /* cleanup user added nodes */
2367 nix_tm_free_resources(dev, NIX_TM_NODE_USER,
2368 NIX_TM_NODE_USER, false);
2369 error->message = "failed to add node";
2372 error->type = RTE_TM_ERROR_TYPE_NONE;
2377 otx2_nix_tm_node_delete(struct rte_eth_dev *eth_dev, uint32_t node_id,
2378 struct rte_tm_error *error)
2380 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2381 struct otx2_nix_tm_node *tm_node, *child_node;
2382 struct otx2_nix_tm_shaper_profile *profile;
2383 uint32_t profile_id;
2385 /* we don't support dynamic updates yet */
2386 if (dev->tm_flags & NIX_TM_COMMITTED) {
2387 error->type = RTE_TM_ERROR_TYPE_CAPABILITIES;
2388 error->message = "hierarchy exists";
2392 if (node_id == RTE_TM_NODE_ID_NULL) {
2393 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2394 error->message = "invalid node id";
2398 tm_node = nix_tm_node_search(dev, node_id, true);
2400 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2401 error->message = "no such node";
2405 /* Check for any existing children */
2406 TAILQ_FOREACH(child_node, &dev->node_list, node) {
2407 if (child_node->parent == tm_node) {
2408 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2409 error->message = "children exist";
2414 /* Remove shaper profile reference */
2415 profile_id = tm_node->params.shaper_profile_id;
2416 profile = nix_tm_shaper_profile_search(dev, profile_id);
2417 profile->reference_count--;
2419 TAILQ_REMOVE(&dev->node_list, tm_node, node);
2425 nix_tm_node_suspend_resume(struct rte_eth_dev *eth_dev, uint32_t node_id,
2426 struct rte_tm_error *error, bool suspend)
2428 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2429 struct otx2_mbox *mbox = dev->mbox;
2430 struct otx2_nix_tm_node *tm_node;
2431 struct nix_txschq_config *req;
2435 tm_node = nix_tm_node_search(dev, node_id, true);
2437 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2438 error->message = "no such node";
2442 if (!(dev->tm_flags & NIX_TM_COMMITTED)) {
2443 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2444 error->message = "hierarchy doesn't exist";
2448 flags = tm_node->flags;
2449 flags = suspend ? (flags & ~NIX_TM_NODE_ENABLED) :
2450 (flags | NIX_TM_NODE_ENABLED);
2452 if (tm_node->flags == flags)
2455 /* send mbox for state change */
2456 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
2458 req->lvl = tm_node->hw_lvl;
2459 req->num_regs = prepare_tm_sw_xoff(tm_node, suspend,
2460 req->reg, req->regval);
2461 rc = send_tm_reqval(mbox, req, error);
2463 tm_node->flags = flags;
2468 otx2_nix_tm_node_suspend(struct rte_eth_dev *eth_dev, uint32_t node_id,
2469 struct rte_tm_error *error)
2471 return nix_tm_node_suspend_resume(eth_dev, node_id, error, true);
2475 otx2_nix_tm_node_resume(struct rte_eth_dev *eth_dev, uint32_t node_id,
2476 struct rte_tm_error *error)
2478 return nix_tm_node_suspend_resume(eth_dev, node_id, error, false);
2482 otx2_nix_tm_hierarchy_commit(struct rte_eth_dev *eth_dev,
2484 struct rte_tm_error *error)
2486 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2487 struct otx2_nix_tm_node *tm_node;
2488 uint32_t leaf_cnt = 0;
2491 if (dev->tm_flags & NIX_TM_COMMITTED) {
2492 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2493 error->message = "hierarchy exists";
2497 /* Check if we have all the leaf nodes */
2498 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
2499 if (tm_node->flags & NIX_TM_NODE_USER &&
2500 tm_node->id < dev->tm_leaf_cnt)
2504 if (leaf_cnt != dev->tm_leaf_cnt) {
2505 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2506 error->message = "incomplete hierarchy";
2511 * Disable xmit will be enabled when
2512 * new topology is available.
2514 rc = nix_xmit_disable(eth_dev);
2516 otx2_err("failed to disable TX, rc=%d", rc);
2520 /* Delete default/ratelimit tree */
2521 if (dev->tm_flags & (NIX_TM_DEFAULT_TREE | NIX_TM_RATE_LIMIT_TREE)) {
2522 rc = nix_tm_free_resources(dev, NIX_TM_NODE_USER, 0, false);
2524 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2525 error->message = "failed to free default resources";
2528 dev->tm_flags &= ~(NIX_TM_DEFAULT_TREE |
2529 NIX_TM_RATE_LIMIT_TREE);
2532 /* Free up user alloc'ed resources */
2533 rc = nix_tm_free_resources(dev, NIX_TM_NODE_USER,
2534 NIX_TM_NODE_USER, true);
2536 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2537 error->message = "failed to free user resources";
2541 rc = nix_tm_alloc_resources(eth_dev, true);
2543 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2544 error->message = "alloc resources failed";
2545 /* TODO should we restore default config ? */
2547 nix_tm_free_resources(dev, 0, 0, false);
2551 error->type = RTE_TM_ERROR_TYPE_NONE;
2552 dev->tm_flags |= NIX_TM_COMMITTED;
2557 otx2_nix_tm_node_shaper_update(struct rte_eth_dev *eth_dev,
2559 uint32_t profile_id,
2560 struct rte_tm_error *error)
2562 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2563 struct otx2_nix_tm_shaper_profile *profile = NULL;
2564 struct otx2_mbox *mbox = dev->mbox;
2565 struct otx2_nix_tm_node *tm_node;
2566 struct nix_txschq_config *req;
2570 tm_node = nix_tm_node_search(dev, node_id, true);
2571 if (!tm_node || nix_tm_is_leaf(dev, tm_node->lvl)) {
2572 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2573 error->message = "invalid node";
2577 if (profile_id == tm_node->params.shaper_profile_id)
2580 if (profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE) {
2581 profile = nix_tm_shaper_profile_search(dev, profile_id);
2583 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
2584 error->message = "shaper profile ID not exist";
2589 if (profile && profile->params.packet_mode != tm_node->pkt_mode) {
2590 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
2591 error->message = "shaper profile pkt mode mismatch";
2595 tm_node->params.shaper_profile_id = profile_id;
2597 /* Nothing to do if not yet committed */
2598 if (!(dev->tm_flags & NIX_TM_COMMITTED))
2601 tm_node->flags &= ~NIX_TM_NODE_ENABLED;
2603 /* Flush the specific node with SW_XOFF */
2604 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
2605 req->lvl = tm_node->hw_lvl;
2606 k = prepare_tm_sw_xoff(tm_node, true, req->reg, req->regval);
2609 rc = send_tm_reqval(mbox, req, error);
2613 shaper_default_red_algo(dev, tm_node, profile);
2615 /* Update the PIR/CIR and clear SW XOFF */
2616 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
2617 req->lvl = tm_node->hw_lvl;
2619 k = prepare_tm_shaper_reg(tm_node, profile, req->reg, req->regval);
2621 k += prepare_tm_sw_xoff(tm_node, false, &req->reg[k], &req->regval[k]);
2624 rc = send_tm_reqval(mbox, req, error);
2626 tm_node->flags |= NIX_TM_NODE_ENABLED;
2631 otx2_nix_tm_node_parent_update(struct rte_eth_dev *eth_dev,
2632 uint32_t node_id, uint32_t new_parent_id,
2633 uint32_t priority, uint32_t weight,
2634 struct rte_tm_error *error)
2636 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2637 struct otx2_nix_tm_node *tm_node, *sibling;
2638 struct otx2_nix_tm_node *new_parent;
2639 struct nix_txschq_config *req;
2643 if (!(dev->tm_flags & NIX_TM_COMMITTED)) {
2644 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2645 error->message = "hierarchy doesn't exist";
2649 tm_node = nix_tm_node_search(dev, node_id, true);
2651 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2652 error->message = "no such node";
2656 /* Parent id valid only for non root nodes */
2657 if (tm_node->hw_lvl != dev->otx2_tm_root_lvl) {
2658 new_parent = nix_tm_node_search(dev, new_parent_id, true);
2660 error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
2661 error->message = "no such parent node";
2665 /* Current support is only for dynamic weight update */
2666 if (tm_node->parent != new_parent ||
2667 tm_node->priority != priority) {
2668 error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
2669 error->message = "only weight update supported";
2674 /* Skip if no change */
2675 if (tm_node->weight == weight)
2678 tm_node->weight = weight;
2680 /* For leaf nodes, SQ CTX needs update */
2681 if (nix_tm_is_leaf(dev, tm_node->lvl)) {
2682 /* Update SQ quantum data on the fly */
2683 rc = nix_sq_sched_data(dev, tm_node, true);
2685 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2686 error->message = "sq sched data update failed";
2690 /* XOFF Parent node */
2691 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2692 req->lvl = tm_node->parent->hw_lvl;
2693 req->num_regs = prepare_tm_sw_xoff(tm_node->parent, true,
2694 req->reg, req->regval);
2695 rc = send_tm_reqval(dev->mbox, req, error);
2699 /* XOFF this node and all other siblings */
2700 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2701 req->lvl = tm_node->hw_lvl;
2704 TAILQ_FOREACH(sibling, &dev->node_list, node) {
2705 if (sibling->parent != tm_node->parent)
2707 k += prepare_tm_sw_xoff(sibling, true, &req->reg[k],
2711 rc = send_tm_reqval(dev->mbox, req, error);
2715 /* Update new weight for current node */
2716 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2717 req->lvl = tm_node->hw_lvl;
2718 req->num_regs = prepare_tm_sched_reg(dev, tm_node,
2719 req->reg, req->regval);
2720 rc = send_tm_reqval(dev->mbox, req, error);
2724 /* XON this node and all other siblings */
2725 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2726 req->lvl = tm_node->hw_lvl;
2729 TAILQ_FOREACH(sibling, &dev->node_list, node) {
2730 if (sibling->parent != tm_node->parent)
2732 k += prepare_tm_sw_xoff(sibling, false, &req->reg[k],
2736 rc = send_tm_reqval(dev->mbox, req, error);
2740 /* XON Parent node */
2741 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2742 req->lvl = tm_node->parent->hw_lvl;
2743 req->num_regs = prepare_tm_sw_xoff(tm_node->parent, false,
2744 req->reg, req->regval);
2745 rc = send_tm_reqval(dev->mbox, req, error);
2753 otx2_nix_tm_node_stats_read(struct rte_eth_dev *eth_dev, uint32_t node_id,
2754 struct rte_tm_node_stats *stats,
2755 uint64_t *stats_mask, int clear,
2756 struct rte_tm_error *error)
2758 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2759 struct otx2_nix_tm_node *tm_node;
2764 tm_node = nix_tm_node_search(dev, node_id, true);
2766 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2767 error->message = "no such node";
2771 /* Stats support only for leaf node or TL1 root */
2772 if (nix_tm_is_leaf(dev, tm_node->lvl)) {
2773 reg = (((uint64_t)tm_node->id) << 32);
2776 addr = (int64_t *)(dev->base + NIX_LF_SQ_OP_PKTS);
2777 val = otx2_atomic64_add_nosync(reg, addr);
2780 stats->n_pkts = val - tm_node->last_pkts;
2783 addr = (int64_t *)(dev->base + NIX_LF_SQ_OP_OCTS);
2784 val = otx2_atomic64_add_nosync(reg, addr);
2787 stats->n_bytes = val - tm_node->last_bytes;
2790 tm_node->last_pkts = stats->n_pkts;
2791 tm_node->last_bytes = stats->n_bytes;
2794 *stats_mask = RTE_TM_STATS_N_PKTS | RTE_TM_STATS_N_BYTES;
2796 } else if (tm_node->hw_lvl == NIX_TXSCH_LVL_TL1) {
2797 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2798 error->message = "stats read error";
2800 /* RED Drop packets */
2801 reg = NIX_AF_TL1X_DROPPED_PACKETS(tm_node->hw_id);
2802 rc = read_tm_reg(dev->mbox, reg, &val, NIX_TXSCH_LVL_TL1);
2805 stats->leaf.n_pkts_dropped[RTE_COLOR_RED] =
2806 val - tm_node->last_pkts;
2808 /* RED Drop bytes */
2809 reg = NIX_AF_TL1X_DROPPED_BYTES(tm_node->hw_id);
2810 rc = read_tm_reg(dev->mbox, reg, &val, NIX_TXSCH_LVL_TL1);
2813 stats->leaf.n_bytes_dropped[RTE_COLOR_RED] =
2814 val - tm_node->last_bytes;
2818 tm_node->last_pkts =
2819 stats->leaf.n_pkts_dropped[RTE_COLOR_RED];
2820 tm_node->last_bytes =
2821 stats->leaf.n_bytes_dropped[RTE_COLOR_RED];
2824 *stats_mask = RTE_TM_STATS_N_PKTS_RED_DROPPED |
2825 RTE_TM_STATS_N_BYTES_RED_DROPPED;
2828 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2829 error->message = "unsupported node";
2837 const struct rte_tm_ops otx2_tm_ops = {
2838 .node_type_get = otx2_nix_tm_node_type_get,
2840 .capabilities_get = otx2_nix_tm_capa_get,
2841 .level_capabilities_get = otx2_nix_tm_level_capa_get,
2842 .node_capabilities_get = otx2_nix_tm_node_capa_get,
2844 .shaper_profile_add = otx2_nix_tm_shaper_profile_add,
2845 .shaper_profile_delete = otx2_nix_tm_shaper_profile_delete,
2847 .node_add = otx2_nix_tm_node_add,
2848 .node_delete = otx2_nix_tm_node_delete,
2849 .node_suspend = otx2_nix_tm_node_suspend,
2850 .node_resume = otx2_nix_tm_node_resume,
2851 .hierarchy_commit = otx2_nix_tm_hierarchy_commit,
2853 .node_shaper_update = otx2_nix_tm_node_shaper_update,
2854 .node_parent_update = otx2_nix_tm_node_parent_update,
2855 .node_stats_read = otx2_nix_tm_node_stats_read,
2859 nix_tm_prepare_default_tree(struct rte_eth_dev *eth_dev)
2861 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2862 uint32_t def = eth_dev->data->nb_tx_queues;
2863 struct rte_tm_node_params params;
2864 uint32_t leaf_parent, i;
2865 int rc = 0, leaf_level;
2867 /* Default params */
2868 memset(¶ms, 0, sizeof(params));
2869 params.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
2871 if (nix_tm_have_tl1_access(dev)) {
2872 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL1;
2873 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
2876 OTX2_TM_LVL_ROOT, false, ¶ms);
2879 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
2882 OTX2_TM_LVL_SCH1, false, ¶ms);
2886 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
2889 OTX2_TM_LVL_SCH2, false, ¶ms);
2893 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
2896 OTX2_TM_LVL_SCH3, false, ¶ms);
2900 rc = nix_tm_node_add_to_list(dev, def + 4, def + 3, 0,
2903 OTX2_TM_LVL_SCH4, false, ¶ms);
2907 leaf_parent = def + 4;
2908 leaf_level = OTX2_TM_LVL_QUEUE;
2910 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL2;
2911 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
2914 OTX2_TM_LVL_ROOT, false, ¶ms);
2918 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
2921 OTX2_TM_LVL_SCH1, false, ¶ms);
2925 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
2928 OTX2_TM_LVL_SCH2, false, ¶ms);
2932 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
2935 OTX2_TM_LVL_SCH3, false, ¶ms);
2939 leaf_parent = def + 3;
2940 leaf_level = OTX2_TM_LVL_SCH4;
2943 /* Add leaf nodes */
2944 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
2945 rc = nix_tm_node_add_to_list(dev, i, leaf_parent, 0,
2948 leaf_level, false, ¶ms);
2957 void otx2_nix_tm_conf_init(struct rte_eth_dev *eth_dev)
2959 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2961 TAILQ_INIT(&dev->node_list);
2962 TAILQ_INIT(&dev->shaper_profile_list);
2963 dev->tm_rate_min = 1E9; /* 1Gbps */
2966 int otx2_nix_tm_init_default(struct rte_eth_dev *eth_dev)
2968 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2969 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2970 uint16_t sq_cnt = eth_dev->data->nb_tx_queues;
2973 /* Free up all resources already held */
2974 rc = nix_tm_free_resources(dev, 0, 0, false);
2976 otx2_err("Failed to freeup existing resources,rc=%d", rc);
2980 /* Clear shaper profiles */
2981 nix_tm_clear_shaper_profiles(dev);
2982 dev->tm_flags = NIX_TM_DEFAULT_TREE;
2984 /* Disable TL1 Static Priority when VF's are enabled
2985 * as otherwise VF's TL2 reallocation will be needed
2986 * runtime to support a specific topology of PF.
2988 if (pci_dev->max_vfs)
2989 dev->tm_flags |= NIX_TM_TL1_NO_SP;
2991 rc = nix_tm_prepare_default_tree(eth_dev);
2995 rc = nix_tm_alloc_resources(eth_dev, false);
2998 dev->tm_leaf_cnt = sq_cnt;
3004 nix_tm_prepare_rate_limited_tree(struct rte_eth_dev *eth_dev)
3006 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
3007 uint32_t def = eth_dev->data->nb_tx_queues;
3008 struct rte_tm_node_params params;
3009 uint32_t leaf_parent, i, rc = 0;
3011 memset(¶ms, 0, sizeof(params));
3013 if (nix_tm_have_tl1_access(dev)) {
3014 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL1;
3015 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
3018 OTX2_TM_LVL_ROOT, false, ¶ms);
3021 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
3024 OTX2_TM_LVL_SCH1, false, ¶ms);
3027 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
3030 OTX2_TM_LVL_SCH2, false, ¶ms);
3033 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
3036 OTX2_TM_LVL_SCH3, false, ¶ms);
3039 leaf_parent = def + 3;
3041 /* Add per queue SMQ nodes */
3042 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
3043 rc = nix_tm_node_add_to_list(dev, leaf_parent + 1 + i,
3045 0, DEFAULT_RR_WEIGHT,
3053 /* Add leaf nodes */
3054 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
3055 rc = nix_tm_node_add_to_list(dev, i,
3056 leaf_parent + 1 + i, 0,
3068 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL2;
3069 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
3070 DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL2,
3071 OTX2_TM_LVL_ROOT, false, ¶ms);
3074 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
3075 DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL3,
3076 OTX2_TM_LVL_SCH1, false, ¶ms);
3079 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
3080 DEFAULT_RR_WEIGHT, NIX_TXSCH_LVL_TL4,
3081 OTX2_TM_LVL_SCH2, false, ¶ms);
3084 leaf_parent = def + 2;
3086 /* Add per queue SMQ nodes */
3087 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
3088 rc = nix_tm_node_add_to_list(dev, leaf_parent + 1 + i,
3090 0, DEFAULT_RR_WEIGHT,
3098 /* Add leaf nodes */
3099 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
3100 rc = nix_tm_node_add_to_list(dev, i, leaf_parent + 1 + i, 0,
3113 otx2_nix_tm_rate_limit_mdq(struct rte_eth_dev *eth_dev,
3114 struct otx2_nix_tm_node *tm_node,
3117 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
3118 struct otx2_nix_tm_shaper_profile profile;
3119 struct otx2_mbox *mbox = dev->mbox;
3120 volatile uint64_t *reg, *regval;
3121 struct nix_txschq_config *req;
3126 flags = tm_node->flags;
3128 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
3129 req->lvl = NIX_TXSCH_LVL_MDQ;
3131 regval = req->regval;
3134 k += prepare_tm_sw_xoff(tm_node, true, ®[k], ®val[k]);
3135 flags &= ~NIX_TM_NODE_ENABLED;
3139 if (!(flags & NIX_TM_NODE_ENABLED)) {
3140 k += prepare_tm_sw_xoff(tm_node, false, ®[k], ®val[k]);
3141 flags |= NIX_TM_NODE_ENABLED;
3144 /* Use only PIR for rate limit */
3145 memset(&profile, 0, sizeof(profile));
3146 profile.params.peak.rate = tx_rate;
3147 /* Minimum burst of ~4us Bytes of Tx */
3148 profile.params.peak.size = RTE_MAX(NIX_MAX_HW_FRS,
3149 (4ull * tx_rate) / (1E6 * 8));
3150 if (!dev->tm_rate_min || dev->tm_rate_min > tx_rate)
3151 dev->tm_rate_min = tx_rate;
3153 k += prepare_tm_shaper_reg(tm_node, &profile, ®[k], ®val[k]);
3156 rc = otx2_mbox_process(mbox);
3160 tm_node->flags = flags;
3165 otx2_nix_tm_set_queue_rate_limit(struct rte_eth_dev *eth_dev,
3166 uint16_t queue_idx, uint16_t tx_rate_mbps)
3168 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
3169 uint64_t tx_rate = tx_rate_mbps * (uint64_t)1E6;
3170 struct otx2_nix_tm_node *tm_node;
3173 /* Check for supported revisions */
3174 if (otx2_dev_is_95xx_Ax(dev) ||
3175 otx2_dev_is_96xx_Ax(dev))
3178 if (queue_idx >= eth_dev->data->nb_tx_queues)
3181 if (!(dev->tm_flags & NIX_TM_DEFAULT_TREE) &&
3182 !(dev->tm_flags & NIX_TM_RATE_LIMIT_TREE))
3185 if ((dev->tm_flags & NIX_TM_DEFAULT_TREE) &&
3186 eth_dev->data->nb_tx_queues > 1) {
3187 /* For TM topology change ethdev needs to be stopped */
3188 if (eth_dev->data->dev_started)
3192 * Disable xmit will be enabled when
3193 * new topology is available.
3195 rc = nix_xmit_disable(eth_dev);
3197 otx2_err("failed to disable TX, rc=%d", rc);
3201 rc = nix_tm_free_resources(dev, 0, 0, false);
3203 otx2_tm_dbg("failed to free default resources, rc %d",
3208 rc = nix_tm_prepare_rate_limited_tree(eth_dev);
3210 otx2_tm_dbg("failed to prepare tm tree, rc=%d", rc);
3214 rc = nix_tm_alloc_resources(eth_dev, true);
3216 otx2_tm_dbg("failed to allocate tm tree, rc=%d", rc);
3220 dev->tm_flags &= ~NIX_TM_DEFAULT_TREE;
3221 dev->tm_flags |= NIX_TM_RATE_LIMIT_TREE;
3224 tm_node = nix_tm_node_search(dev, queue_idx, false);
3226 /* check if we found a valid leaf node */
3228 !nix_tm_is_leaf(dev, tm_node->lvl) ||
3230 tm_node->parent->hw_id == UINT32_MAX)
3233 return otx2_nix_tm_rate_limit_mdq(eth_dev, tm_node->parent, tx_rate);
3235 otx2_tm_dbg("Unsupported TM tree 0x%0x", dev->tm_flags);
3240 otx2_nix_tm_ops_get(struct rte_eth_dev *eth_dev, void *arg)
3242 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
3247 /* Check for supported revisions */
3248 if (otx2_dev_is_95xx_Ax(dev) ||
3249 otx2_dev_is_96xx_Ax(dev))
3252 *(const void **)arg = &otx2_tm_ops;
3258 otx2_nix_tm_fini(struct rte_eth_dev *eth_dev)
3260 struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
3263 /* Xmit is assumed to be disabled */
3264 /* Free up resources already held */
3265 rc = nix_tm_free_resources(dev, 0, 0, false);
3267 otx2_err("Failed to freeup existing resources,rc=%d", rc);
3271 /* Clear shaper profiles */
3272 nix_tm_clear_shaper_profiles(dev);
3279 otx2_nix_tm_get_leaf_data(struct otx2_eth_dev *dev, uint16_t sq,
3280 uint32_t *rr_quantum, uint16_t *smq)
3282 struct otx2_nix_tm_node *tm_node;
3285 /* 0..sq_cnt-1 are leaf nodes */
3286 if (sq >= dev->tm_leaf_cnt)
3289 /* Search for internal node first */
3290 tm_node = nix_tm_node_search(dev, sq, false);
3292 tm_node = nix_tm_node_search(dev, sq, true);
3294 /* Check if we found a valid leaf node */
3295 if (!tm_node || !nix_tm_is_leaf(dev, tm_node->lvl) ||
3296 !tm_node->parent || tm_node->parent->hw_id == UINT32_MAX) {
3300 /* Get SMQ Id of leaf node's parent */
3301 *smq = tm_node->parent->hw_id;
3302 *rr_quantum = NIX_TM_WEIGHT_TO_RR_QUANTUM(tm_node->weight);
3304 rc = nix_smq_xoff(dev, tm_node->parent, false);
3307 tm_node->flags |= NIX_TM_NODE_ENABLED;