drivers/net/octeontx2/otx2_tm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <rte_malloc.h>
   6
   7 #include "otx2_ethdev.h"
   8 #include "otx2_tm.h"
   9
  10 /* Use last LVL_CNT nodes as default nodes */
  11 #define NIX_DEFAULT_NODE_ID_START (RTE_TM_NODE_ID_NULL - NIX_TXSCH_LVL_CNT)
  12
  13 enum otx2_tm_node_level {
  14         OTX2_TM_LVL_ROOT = 0,
  15         OTX2_TM_LVL_SCH1,
  16         OTX2_TM_LVL_SCH2,
  17         OTX2_TM_LVL_SCH3,
  18         OTX2_TM_LVL_SCH4,
  19         OTX2_TM_LVL_QUEUE,
  20         OTX2_TM_LVL_MAX,
  21 };
  22
  23 static inline
  24 uint64_t shaper2regval(struct shaper_params *shaper)
  25 {
  26         return (shaper->burst_exponent << 37) | (shaper->burst_mantissa << 29) |
  27                 (shaper->div_exp << 13) | (shaper->exponent << 9) |
  28                 (shaper->mantissa << 1);
  29 }
  30
  31 static int
  32 nix_get_link(struct otx2_eth_dev *dev)
  33 {
  34         int link = 13 /* SDP */;
  35         uint16_t lmac_chan;
  36         uint16_t map;
  37
  38         lmac_chan = dev->tx_chan_base;
  39
  40         /* CGX lmac link */
  41         if (lmac_chan >= 0x800) {
  42                 map = lmac_chan & 0x7FF;
  43                 link = 4 * ((map >> 8) & 0xF) + ((map >> 4) & 0xF);
  44         } else if (lmac_chan < 0x700) {
  45                 /* LBK channel */
  46                 link = 12;
  47         }
  48
  49         return link;
  50 }
  51
  52 static uint8_t
  53 nix_get_relchan(struct otx2_eth_dev *dev)
  54 {
  55         return dev->tx_chan_base & 0xff;
  56 }
  57
  58 static bool
  59 nix_tm_have_tl1_access(struct otx2_eth_dev *dev)
  60 {
  61         bool is_lbk = otx2_dev_is_lbk(dev);
  62         return otx2_dev_is_pf(dev) && !otx2_dev_is_Ax(dev) &&
  63                 !is_lbk && !dev->maxvf;
  64 }
  65
  66 static int
  67 find_prio_anchor(struct otx2_eth_dev *dev, uint32_t node_id)
  68 {
  69         struct otx2_nix_tm_node *child_node;
  70
  71         TAILQ_FOREACH(child_node, &dev->node_list, node) {
  72                 if (!child_node->parent)
  73                         continue;
  74                 if (!(child_node->parent->id == node_id))
  75                         continue;
  76                 if (child_node->priority == child_node->parent->rr_prio)
  77                         continue;
  78                 return child_node->hw_id - child_node->priority;
  79         }
  80         return 0;
  81 }
  82
  83
  84 static struct otx2_nix_tm_shaper_profile *
  85 nix_tm_shaper_profile_search(struct otx2_eth_dev *dev, uint32_t shaper_id)
  86 {
  87         struct otx2_nix_tm_shaper_profile *tm_shaper_profile;
  88
  89         TAILQ_FOREACH(tm_shaper_profile, &dev->shaper_profile_list, shaper) {
  90                 if (tm_shaper_profile->shaper_profile_id == shaper_id)
  91                         return tm_shaper_profile;
  92         }
  93         return NULL;
  94 }
  95
  96 static inline uint64_t
  97 shaper_rate_to_nix(uint64_t cclk_hz, uint64_t cclk_ticks,
  98                    uint64_t value, uint64_t *exponent_p,
  99                    uint64_t *mantissa_p, uint64_t *div_exp_p)
 100 {
 101         uint64_t div_exp, exponent, mantissa;
 102
 103         /* Boundary checks */
 104         if (value < MIN_SHAPER_RATE(cclk_hz, cclk_ticks) ||
 105             value > MAX_SHAPER_RATE(cclk_hz, cclk_ticks))
 106                 return 0;
 107
 108         if (value <= SHAPER_RATE(cclk_hz, cclk_ticks, 0, 0, 0)) {
 109                 /* Calculate rate div_exp and mantissa using
 110                  * the following formula:
 111                  *
 112                  * value = (cclk_hz * (256 + mantissa)
 113                  *              / ((cclk_ticks << div_exp) * 256)
 114                  */
 115                 div_exp = 0;
 116                 exponent = 0;
 117                 mantissa = MAX_RATE_MANTISSA;
 118
 119                 while (value < (cclk_hz / (cclk_ticks << div_exp)))
 120                         div_exp += 1;
 121
 122                 while (value <
 123                        ((cclk_hz * (256 + mantissa)) /
 124                         ((cclk_ticks << div_exp) * 256)))
 125                         mantissa -= 1;
 126         } else {
 127                 /* Calculate rate exponent and mantissa using
 128                  * the following formula:
 129                  *
 130                  * value = (cclk_hz * ((256 + mantissa) << exponent)
 131                  *              / (cclk_ticks * 256)
 132                  *
 133                  */
 134                 div_exp = 0;
 135                 exponent = MAX_RATE_EXPONENT;
 136                 mantissa = MAX_RATE_MANTISSA;
 137
 138                 while (value < (cclk_hz * (1 << exponent)) / cclk_ticks)
 139                         exponent -= 1;
 140
 141                 while (value < (cclk_hz * ((256 + mantissa) << exponent)) /
 142                        (cclk_ticks * 256))
 143                         mantissa -= 1;
 144         }
 145
 146         if (div_exp > MAX_RATE_DIV_EXP ||
 147             exponent > MAX_RATE_EXPONENT || mantissa > MAX_RATE_MANTISSA)
 148                 return 0;
 149
 150         if (div_exp_p)
 151                 *div_exp_p = div_exp;
 152         if (exponent_p)
 153                 *exponent_p = exponent;
 154         if (mantissa_p)
 155                 *mantissa_p = mantissa;
 156
 157         /* Calculate real rate value */
 158         return SHAPER_RATE(cclk_hz, cclk_ticks, exponent, mantissa, div_exp);
 159 }
 160
 161 static inline uint64_t
 162 lx_shaper_rate_to_nix(uint64_t cclk_hz, uint32_t hw_lvl,
 163                       uint64_t value, uint64_t *exponent,
 164                       uint64_t *mantissa, uint64_t *div_exp)
 165 {
 166         if (hw_lvl == NIX_TXSCH_LVL_TL1)
 167                 return shaper_rate_to_nix(cclk_hz, L1_TIME_WHEEL_CCLK_TICKS,
 168                                           value, exponent, mantissa, div_exp);
 169         else
 170                 return shaper_rate_to_nix(cclk_hz, LX_TIME_WHEEL_CCLK_TICKS,
 171                                           value, exponent, mantissa, div_exp);
 172 }
 173
 174 static inline uint64_t
 175 shaper_burst_to_nix(uint64_t value, uint64_t *exponent_p,
 176                     uint64_t *mantissa_p)
 177 {
 178         uint64_t exponent, mantissa;
 179
 180         if (value < MIN_SHAPER_BURST || value > MAX_SHAPER_BURST)
 181                 return 0;
 182
 183         /* Calculate burst exponent and mantissa using
 184          * the following formula:
 185          *
 186          * value = (((256 + mantissa) << (exponent + 1)
 187          / 256)
 188          *
 189          */
 190         exponent = MAX_BURST_EXPONENT;
 191         mantissa = MAX_BURST_MANTISSA;
 192
 193         while (value < (1ull << (exponent + 1)))
 194                 exponent -= 1;
 195
 196         while (value < ((256 + mantissa) << (exponent + 1)) / 256)
 197                 mantissa -= 1;
 198
 199         if (exponent > MAX_BURST_EXPONENT || mantissa > MAX_BURST_MANTISSA)
 200                 return 0;
 201
 202         if (exponent_p)
 203                 *exponent_p = exponent;
 204         if (mantissa_p)
 205                 *mantissa_p = mantissa;
 206
 207         return SHAPER_BURST(exponent, mantissa);
 208 }
 209
 210 static int
 211 configure_shaper_cir_pir_reg(struct otx2_eth_dev *dev,
 212                              struct otx2_nix_tm_node *tm_node,
 213                              struct shaper_params *cir,
 214                              struct shaper_params *pir)
 215 {
 216         uint32_t shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
 217         struct otx2_nix_tm_shaper_profile *shaper_profile = NULL;
 218         struct rte_tm_shaper_params *param;
 219
 220         shaper_profile_id = tm_node->params.shaper_profile_id;
 221
 222         shaper_profile = nix_tm_shaper_profile_search(dev, shaper_profile_id);
 223         if (shaper_profile) {
 224                 param = &shaper_profile->profile;
 225                 /* Calculate CIR exponent and mantissa */
 226                 if (param->committed.rate)
 227                         cir->rate = lx_shaper_rate_to_nix(CCLK_HZ,
 228                                                           tm_node->hw_lvl_id,
 229                                                           param->committed.rate,
 230                                                           &cir->exponent,
 231                                                           &cir->mantissa,
 232                                                           &cir->div_exp);
 233
 234                 /* Calculate PIR exponent and mantissa */
 235                 if (param->peak.rate)
 236                         pir->rate = lx_shaper_rate_to_nix(CCLK_HZ,
 237                                                           tm_node->hw_lvl_id,
 238                                                           param->peak.rate,
 239                                                           &pir->exponent,
 240                                                           &pir->mantissa,
 241                                                           &pir->div_exp);
 242
 243                 /* Calculate CIR burst exponent and mantissa */
 244                 if (param->committed.size)
 245                         cir->burst = shaper_burst_to_nix(param->committed.size,
 246                                                          &cir->burst_exponent,
 247                                                          &cir->burst_mantissa);
 248
 249                 /* Calculate PIR burst exponent and mantissa */
 250                 if (param->peak.size)
 251                         pir->burst = shaper_burst_to_nix(param->peak.size,
 252                                                          &pir->burst_exponent,
 253                                                          &pir->burst_mantissa);
 254         }
 255
 256         return 0;
 257 }
 258
 259 static int
 260 send_tm_reqval(struct otx2_mbox *mbox, struct nix_txschq_config *req)
 261 {
 262         int rc;
 263
 264         if (req->num_regs > MAX_REGS_PER_MBOX_MSG)
 265                 return -ERANGE;
 266
 267         rc = otx2_mbox_process(mbox);
 268         if (rc)
 269                 return rc;
 270
 271         req->num_regs = 0;
 272         return 0;
 273 }
 274
 275 static int
 276 populate_tm_registers(struct otx2_eth_dev *dev,
 277                       struct otx2_nix_tm_node *tm_node)
 278 {
 279         uint64_t strict_schedul_prio, rr_prio;
 280         struct otx2_mbox *mbox = dev->mbox;
 281         volatile uint64_t *reg, *regval;
 282         uint64_t parent = 0, child = 0;
 283         struct shaper_params cir, pir;
 284         struct nix_txschq_config *req;
 285         uint64_t rr_quantum;
 286         uint32_t hw_lvl;
 287         uint32_t schq;
 288         int rc;
 289
 290         memset(&cir, 0, sizeof(cir));
 291         memset(&pir, 0, sizeof(pir));
 292
 293         /* Skip leaf nodes */
 294         if (tm_node->hw_lvl_id == NIX_TXSCH_LVL_CNT)
 295                 return 0;
 296
 297         /* Root node will not have a parent node */
 298         if (tm_node->hw_lvl_id == dev->otx2_tm_root_lvl)
 299                 parent = tm_node->parent_hw_id;
 300         else
 301                 parent = tm_node->parent->hw_id;
 302
 303         /* Do we need this trigger to configure TL1 */
 304         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
 305             tm_node->hw_lvl_id == dev->otx2_tm_root_lvl) {
 306                 schq = parent;
 307                 /*
 308                  * Default config for TL1.
 309                  * For VF this is always ignored.
 310                  */
 311
 312                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 313                 req->lvl = NIX_TXSCH_LVL_TL1;
 314
 315                 /* Set DWRR quantum */
 316                 req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq);
 317                 req->regval[0] = TXSCH_TL1_DFLT_RR_QTM;
 318                 req->num_regs++;
 319
 320                 req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq);
 321                 req->regval[1] = (TXSCH_TL1_DFLT_RR_PRIO << 1);
 322                 req->num_regs++;
 323
 324                 req->reg[2] = NIX_AF_TL1X_CIR(schq);
 325                 req->regval[2] = 0;
 326                 req->num_regs++;
 327
 328                 rc = send_tm_reqval(mbox, req);
 329                 if (rc)
 330                         goto error;
 331         }
 332
 333         if (tm_node->hw_lvl_id != NIX_TXSCH_LVL_SMQ)
 334                 child = find_prio_anchor(dev, tm_node->id);
 335
 336         rr_prio = tm_node->rr_prio;
 337         hw_lvl = tm_node->hw_lvl_id;
 338         strict_schedul_prio = tm_node->priority;
 339         schq = tm_node->hw_id;
 340         rr_quantum = (tm_node->weight * NIX_TM_RR_QUANTUM_MAX) /
 341                 MAX_SCHED_WEIGHT;
 342
 343         configure_shaper_cir_pir_reg(dev, tm_node, &cir, &pir);
 344
 345         otx2_tm_dbg("Configure node %p, lvl %u hw_lvl %u, id %u, hw_id %u,"
 346                      "parent_hw_id %" PRIx64 ", pir %" PRIx64 ", cir %" PRIx64,
 347                      tm_node, tm_node->level_id, hw_lvl,
 348                      tm_node->id, schq, parent, pir.rate, cir.rate);
 349
 350         rc = -EFAULT;
 351
 352         switch (hw_lvl) {
 353         case NIX_TXSCH_LVL_SMQ:
 354                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 355                 req->lvl = hw_lvl;
 356                 reg = req->reg;
 357                 regval = req->regval;
 358                 req->num_regs = 0;
 359
 360                 /* Set xoff which will be cleared later */
 361                 *reg++ = NIX_AF_SMQX_CFG(schq);
 362                 *regval++ = BIT_ULL(50) | ((uint64_t)NIX_MAX_VTAG_INS << 36) |
 363                                 (NIX_MAX_HW_FRS << 8) | NIX_MIN_HW_FRS;
 364                 req->num_regs++;
 365                 *reg++ = NIX_AF_MDQX_PARENT(schq);
 366                 *regval++ = parent << 16;
 367                 req->num_regs++;
 368                 *reg++ = NIX_AF_MDQX_SCHEDULE(schq);
 369                 *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 370                 req->num_regs++;
 371                 if (pir.rate && pir.burst) {
 372                         *reg++ = NIX_AF_MDQX_PIR(schq);
 373                         *regval++ = shaper2regval(&pir) | 1;
 374                         req->num_regs++;
 375                 }
 376
 377                 if (cir.rate && cir.burst) {
 378                         *reg++ = NIX_AF_MDQX_CIR(schq);
 379                         *regval++ = shaper2regval(&cir) | 1;
 380                         req->num_regs++;
 381                 }
 382
 383                 rc = send_tm_reqval(mbox, req);
 384                 if (rc)
 385                         goto error;
 386                 break;
 387         case NIX_TXSCH_LVL_TL4:
 388                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 389                 req->lvl = hw_lvl;
 390                 req->num_regs = 0;
 391                 reg = req->reg;
 392                 regval = req->regval;
 393
 394                 *reg++ = NIX_AF_TL4X_PARENT(schq);
 395                 *regval++ = parent << 16;
 396                 req->num_regs++;
 397                 *reg++ = NIX_AF_TL4X_TOPOLOGY(schq);
 398                 *regval++ = (child << 32) | (rr_prio << 1);
 399                 req->num_regs++;
 400                 *reg++ = NIX_AF_TL4X_SCHEDULE(schq);
 401                 *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 402                 req->num_regs++;
 403                 if (pir.rate && pir.burst) {
 404                         *reg++ = NIX_AF_TL4X_PIR(schq);
 405                         *regval++ = shaper2regval(&pir) | 1;
 406                         req->num_regs++;
 407                 }
 408                 if (cir.rate && cir.burst) {
 409                         *reg++ = NIX_AF_TL4X_CIR(schq);
 410                         *regval++ = shaper2regval(&cir) | 1;
 411                         req->num_regs++;
 412                 }
 413                 /* Configure TL4 to send to SDP channel instead of CGX/LBK */
 414                 if (otx2_dev_is_sdp(dev)) {
 415                         *reg++ = NIX_AF_TL4X_SDP_LINK_CFG(schq);
 416                         *regval++ = BIT_ULL(12);
 417                         req->num_regs++;
 418                 }
 419
 420                 rc = send_tm_reqval(mbox, req);
 421                 if (rc)
 422                         goto error;
 423                 break;
 424         case NIX_TXSCH_LVL_TL3:
 425                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 426                 req->lvl = hw_lvl;
 427                 req->num_regs = 0;
 428                 reg = req->reg;
 429                 regval = req->regval;
 430
 431                 *reg++ = NIX_AF_TL3X_PARENT(schq);
 432                 *regval++ = parent << 16;
 433                 req->num_regs++;
 434                 *reg++ = NIX_AF_TL3X_TOPOLOGY(schq);
 435                 *regval++ = (child << 32) | (rr_prio << 1);
 436                 req->num_regs++;
 437                 *reg++ = NIX_AF_TL3X_SCHEDULE(schq);
 438                 *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 439                 req->num_regs++;
 440                 if (pir.rate && pir.burst) {
 441                         *reg++ = NIX_AF_TL3X_PIR(schq);
 442                         *regval++ = shaper2regval(&pir) | 1;
 443                         req->num_regs++;
 444                 }
 445                 if (cir.rate && cir.burst) {
 446                         *reg++ = NIX_AF_TL3X_CIR(schq);
 447                         *regval++ = shaper2regval(&cir) | 1;
 448                         req->num_regs++;
 449                 }
 450
 451                 rc = send_tm_reqval(mbox, req);
 452                 if (rc)
 453                         goto error;
 454                 break;
 455         case NIX_TXSCH_LVL_TL2:
 456                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 457                 req->lvl = hw_lvl;
 458                 req->num_regs = 0;
 459                 reg = req->reg;
 460                 regval = req->regval;
 461
 462                 *reg++ = NIX_AF_TL2X_PARENT(schq);
 463                 *regval++ = parent << 16;
 464                 req->num_regs++;
 465                 *reg++ = NIX_AF_TL2X_TOPOLOGY(schq);
 466                 *regval++ = (child << 32) | (rr_prio << 1);
 467                 req->num_regs++;
 468                 *reg++ = NIX_AF_TL2X_SCHEDULE(schq);
 469                 if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2)
 470                         *regval++ = (1 << 24) | rr_quantum;
 471                 else
 472                         *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 473                 req->num_regs++;
 474                 if (!otx2_dev_is_sdp(dev)) {
 475                         *reg++ = NIX_AF_TL3_TL2X_LINKX_CFG(schq,
 476                                                 nix_get_link(dev));
 477                         *regval++ = BIT_ULL(12) | nix_get_relchan(dev);
 478                         req->num_regs++;
 479                 }
 480                 if (pir.rate && pir.burst) {
 481                         *reg++ = NIX_AF_TL2X_PIR(schq);
 482                         *regval++ = shaper2regval(&pir) | 1;
 483                         req->num_regs++;
 484                 }
 485                 if (cir.rate && cir.burst) {
 486                         *reg++ = NIX_AF_TL2X_CIR(schq);
 487                         *regval++ = shaper2regval(&cir) | 1;
 488                         req->num_regs++;
 489                 }
 490
 491                 rc = send_tm_reqval(mbox, req);
 492                 if (rc)
 493                         goto error;
 494                 break;
 495         case NIX_TXSCH_LVL_TL1:
 496                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 497                 req->lvl = hw_lvl;
 498                 req->num_regs = 0;
 499                 reg = req->reg;
 500                 regval = req->regval;
 501
 502                 *reg++ = NIX_AF_TL1X_SCHEDULE(schq);
 503                 *regval++ = rr_quantum;
 504                 req->num_regs++;
 505                 *reg++ = NIX_AF_TL1X_TOPOLOGY(schq);
 506                 *regval++ = (child << 32) | (rr_prio << 1 /*RR_PRIO*/);
 507                 req->num_regs++;
 508                 if (cir.rate && cir.burst) {
 509                         *reg++ = NIX_AF_TL1X_CIR(schq);
 510                         *regval++ = shaper2regval(&cir) | 1;
 511                         req->num_regs++;
 512                 }
 513
 514                 rc = send_tm_reqval(mbox, req);
 515                 if (rc)
 516                         goto error;
 517                 break;
 518         }
 519
 520         return 0;
 521 error:
 522         otx2_err("Txschq cfg request failed for node %p, rc=%d", tm_node, rc);
 523         return rc;
 524 }
 525
 526
 527 static int
 528 nix_tm_txsch_reg_config(struct otx2_eth_dev *dev)
 529 {
 530         struct otx2_nix_tm_node *tm_node;
 531         uint32_t lvl;
 532         int rc = 0;
 533
 534         for (lvl = 0; lvl < (uint32_t)dev->otx2_tm_root_lvl + 1; lvl++) {
 535                 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 536                         if (tm_node->hw_lvl_id == lvl) {
 537                                 rc = populate_tm_registers(dev, tm_node);
 538                                 if (rc)
 539                                         goto exit;
 540                         }
 541                 }
 542         }
 543 exit:
 544         return rc;
 545 }
 546
 547 static struct otx2_nix_tm_node *
 548 nix_tm_node_search(struct otx2_eth_dev *dev,
 549                    uint32_t node_id, bool user)
 550 {
 551         struct otx2_nix_tm_node *tm_node;
 552
 553         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 554                 if (tm_node->id == node_id &&
 555                     (user == !!(tm_node->flags & NIX_TM_NODE_USER)))
 556                         return tm_node;
 557         }
 558         return NULL;
 559 }
 560
 561 static uint32_t
 562 check_rr(struct otx2_eth_dev *dev, uint32_t priority, uint32_t parent_id)
 563 {
 564         struct otx2_nix_tm_node *tm_node;
 565         uint32_t rr_num = 0;
 566
 567         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 568                 if (!tm_node->parent)
 569                         continue;
 570
 571                 if (!(tm_node->parent->id == parent_id))
 572                         continue;
 573
 574                 if (tm_node->priority == priority)
 575                         rr_num++;
 576         }
 577         return rr_num;
 578 }
 579
 580 static int
 581 nix_tm_update_parent_info(struct otx2_eth_dev *dev)
 582 {
 583         struct otx2_nix_tm_node *tm_node_child;
 584         struct otx2_nix_tm_node *tm_node;
 585         struct otx2_nix_tm_node *parent;
 586         uint32_t rr_num = 0;
 587         uint32_t priority;
 588
 589         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 590                 if (!tm_node->parent)
 591                         continue;
 592                 /* Count group of children of same priority i.e are RR */
 593                 parent = tm_node->parent;
 594                 priority = tm_node->priority;
 595                 rr_num = check_rr(dev, priority, parent->id);
 596
 597                 /* Assuming that multiple RR groups are
 598                  * not configured based on capability.
 599                  */
 600                 if (rr_num > 1) {
 601                         parent->rr_prio = priority;
 602                         parent->rr_num = rr_num;
 603                 }
 604
 605                 /* Find out static priority children that are not in RR */
 606                 TAILQ_FOREACH(tm_node_child, &dev->node_list, node) {
 607                         if (!tm_node_child->parent)
 608                                 continue;
 609                         if (parent->id != tm_node_child->parent->id)
 610                                 continue;
 611                         if (parent->max_prio == UINT32_MAX &&
 612                             tm_node_child->priority != parent->rr_prio)
 613                                 parent->max_prio = 0;
 614
 615                         if (parent->max_prio < tm_node_child->priority &&
 616                             parent->rr_prio != tm_node_child->priority)
 617                                 parent->max_prio = tm_node_child->priority;
 618                 }
 619         }
 620
 621         return 0;
 622 }
 623
 624 static int
 625 nix_tm_node_add_to_list(struct otx2_eth_dev *dev, uint32_t node_id,
 626                         uint32_t parent_node_id, uint32_t priority,
 627                         uint32_t weight, uint16_t hw_lvl_id,
 628                         uint16_t level_id, bool user,
 629                         struct rte_tm_node_params *params)
 630 {
 631         struct otx2_nix_tm_shaper_profile *shaper_profile;
 632         struct otx2_nix_tm_node *tm_node, *parent_node;
 633         uint32_t shaper_profile_id;
 634
 635         shaper_profile_id = params->shaper_profile_id;
 636         shaper_profile = nix_tm_shaper_profile_search(dev, shaper_profile_id);
 637
 638         parent_node = nix_tm_node_search(dev, parent_node_id, user);
 639
 640         tm_node = rte_zmalloc("otx2_nix_tm_node",
 641                               sizeof(struct otx2_nix_tm_node), 0);
 642         if (!tm_node)
 643                 return -ENOMEM;
 644
 645         tm_node->level_id = level_id;
 646         tm_node->hw_lvl_id = hw_lvl_id;
 647
 648         tm_node->id = node_id;
 649         tm_node->priority = priority;
 650         tm_node->weight = weight;
 651         tm_node->rr_prio = 0xf;
 652         tm_node->max_prio = UINT32_MAX;
 653         tm_node->hw_id = UINT32_MAX;
 654         tm_node->flags = 0;
 655         if (user)
 656                 tm_node->flags = NIX_TM_NODE_USER;
 657         rte_memcpy(&tm_node->params, params, sizeof(struct rte_tm_node_params));
 658
 659         if (shaper_profile)
 660                 shaper_profile->reference_count++;
 661         tm_node->parent = parent_node;
 662         tm_node->parent_hw_id = UINT32_MAX;
 663
 664         TAILQ_INSERT_TAIL(&dev->node_list, tm_node, node);
 665
 666         return 0;
 667 }
 668
 669 static int
 670 nix_tm_clear_shaper_profiles(struct otx2_eth_dev *dev)
 671 {
 672         struct otx2_nix_tm_shaper_profile *shaper_profile;
 673
 674         while ((shaper_profile = TAILQ_FIRST(&dev->shaper_profile_list))) {
 675                 if (shaper_profile->reference_count)
 676                         otx2_tm_dbg("Shaper profile %u has non zero references",
 677                                     shaper_profile->shaper_profile_id);
 678                 TAILQ_REMOVE(&dev->shaper_profile_list, shaper_profile, shaper);
 679                 rte_free(shaper_profile);
 680         }
 681
 682         return 0;
 683 }
 684
 685 static int
 686 nix_smq_xoff(struct otx2_eth_dev *dev, uint16_t smq, bool enable)
 687 {
 688         struct otx2_mbox *mbox = dev->mbox;
 689         struct nix_txschq_config *req;
 690
 691         req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 692         req->lvl = NIX_TXSCH_LVL_SMQ;
 693         req->num_regs = 1;
 694
 695         req->reg[0] = NIX_AF_SMQX_CFG(smq);
 696         /* Unmodified fields */
 697         req->regval[0] = ((uint64_t)NIX_MAX_VTAG_INS << 36) |
 698                                 (NIX_MAX_HW_FRS << 8) | NIX_MIN_HW_FRS;
 699
 700         if (enable)
 701                 req->regval[0] |= BIT_ULL(50) | BIT_ULL(49);
 702         else
 703                 req->regval[0] |= 0;
 704
 705         return otx2_mbox_process(mbox);
 706 }
 707
 708 int
 709 otx2_nix_sq_sqb_aura_fc(void *__txq, bool enable)
 710 {
 711         struct otx2_eth_txq *txq = __txq;
 712         struct npa_aq_enq_req *req;
 713         struct npa_aq_enq_rsp *rsp;
 714         struct otx2_npa_lf *lf;
 715         struct otx2_mbox *mbox;
 716         uint64_t aura_handle;
 717         int rc;
 718
 719         lf = otx2_npa_lf_obj_get();
 720         if (!lf)
 721                 return -EFAULT;
 722         mbox = lf->mbox;
 723         /* Set/clear sqb aura fc_ena */
 724         aura_handle = txq->sqb_pool->pool_id;
 725         req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
 726
 727         req->aura_id = npa_lf_aura_handle_to_aura(aura_handle);
 728         req->ctype = NPA_AQ_CTYPE_AURA;
 729         req->op = NPA_AQ_INSTOP_WRITE;
 730         /* Below is not needed for aura writes but AF driver needs it */
 731         /* AF will translate to associated poolctx */
 732         req->aura.pool_addr = req->aura_id;
 733
 734         req->aura.fc_ena = enable;
 735         req->aura_mask.fc_ena = 1;
 736
 737         rc = otx2_mbox_process(mbox);
 738         if (rc)
 739                 return rc;
 740
 741         /* Read back npa aura ctx */
 742         req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
 743
 744         req->aura_id = npa_lf_aura_handle_to_aura(aura_handle);
 745         req->ctype = NPA_AQ_CTYPE_AURA;
 746         req->op = NPA_AQ_INSTOP_READ;
 747
 748         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
 749         if (rc)
 750                 return rc;
 751
 752         /* Init when enabled as there might be no triggers */
 753         if (enable)
 754                 *(volatile uint64_t *)txq->fc_mem = rsp->aura.count;
 755         else
 756                 *(volatile uint64_t *)txq->fc_mem = txq->nb_sqb_bufs;
 757         /* Sync write barrier */
 758         rte_wmb();
 759
 760         return 0;
 761 }
 762
 763 static void
 764 nix_txq_flush_sq_spin(struct otx2_eth_txq *txq)
 765 {
 766         uint16_t sqb_cnt, head_off, tail_off;
 767         struct otx2_eth_dev *dev = txq->dev;
 768         uint16_t sq = txq->sq;
 769         uint64_t reg, val;
 770         int64_t *regaddr;
 771
 772         while (true) {
 773                 reg = ((uint64_t)sq << 32);
 774                 regaddr = (int64_t *)(dev->base + NIX_LF_SQ_OP_PKTS);
 775                 val = otx2_atomic64_add_nosync(reg, regaddr);
 776
 777                 regaddr = (int64_t *)(dev->base + NIX_LF_SQ_OP_STATUS);
 778                 val = otx2_atomic64_add_nosync(reg, regaddr);
 779                 sqb_cnt = val & 0xFFFF;
 780                 head_off = (val >> 20) & 0x3F;
 781                 tail_off = (val >> 28) & 0x3F;
 782
 783                 /* SQ reached quiescent state */
 784                 if (sqb_cnt <= 1 && head_off == tail_off &&
 785                     (*txq->fc_mem == txq->nb_sqb_bufs)) {
 786                         break;
 787                 }
 788
 789                 rte_pause();
 790         }
 791 }
 792
 793 int
 794 otx2_nix_tm_sw_xoff(void *__txq, bool dev_started)
 795 {
 796         struct otx2_eth_txq *txq = __txq;
 797         struct otx2_eth_dev *dev = txq->dev;
 798         struct otx2_mbox *mbox = dev->mbox;
 799         struct nix_aq_enq_req *req;
 800         struct nix_aq_enq_rsp *rsp;
 801         uint16_t smq;
 802         int rc;
 803
 804         /* Get smq from sq */
 805         req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
 806         req->qidx = txq->sq;
 807         req->ctype = NIX_AQ_CTYPE_SQ;
 808         req->op = NIX_AQ_INSTOP_READ;
 809         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
 810         if (rc) {
 811                 otx2_err("Failed to get smq, rc=%d", rc);
 812                 return -EIO;
 813         }
 814
 815         /* Check if sq is enabled */
 816         if (!rsp->sq.ena)
 817                 return 0;
 818
 819         smq = rsp->sq.smq;
 820
 821         /* Enable CGX RXTX to drain pkts */
 822         if (!dev_started) {
 823                 rc = otx2_cgx_rxtx_start(dev);
 824                 if (rc)
 825                         return rc;
 826         }
 827
 828         rc = otx2_nix_sq_sqb_aura_fc(txq, false);
 829         if (rc < 0) {
 830                 otx2_err("Failed to disable sqb aura fc, rc=%d", rc);
 831                 goto cleanup;
 832         }
 833
 834         /* Disable smq xoff for case it was enabled earlier */
 835         rc = nix_smq_xoff(dev, smq, false);
 836         if (rc) {
 837                 otx2_err("Failed to enable smq for sq %u, rc=%d", txq->sq, rc);
 838                 goto cleanup;
 839         }
 840
 841         /* Wait for sq entries to be flushed */
 842         nix_txq_flush_sq_spin(txq);
 843
 844         /* Flush and enable smq xoff */
 845         rc = nix_smq_xoff(dev, smq, true);
 846         if (rc) {
 847                 otx2_err("Failed to disable smq for sq %u, rc=%d", txq->sq, rc);
 848                 return rc;
 849         }
 850
 851 cleanup:
 852         /* Restore cgx state */
 853         if (!dev_started)
 854                 rc |= otx2_cgx_rxtx_stop(dev);
 855
 856         return rc;
 857 }
 858
 859 static int
 860 nix_tm_sw_xon(struct otx2_eth_txq *txq,
 861               uint16_t smq, uint32_t rr_quantum)
 862 {
 863         struct otx2_eth_dev *dev = txq->dev;
 864         struct otx2_mbox *mbox = dev->mbox;
 865         struct nix_aq_enq_req *req;
 866         int rc;
 867
 868         otx2_tm_dbg("Enabling sq(%u)->smq(%u), rr_quantum %u",
 869                     txq->sq, txq->sq, rr_quantum);
 870         /* Set smq from sq */
 871         req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
 872         req->qidx = txq->sq;
 873         req->ctype = NIX_AQ_CTYPE_SQ;
 874         req->op = NIX_AQ_INSTOP_WRITE;
 875         req->sq.smq = smq;
 876         req->sq.smq_rr_quantum = rr_quantum;
 877         req->sq_mask.smq = ~req->sq_mask.smq;
 878         req->sq_mask.smq_rr_quantum = ~req->sq_mask.smq_rr_quantum;
 879
 880         rc = otx2_mbox_process(mbox);
 881         if (rc) {
 882                 otx2_err("Failed to set smq, rc=%d", rc);
 883                 return -EIO;
 884         }
 885
 886         /* Enable sqb_aura fc */
 887         rc = otx2_nix_sq_sqb_aura_fc(txq, true);
 888         if (rc < 0) {
 889                 otx2_err("Failed to enable sqb aura fc, rc=%d", rc);
 890                 return rc;
 891         }
 892
 893         /* Disable smq xoff */
 894         rc = nix_smq_xoff(dev, smq, false);
 895         if (rc) {
 896                 otx2_err("Failed to enable smq for sq %u", txq->sq);
 897                 return rc;
 898         }
 899
 900         return 0;
 901 }
 902
 903 static int
 904 nix_tm_free_resources(struct otx2_eth_dev *dev, uint32_t flags_mask,
 905                       uint32_t flags, bool hw_only)
 906 {
 907         struct otx2_nix_tm_shaper_profile *shaper_profile;
 908         struct otx2_nix_tm_node *tm_node, *next_node;
 909         struct otx2_mbox *mbox = dev->mbox;
 910         struct nix_txsch_free_req *req;
 911         uint32_t shaper_profile_id;
 912         bool skip_node = false;
 913         int rc = 0;
 914
 915         next_node = TAILQ_FIRST(&dev->node_list);
 916         while (next_node) {
 917                 tm_node = next_node;
 918                 next_node = TAILQ_NEXT(tm_node, node);
 919
 920                 /* Check for only requested nodes */
 921                 if ((tm_node->flags & flags_mask) != flags)
 922                         continue;
 923
 924                 if (nix_tm_have_tl1_access(dev) &&
 925                     tm_node->hw_lvl_id ==  NIX_TXSCH_LVL_TL1)
 926                         skip_node = true;
 927
 928                 otx2_tm_dbg("Free hwres for node %u, hwlvl %u, hw_id %u (%p)",
 929                             tm_node->id,  tm_node->hw_lvl_id,
 930                             tm_node->hw_id, tm_node);
 931                 /* Free specific HW resource if requested */
 932                 if (!skip_node && flags_mask &&
 933                     tm_node->flags & NIX_TM_NODE_HWRES) {
 934                         req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
 935                         req->flags = 0;
 936                         req->schq_lvl = tm_node->hw_lvl_id;
 937                         req->schq = tm_node->hw_id;
 938                         rc = otx2_mbox_process(mbox);
 939                         if (rc)
 940                                 break;
 941                 } else {
 942                         skip_node = false;
 943                 }
 944                 tm_node->flags &= ~NIX_TM_NODE_HWRES;
 945
 946                 /* Leave software elements if needed */
 947                 if (hw_only)
 948                         continue;
 949
 950                 shaper_profile_id = tm_node->params.shaper_profile_id;
 951                 shaper_profile =
 952                         nix_tm_shaper_profile_search(dev, shaper_profile_id);
 953                 if (shaper_profile)
 954                         shaper_profile->reference_count--;
 955
 956                 TAILQ_REMOVE(&dev->node_list, tm_node, node);
 957                 rte_free(tm_node);
 958         }
 959
 960         if (!flags_mask) {
 961                 /* Free all hw resources */
 962                 req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
 963                 req->flags = TXSCHQ_FREE_ALL;
 964
 965                 return otx2_mbox_process(mbox);
 966         }
 967
 968         return rc;
 969 }
 970
 971 static uint8_t
 972 nix_tm_copy_rsp_to_dev(struct otx2_eth_dev *dev,
 973                        struct nix_txsch_alloc_rsp *rsp)
 974 {
 975         uint16_t schq;
 976         uint8_t lvl;
 977
 978         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
 979                 for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) {
 980                         dev->txschq_list[lvl][schq] = rsp->schq_list[lvl][schq];
 981                         dev->txschq_contig_list[lvl][schq] =
 982                                 rsp->schq_contig_list[lvl][schq];
 983                 }
 984
 985                 dev->txschq[lvl] = rsp->schq[lvl];
 986                 dev->txschq_contig[lvl] = rsp->schq_contig[lvl];
 987         }
 988         return 0;
 989 }
 990
 991 static int
 992 nix_tm_assign_id_to_node(struct otx2_eth_dev *dev,
 993                          struct otx2_nix_tm_node *child,
 994                          struct otx2_nix_tm_node *parent)
 995 {
 996         uint32_t hw_id, schq_con_index, prio_offset;
 997         uint32_t l_id, schq_index;
 998
 999         otx2_tm_dbg("Assign hw id for child node %u, lvl %u, hw_lvl %u (%p)",
1000                     child->id, child->level_id, child->hw_lvl_id, child);
1001
1002         child->flags |= NIX_TM_NODE_HWRES;
1003
1004         /* Process root nodes */
1005         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
1006             child->hw_lvl_id == dev->otx2_tm_root_lvl && !parent) {
1007                 int idx = 0;
1008                 uint32_t tschq_con_index;
1009
1010                 l_id = child->hw_lvl_id;
1011                 tschq_con_index = dev->txschq_contig_index[l_id];
1012                 hw_id = dev->txschq_contig_list[l_id][tschq_con_index];
1013                 child->hw_id = hw_id;
1014                 dev->txschq_contig_index[l_id]++;
1015                 /* Update TL1 hw_id for its parent for config purpose */
1016                 idx = dev->txschq_index[NIX_TXSCH_LVL_TL1]++;
1017                 hw_id = dev->txschq_list[NIX_TXSCH_LVL_TL1][idx];
1018                 child->parent_hw_id = hw_id;
1019                 return 0;
1020         }
1021         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL1 &&
1022             child->hw_lvl_id == dev->otx2_tm_root_lvl && !parent) {
1023                 uint32_t tschq_con_index;
1024
1025                 l_id = child->hw_lvl_id;
1026                 tschq_con_index = dev->txschq_index[l_id];
1027                 hw_id = dev->txschq_list[l_id][tschq_con_index];
1028                 child->hw_id = hw_id;
1029                 dev->txschq_index[l_id]++;
1030                 return 0;
1031         }
1032
1033         /* Process children with parents */
1034         l_id = child->hw_lvl_id;
1035         schq_index = dev->txschq_index[l_id];
1036         schq_con_index = dev->txschq_contig_index[l_id];
1037
1038         if (child->priority == parent->rr_prio) {
1039                 hw_id = dev->txschq_list[l_id][schq_index];
1040                 child->hw_id = hw_id;
1041                 child->parent_hw_id = parent->hw_id;
1042                 dev->txschq_index[l_id]++;
1043         } else {
1044                 prio_offset = schq_con_index + child->priority;
1045                 hw_id = dev->txschq_contig_list[l_id][prio_offset];
1046                 child->hw_id = hw_id;
1047         }
1048         return 0;
1049 }
1050
1051 static int
1052 nix_tm_assign_hw_id(struct otx2_eth_dev *dev)
1053 {
1054         struct otx2_nix_tm_node *parent, *child;
1055         uint32_t child_hw_lvl, con_index_inc, i;
1056
1057         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--) {
1058                 TAILQ_FOREACH(parent, &dev->node_list, node) {
1059                         child_hw_lvl = parent->hw_lvl_id - 1;
1060                         if (parent->hw_lvl_id != i)
1061                                 continue;
1062                         TAILQ_FOREACH(child, &dev->node_list, node) {
1063                                 if (!child->parent)
1064                                         continue;
1065                                 if (child->parent->id != parent->id)
1066                                         continue;
1067                                 nix_tm_assign_id_to_node(dev, child, parent);
1068                         }
1069
1070                         con_index_inc = parent->max_prio + 1;
1071                         dev->txschq_contig_index[child_hw_lvl] += con_index_inc;
1072
1073                         /*
1074                          * Explicitly assign id to parent node if it
1075                          * doesn't have a parent
1076                          */
1077                         if (parent->hw_lvl_id == dev->otx2_tm_root_lvl)
1078                                 nix_tm_assign_id_to_node(dev, parent, NULL);
1079                 }
1080         }
1081         return 0;
1082 }
1083
1084 static uint8_t
1085 nix_tm_count_req_schq(struct otx2_eth_dev *dev,
1086                       struct nix_txsch_alloc_req *req, uint8_t lvl)
1087 {
1088         struct otx2_nix_tm_node *tm_node;
1089         uint8_t contig_count;
1090
1091         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1092                 if (lvl == tm_node->hw_lvl_id) {
1093                         req->schq[lvl - 1] += tm_node->rr_num;
1094                         if (tm_node->max_prio != UINT32_MAX) {
1095                                 contig_count = tm_node->max_prio + 1;
1096                                 req->schq_contig[lvl - 1] += contig_count;
1097                         }
1098                 }
1099                 if (lvl == dev->otx2_tm_root_lvl &&
1100                     dev->otx2_tm_root_lvl && lvl == NIX_TXSCH_LVL_TL2 &&
1101                     tm_node->hw_lvl_id == dev->otx2_tm_root_lvl) {
1102                         req->schq_contig[dev->otx2_tm_root_lvl]++;
1103                 }
1104         }
1105
1106         req->schq[NIX_TXSCH_LVL_TL1] = 1;
1107         req->schq_contig[NIX_TXSCH_LVL_TL1] = 0;
1108
1109         return 0;
1110 }
1111
1112 static int
1113 nix_tm_prepare_txschq_req(struct otx2_eth_dev *dev,
1114                           struct nix_txsch_alloc_req *req)
1115 {
1116         uint8_t i;
1117
1118         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--)
1119                 nix_tm_count_req_schq(dev, req, i);
1120
1121         for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
1122                 dev->txschq_index[i] = 0;
1123                 dev->txschq_contig_index[i] = 0;
1124         }
1125         return 0;
1126 }
1127
1128 static int
1129 nix_tm_send_txsch_alloc_msg(struct otx2_eth_dev *dev)
1130 {
1131         struct otx2_mbox *mbox = dev->mbox;
1132         struct nix_txsch_alloc_req *req;
1133         struct nix_txsch_alloc_rsp *rsp;
1134         int rc;
1135
1136         req = otx2_mbox_alloc_msg_nix_txsch_alloc(mbox);
1137
1138         rc = nix_tm_prepare_txschq_req(dev, req);
1139         if (rc)
1140                 return rc;
1141
1142         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
1143         if (rc)
1144                 return rc;
1145
1146         nix_tm_copy_rsp_to_dev(dev, rsp);
1147
1148         nix_tm_assign_hw_id(dev);
1149         return 0;
1150 }
1151
1152 static int
1153 nix_tm_alloc_resources(struct rte_eth_dev *eth_dev, bool xmit_enable)
1154 {
1155         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1156         struct otx2_nix_tm_node *tm_node;
1157         uint16_t sq, smq, rr_quantum;
1158         struct otx2_eth_txq *txq;
1159         int rc;
1160
1161         nix_tm_update_parent_info(dev);
1162
1163         rc = nix_tm_send_txsch_alloc_msg(dev);
1164         if (rc) {
1165                 otx2_err("TM failed to alloc tm resources=%d", rc);
1166                 return rc;
1167         }
1168
1169         rc = nix_tm_txsch_reg_config(dev);
1170         if (rc) {
1171                 otx2_err("TM failed to configure sched registers=%d", rc);
1172                 return rc;
1173         }
1174
1175         /* Enable xmit as all the topology is ready */
1176         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1177                 if (tm_node->flags & NIX_TM_NODE_ENABLED)
1178                         continue;
1179
1180                 /* Enable xmit on sq */
1181                 if (tm_node->level_id != OTX2_TM_LVL_QUEUE) {
1182                         tm_node->flags |= NIX_TM_NODE_ENABLED;
1183                         continue;
1184                 }
1185
1186                 /* Don't enable SMQ or mark as enable */
1187                 if (!xmit_enable)
1188                         continue;
1189
1190                 sq = tm_node->id;
1191                 if (sq > eth_dev->data->nb_tx_queues) {
1192                         rc = -EFAULT;
1193                         break;
1194                 }
1195
1196                 txq = eth_dev->data->tx_queues[sq];
1197
1198                 smq = tm_node->parent->hw_id;
1199                 rr_quantum = (tm_node->weight *
1200                               NIX_TM_RR_QUANTUM_MAX) / MAX_SCHED_WEIGHT;
1201
1202                 rc = nix_tm_sw_xon(txq, smq, rr_quantum);
1203                 if (rc)
1204                         break;
1205                 tm_node->flags |= NIX_TM_NODE_ENABLED;
1206         }
1207
1208         if (rc)
1209                 otx2_err("TM failed to enable xmit on sq %u, rc=%d", sq, rc);
1210
1211         return rc;
1212 }
1213
1214 static int
1215 nix_tm_prepare_default_tree(struct rte_eth_dev *eth_dev)
1216 {
1217         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1218         uint32_t def = eth_dev->data->nb_tx_queues;
1219         struct rte_tm_node_params params;
1220         uint32_t leaf_parent, i;
1221         int rc = 0;
1222
1223         /* Default params */
1224         memset(&params, 0, sizeof(params));
1225         params.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
1226
1227         if (nix_tm_have_tl1_access(dev)) {
1228                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL1;
1229                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
1230                                              DEFAULT_RR_WEIGHT,
1231                                              NIX_TXSCH_LVL_TL1,
1232                                              OTX2_TM_LVL_ROOT, false, &params);
1233                 if (rc)
1234                         goto exit;
1235                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
1236                                              DEFAULT_RR_WEIGHT,
1237                                              NIX_TXSCH_LVL_TL2,
1238                                              OTX2_TM_LVL_SCH1, false, &params);
1239                 if (rc)
1240                         goto exit;
1241
1242                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
1243                                              DEFAULT_RR_WEIGHT,
1244                                              NIX_TXSCH_LVL_TL3,
1245                                              OTX2_TM_LVL_SCH2, false, &params);
1246                 if (rc)
1247                         goto exit;
1248
1249                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
1250                                              DEFAULT_RR_WEIGHT,
1251                                              NIX_TXSCH_LVL_TL4,
1252                                              OTX2_TM_LVL_SCH3, false, &params);
1253                 if (rc)
1254                         goto exit;
1255
1256                 rc = nix_tm_node_add_to_list(dev, def + 4, def + 3, 0,
1257                                              DEFAULT_RR_WEIGHT,
1258                                              NIX_TXSCH_LVL_SMQ,
1259                                              OTX2_TM_LVL_SCH4, false, &params);
1260                 if (rc)
1261                         goto exit;
1262
1263                 leaf_parent = def + 4;
1264         } else {
1265                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL2;
1266                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
1267                                              DEFAULT_RR_WEIGHT,
1268                                              NIX_TXSCH_LVL_TL2,
1269                                              OTX2_TM_LVL_ROOT, false, &params);
1270                 if (rc)
1271                         goto exit;
1272
1273                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
1274                                              DEFAULT_RR_WEIGHT,
1275                                              NIX_TXSCH_LVL_TL3,
1276                                              OTX2_TM_LVL_SCH1, false, &params);
1277                 if (rc)
1278                         goto exit;
1279
1280                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
1281                                              DEFAULT_RR_WEIGHT,
1282                                              NIX_TXSCH_LVL_TL4,
1283                                              OTX2_TM_LVL_SCH2, false, &params);
1284                 if (rc)
1285                         goto exit;
1286
1287                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
1288                                              DEFAULT_RR_WEIGHT,
1289                                              NIX_TXSCH_LVL_SMQ,
1290                                              OTX2_TM_LVL_SCH3, false, &params);
1291                 if (rc)
1292                         goto exit;
1293
1294                 leaf_parent = def + 3;
1295         }
1296
1297         /* Add leaf nodes */
1298         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
1299                 rc = nix_tm_node_add_to_list(dev, i, leaf_parent, 0,
1300                                              DEFAULT_RR_WEIGHT,
1301                                              NIX_TXSCH_LVL_CNT,
1302                                              OTX2_TM_LVL_QUEUE, false, &params);
1303                 if (rc)
1304                         break;
1305         }
1306
1307 exit:
1308         return rc;
1309 }
1310
1311 void otx2_nix_tm_conf_init(struct rte_eth_dev *eth_dev)
1312 {
1313         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1314
1315         TAILQ_INIT(&dev->node_list);
1316         TAILQ_INIT(&dev->shaper_profile_list);
1317 }
1318
1319 int otx2_nix_tm_init_default(struct rte_eth_dev *eth_dev)
1320 {
1321         struct otx2_eth_dev  *dev = otx2_eth_pmd_priv(eth_dev);
1322         uint16_t sq_cnt = eth_dev->data->nb_tx_queues;
1323         int rc;
1324
1325         /* Free up all resources already held */
1326         rc = nix_tm_free_resources(dev, 0, 0, false);
1327         if (rc) {
1328                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
1329                 return rc;
1330         }
1331
1332         /* Clear shaper profiles */
1333         nix_tm_clear_shaper_profiles(dev);
1334         dev->tm_flags = NIX_TM_DEFAULT_TREE;
1335
1336         rc = nix_tm_prepare_default_tree(eth_dev);
1337         if (rc != 0)
1338                 return rc;
1339
1340         rc = nix_tm_alloc_resources(eth_dev, false);
1341         if (rc != 0)
1342                 return rc;
1343         dev->tm_leaf_cnt = sq_cnt;
1344
1345         return 0;
1346 }
1347
1348 int
1349 otx2_nix_tm_fini(struct rte_eth_dev *eth_dev)
1350 {
1351         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1352         int rc;
1353
1354         /* Xmit is assumed to be disabled */
1355         /* Free up resources already held */
1356         rc = nix_tm_free_resources(dev, 0, 0, false);
1357         if (rc) {
1358                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
1359                 return rc;
1360         }
1361
1362         /* Clear shaper profiles */
1363         nix_tm_clear_shaper_profiles(dev);
1364
1365         dev->tm_flags = 0;
1366         return 0;
1367 }
1368
1369 int
1370 otx2_nix_tm_get_leaf_data(struct otx2_eth_dev *dev, uint16_t sq,
1371                           uint32_t *rr_quantum, uint16_t *smq)
1372 {
1373         struct otx2_nix_tm_node *tm_node;
1374         int rc;
1375
1376         /* 0..sq_cnt-1 are leaf nodes */
1377         if (sq >= dev->tm_leaf_cnt)
1378                 return -EINVAL;
1379
1380         /* Search for internal node first */
1381         tm_node = nix_tm_node_search(dev, sq, false);
1382         if (!tm_node)
1383                 tm_node = nix_tm_node_search(dev, sq, true);
1384
1385         /* Check if we found a valid leaf node */
1386         if (!tm_node || tm_node->level_id != OTX2_TM_LVL_QUEUE ||
1387             !tm_node->parent || tm_node->parent->hw_id == UINT32_MAX) {
1388                 return -EIO;
1389         }
1390
1391         /* Get SMQ Id of leaf node's parent */
1392         *smq = tm_node->parent->hw_id;
1393         *rr_quantum = (tm_node->weight * NIX_TM_RR_QUANTUM_MAX)
1394                 / MAX_SCHED_WEIGHT;
1395
1396         rc = nix_smq_xoff(dev, *smq, false);
1397         if (rc)
1398                 return rc;
1399         tm_node->flags |= NIX_TM_NODE_ENABLED;
1400
1401         return 0;
1402 }