drivers/net/octeontx2/otx2_tm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <rte_malloc.h>
   6
   7 #include "otx2_ethdev.h"
   8 #include "otx2_tm.h"
   9
  10 /* Use last LVL_CNT nodes as default nodes */
  11 #define NIX_DEFAULT_NODE_ID_START (RTE_TM_NODE_ID_NULL - NIX_TXSCH_LVL_CNT)
  12
  13 enum otx2_tm_node_level {
  14         OTX2_TM_LVL_ROOT = 0,
  15         OTX2_TM_LVL_SCH1,
  16         OTX2_TM_LVL_SCH2,
  17         OTX2_TM_LVL_SCH3,
  18         OTX2_TM_LVL_SCH4,
  19         OTX2_TM_LVL_QUEUE,
  20         OTX2_TM_LVL_MAX,
  21 };
  22
  23 static inline
  24 uint64_t shaper2regval(struct shaper_params *shaper)
  25 {
  26         return (shaper->burst_exponent << 37) | (shaper->burst_mantissa << 29) |
  27                 (shaper->div_exp << 13) | (shaper->exponent << 9) |
  28                 (shaper->mantissa << 1);
  29 }
  30
  31 static int
  32 nix_get_link(struct otx2_eth_dev *dev)
  33 {
  34         int link = 13 /* SDP */;
  35         uint16_t lmac_chan;
  36         uint16_t map;
  37
  38         lmac_chan = dev->tx_chan_base;
  39
  40         /* CGX lmac link */
  41         if (lmac_chan >= 0x800) {
  42                 map = lmac_chan & 0x7FF;
  43                 link = 4 * ((map >> 8) & 0xF) + ((map >> 4) & 0xF);
  44         } else if (lmac_chan < 0x700) {
  45                 /* LBK channel */
  46                 link = 12;
  47         }
  48
  49         return link;
  50 }
  51
  52 static uint8_t
  53 nix_get_relchan(struct otx2_eth_dev *dev)
  54 {
  55         return dev->tx_chan_base & 0xff;
  56 }
  57
  58 static bool
  59 nix_tm_have_tl1_access(struct otx2_eth_dev *dev)
  60 {
  61         bool is_lbk = otx2_dev_is_lbk(dev);
  62         return otx2_dev_is_pf(dev) && !otx2_dev_is_Ax(dev) &&
  63                 !is_lbk && !dev->maxvf;
  64 }
  65
  66 static int
  67 find_prio_anchor(struct otx2_eth_dev *dev, uint32_t node_id)
  68 {
  69         struct otx2_nix_tm_node *child_node;
  70
  71         TAILQ_FOREACH(child_node, &dev->node_list, node) {
  72                 if (!child_node->parent)
  73                         continue;
  74                 if (!(child_node->parent->id == node_id))
  75                         continue;
  76                 if (child_node->priority == child_node->parent->rr_prio)
  77                         continue;
  78                 return child_node->hw_id - child_node->priority;
  79         }
  80         return 0;
  81 }
  82
  83
  84 static struct otx2_nix_tm_shaper_profile *
  85 nix_tm_shaper_profile_search(struct otx2_eth_dev *dev, uint32_t shaper_id)
  86 {
  87         struct otx2_nix_tm_shaper_profile *tm_shaper_profile;
  88
  89         TAILQ_FOREACH(tm_shaper_profile, &dev->shaper_profile_list, shaper) {
  90                 if (tm_shaper_profile->shaper_profile_id == shaper_id)
  91                         return tm_shaper_profile;
  92         }
  93         return NULL;
  94 }
  95
  96 static inline uint64_t
  97 shaper_rate_to_nix(uint64_t cclk_hz, uint64_t cclk_ticks,
  98                    uint64_t value, uint64_t *exponent_p,
  99                    uint64_t *mantissa_p, uint64_t *div_exp_p)
 100 {
 101         uint64_t div_exp, exponent, mantissa;
 102
 103         /* Boundary checks */
 104         if (value < MIN_SHAPER_RATE(cclk_hz, cclk_ticks) ||
 105             value > MAX_SHAPER_RATE(cclk_hz, cclk_ticks))
 106                 return 0;
 107
 108         if (value <= SHAPER_RATE(cclk_hz, cclk_ticks, 0, 0, 0)) {
 109                 /* Calculate rate div_exp and mantissa using
 110                  * the following formula:
 111                  *
 112                  * value = (cclk_hz * (256 + mantissa)
 113                  *              / ((cclk_ticks << div_exp) * 256)
 114                  */
 115                 div_exp = 0;
 116                 exponent = 0;
 117                 mantissa = MAX_RATE_MANTISSA;
 118
 119                 while (value < (cclk_hz / (cclk_ticks << div_exp)))
 120                         div_exp += 1;
 121
 122                 while (value <
 123                        ((cclk_hz * (256 + mantissa)) /
 124                         ((cclk_ticks << div_exp) * 256)))
 125                         mantissa -= 1;
 126         } else {
 127                 /* Calculate rate exponent and mantissa using
 128                  * the following formula:
 129                  *
 130                  * value = (cclk_hz * ((256 + mantissa) << exponent)
 131                  *              / (cclk_ticks * 256)
 132                  *
 133                  */
 134                 div_exp = 0;
 135                 exponent = MAX_RATE_EXPONENT;
 136                 mantissa = MAX_RATE_MANTISSA;
 137
 138                 while (value < (cclk_hz * (1 << exponent)) / cclk_ticks)
 139                         exponent -= 1;
 140
 141                 while (value < (cclk_hz * ((256 + mantissa) << exponent)) /
 142                        (cclk_ticks * 256))
 143                         mantissa -= 1;
 144         }
 145
 146         if (div_exp > MAX_RATE_DIV_EXP ||
 147             exponent > MAX_RATE_EXPONENT || mantissa > MAX_RATE_MANTISSA)
 148                 return 0;
 149
 150         if (div_exp_p)
 151                 *div_exp_p = div_exp;
 152         if (exponent_p)
 153                 *exponent_p = exponent;
 154         if (mantissa_p)
 155                 *mantissa_p = mantissa;
 156
 157         /* Calculate real rate value */
 158         return SHAPER_RATE(cclk_hz, cclk_ticks, exponent, mantissa, div_exp);
 159 }
 160
 161 static inline uint64_t
 162 lx_shaper_rate_to_nix(uint64_t cclk_hz, uint32_t hw_lvl,
 163                       uint64_t value, uint64_t *exponent,
 164                       uint64_t *mantissa, uint64_t *div_exp)
 165 {
 166         if (hw_lvl == NIX_TXSCH_LVL_TL1)
 167                 return shaper_rate_to_nix(cclk_hz, L1_TIME_WHEEL_CCLK_TICKS,
 168                                           value, exponent, mantissa, div_exp);
 169         else
 170                 return shaper_rate_to_nix(cclk_hz, LX_TIME_WHEEL_CCLK_TICKS,
 171                                           value, exponent, mantissa, div_exp);
 172 }
 173
 174 static inline uint64_t
 175 shaper_burst_to_nix(uint64_t value, uint64_t *exponent_p,
 176                     uint64_t *mantissa_p)
 177 {
 178         uint64_t exponent, mantissa;
 179
 180         if (value < MIN_SHAPER_BURST || value > MAX_SHAPER_BURST)
 181                 return 0;
 182
 183         /* Calculate burst exponent and mantissa using
 184          * the following formula:
 185          *
 186          * value = (((256 + mantissa) << (exponent + 1)
 187          / 256)
 188          *
 189          */
 190         exponent = MAX_BURST_EXPONENT;
 191         mantissa = MAX_BURST_MANTISSA;
 192
 193         while (value < (1ull << (exponent + 1)))
 194                 exponent -= 1;
 195
 196         while (value < ((256 + mantissa) << (exponent + 1)) / 256)
 197                 mantissa -= 1;
 198
 199         if (exponent > MAX_BURST_EXPONENT || mantissa > MAX_BURST_MANTISSA)
 200                 return 0;
 201
 202         if (exponent_p)
 203                 *exponent_p = exponent;
 204         if (mantissa_p)
 205                 *mantissa_p = mantissa;
 206
 207         return SHAPER_BURST(exponent, mantissa);
 208 }
 209
 210 static int
 211 configure_shaper_cir_pir_reg(struct otx2_eth_dev *dev,
 212                              struct otx2_nix_tm_node *tm_node,
 213                              struct shaper_params *cir,
 214                              struct shaper_params *pir)
 215 {
 216         uint32_t shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
 217         struct otx2_nix_tm_shaper_profile *shaper_profile = NULL;
 218         struct rte_tm_shaper_params *param;
 219
 220         shaper_profile_id = tm_node->params.shaper_profile_id;
 221
 222         shaper_profile = nix_tm_shaper_profile_search(dev, shaper_profile_id);
 223         if (shaper_profile) {
 224                 param = &shaper_profile->profile;
 225                 /* Calculate CIR exponent and mantissa */
 226                 if (param->committed.rate)
 227                         cir->rate = lx_shaper_rate_to_nix(CCLK_HZ,
 228                                                           tm_node->hw_lvl_id,
 229                                                           param->committed.rate,
 230                                                           &cir->exponent,
 231                                                           &cir->mantissa,
 232                                                           &cir->div_exp);
 233
 234                 /* Calculate PIR exponent and mantissa */
 235                 if (param->peak.rate)
 236                         pir->rate = lx_shaper_rate_to_nix(CCLK_HZ,
 237                                                           tm_node->hw_lvl_id,
 238                                                           param->peak.rate,
 239                                                           &pir->exponent,
 240                                                           &pir->mantissa,
 241                                                           &pir->div_exp);
 242
 243                 /* Calculate CIR burst exponent and mantissa */
 244                 if (param->committed.size)
 245                         cir->burst = shaper_burst_to_nix(param->committed.size,
 246                                                          &cir->burst_exponent,
 247                                                          &cir->burst_mantissa);
 248
 249                 /* Calculate PIR burst exponent and mantissa */
 250                 if (param->peak.size)
 251                         pir->burst = shaper_burst_to_nix(param->peak.size,
 252                                                          &pir->burst_exponent,
 253                                                          &pir->burst_mantissa);
 254         }
 255
 256         return 0;
 257 }
 258
 259 static int
 260 send_tm_reqval(struct otx2_mbox *mbox, struct nix_txschq_config *req)
 261 {
 262         int rc;
 263
 264         if (req->num_regs > MAX_REGS_PER_MBOX_MSG)
 265                 return -ERANGE;
 266
 267         rc = otx2_mbox_process(mbox);
 268         if (rc)
 269                 return rc;
 270
 271         req->num_regs = 0;
 272         return 0;
 273 }
 274
 275 static int
 276 populate_tm_registers(struct otx2_eth_dev *dev,
 277                       struct otx2_nix_tm_node *tm_node)
 278 {
 279         uint64_t strict_schedul_prio, rr_prio;
 280         struct otx2_mbox *mbox = dev->mbox;
 281         volatile uint64_t *reg, *regval;
 282         uint64_t parent = 0, child = 0;
 283         struct shaper_params cir, pir;
 284         struct nix_txschq_config *req;
 285         uint64_t rr_quantum;
 286         uint32_t hw_lvl;
 287         uint32_t schq;
 288         int rc;
 289
 290         memset(&cir, 0, sizeof(cir));
 291         memset(&pir, 0, sizeof(pir));
 292
 293         /* Skip leaf nodes */
 294         if (tm_node->hw_lvl_id == NIX_TXSCH_LVL_CNT)
 295                 return 0;
 296
 297         /* Root node will not have a parent node */
 298         if (tm_node->hw_lvl_id == dev->otx2_tm_root_lvl)
 299                 parent = tm_node->parent_hw_id;
 300         else
 301                 parent = tm_node->parent->hw_id;
 302
 303         /* Do we need this trigger to configure TL1 */
 304         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
 305             tm_node->hw_lvl_id == dev->otx2_tm_root_lvl) {
 306                 schq = parent;
 307                 /*
 308                  * Default config for TL1.
 309                  * For VF this is always ignored.
 310                  */
 311
 312                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 313                 req->lvl = NIX_TXSCH_LVL_TL1;
 314
 315                 /* Set DWRR quantum */
 316                 req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq);
 317                 req->regval[0] = TXSCH_TL1_DFLT_RR_QTM;
 318                 req->num_regs++;
 319
 320                 req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq);
 321                 req->regval[1] = (TXSCH_TL1_DFLT_RR_PRIO << 1);
 322                 req->num_regs++;
 323
 324                 req->reg[2] = NIX_AF_TL1X_CIR(schq);
 325                 req->regval[2] = 0;
 326                 req->num_regs++;
 327
 328                 rc = send_tm_reqval(mbox, req);
 329                 if (rc)
 330                         goto error;
 331         }
 332
 333         if (tm_node->hw_lvl_id != NIX_TXSCH_LVL_SMQ)
 334                 child = find_prio_anchor(dev, tm_node->id);
 335
 336         rr_prio = tm_node->rr_prio;
 337         hw_lvl = tm_node->hw_lvl_id;
 338         strict_schedul_prio = tm_node->priority;
 339         schq = tm_node->hw_id;
 340         rr_quantum = (tm_node->weight * NIX_TM_RR_QUANTUM_MAX) /
 341                 MAX_SCHED_WEIGHT;
 342
 343         configure_shaper_cir_pir_reg(dev, tm_node, &cir, &pir);
 344
 345         otx2_tm_dbg("Configure node %p, lvl %u hw_lvl %u, id %u, hw_id %u,"
 346                      "parent_hw_id %" PRIx64 ", pir %" PRIx64 ", cir %" PRIx64,
 347                      tm_node, tm_node->level_id, hw_lvl,
 348                      tm_node->id, schq, parent, pir.rate, cir.rate);
 349
 350         rc = -EFAULT;
 351
 352         switch (hw_lvl) {
 353         case NIX_TXSCH_LVL_SMQ:
 354                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 355                 req->lvl = hw_lvl;
 356                 reg = req->reg;
 357                 regval = req->regval;
 358                 req->num_regs = 0;
 359
 360                 /* Set xoff which will be cleared later */
 361                 *reg++ = NIX_AF_SMQX_CFG(schq);
 362                 *regval++ = BIT_ULL(50) | ((uint64_t)NIX_MAX_VTAG_INS << 36) |
 363                                 (NIX_MAX_HW_FRS << 8) | NIX_MIN_HW_FRS;
 364                 req->num_regs++;
 365                 *reg++ = NIX_AF_MDQX_PARENT(schq);
 366                 *regval++ = parent << 16;
 367                 req->num_regs++;
 368                 *reg++ = NIX_AF_MDQX_SCHEDULE(schq);
 369                 *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 370                 req->num_regs++;
 371                 if (pir.rate && pir.burst) {
 372                         *reg++ = NIX_AF_MDQX_PIR(schq);
 373                         *regval++ = shaper2regval(&pir) | 1;
 374                         req->num_regs++;
 375                 }
 376
 377                 if (cir.rate && cir.burst) {
 378                         *reg++ = NIX_AF_MDQX_CIR(schq);
 379                         *regval++ = shaper2regval(&cir) | 1;
 380                         req->num_regs++;
 381                 }
 382
 383                 rc = send_tm_reqval(mbox, req);
 384                 if (rc)
 385                         goto error;
 386                 break;
 387         case NIX_TXSCH_LVL_TL4:
 388                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 389                 req->lvl = hw_lvl;
 390                 req->num_regs = 0;
 391                 reg = req->reg;
 392                 regval = req->regval;
 393
 394                 *reg++ = NIX_AF_TL4X_PARENT(schq);
 395                 *regval++ = parent << 16;
 396                 req->num_regs++;
 397                 *reg++ = NIX_AF_TL4X_TOPOLOGY(schq);
 398                 *regval++ = (child << 32) | (rr_prio << 1);
 399                 req->num_regs++;
 400                 *reg++ = NIX_AF_TL4X_SCHEDULE(schq);
 401                 *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 402                 req->num_regs++;
 403                 if (pir.rate && pir.burst) {
 404                         *reg++ = NIX_AF_TL4X_PIR(schq);
 405                         *regval++ = shaper2regval(&pir) | 1;
 406                         req->num_regs++;
 407                 }
 408                 if (cir.rate && cir.burst) {
 409                         *reg++ = NIX_AF_TL4X_CIR(schq);
 410                         *regval++ = shaper2regval(&cir) | 1;
 411                         req->num_regs++;
 412                 }
 413
 414                 rc = send_tm_reqval(mbox, req);
 415                 if (rc)
 416                         goto error;
 417                 break;
 418         case NIX_TXSCH_LVL_TL3:
 419                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 420                 req->lvl = hw_lvl;
 421                 req->num_regs = 0;
 422                 reg = req->reg;
 423                 regval = req->regval;
 424
 425                 *reg++ = NIX_AF_TL3X_PARENT(schq);
 426                 *regval++ = parent << 16;
 427                 req->num_regs++;
 428                 *reg++ = NIX_AF_TL3X_TOPOLOGY(schq);
 429                 *regval++ = (child << 32) | (rr_prio << 1);
 430                 req->num_regs++;
 431                 *reg++ = NIX_AF_TL3X_SCHEDULE(schq);
 432                 *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 433                 req->num_regs++;
 434                 if (pir.rate && pir.burst) {
 435                         *reg++ = NIX_AF_TL3X_PIR(schq);
 436                         *regval++ = shaper2regval(&pir) | 1;
 437                         req->num_regs++;
 438                 }
 439                 if (cir.rate && cir.burst) {
 440                         *reg++ = NIX_AF_TL3X_CIR(schq);
 441                         *regval++ = shaper2regval(&cir) | 1;
 442                         req->num_regs++;
 443                 }
 444
 445                 rc = send_tm_reqval(mbox, req);
 446                 if (rc)
 447                         goto error;
 448                 break;
 449         case NIX_TXSCH_LVL_TL2:
 450                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 451                 req->lvl = hw_lvl;
 452                 req->num_regs = 0;
 453                 reg = req->reg;
 454                 regval = req->regval;
 455
 456                 *reg++ = NIX_AF_TL2X_PARENT(schq);
 457                 *regval++ = parent << 16;
 458                 req->num_regs++;
 459                 *reg++ = NIX_AF_TL2X_TOPOLOGY(schq);
 460                 *regval++ = (child << 32) | (rr_prio << 1);
 461                 req->num_regs++;
 462                 *reg++ = NIX_AF_TL2X_SCHEDULE(schq);
 463                 if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2)
 464                         *regval++ = (1 << 24) | rr_quantum;
 465                 else
 466                         *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 467                 req->num_regs++;
 468                 *reg++ = NIX_AF_TL3_TL2X_LINKX_CFG(schq, nix_get_link(dev));
 469                 *regval++ = BIT_ULL(12) | nix_get_relchan(dev);
 470                 req->num_regs++;
 471                 if (pir.rate && pir.burst) {
 472                         *reg++ = NIX_AF_TL2X_PIR(schq);
 473                         *regval++ = shaper2regval(&pir) | 1;
 474                         req->num_regs++;
 475                 }
 476                 if (cir.rate && cir.burst) {
 477                         *reg++ = NIX_AF_TL2X_CIR(schq);
 478                         *regval++ = shaper2regval(&cir) | 1;
 479                         req->num_regs++;
 480                 }
 481
 482                 rc = send_tm_reqval(mbox, req);
 483                 if (rc)
 484                         goto error;
 485                 break;
 486         case NIX_TXSCH_LVL_TL1:
 487                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 488                 req->lvl = hw_lvl;
 489                 req->num_regs = 0;
 490                 reg = req->reg;
 491                 regval = req->regval;
 492
 493                 *reg++ = NIX_AF_TL1X_SCHEDULE(schq);
 494                 *regval++ = rr_quantum;
 495                 req->num_regs++;
 496                 *reg++ = NIX_AF_TL1X_TOPOLOGY(schq);
 497                 *regval++ = (child << 32) | (rr_prio << 1 /*RR_PRIO*/);
 498                 req->num_regs++;
 499                 if (cir.rate && cir.burst) {
 500                         *reg++ = NIX_AF_TL1X_CIR(schq);
 501                         *regval++ = shaper2regval(&cir) | 1;
 502                         req->num_regs++;
 503                 }
 504
 505                 rc = send_tm_reqval(mbox, req);
 506                 if (rc)
 507                         goto error;
 508                 break;
 509         }
 510
 511         return 0;
 512 error:
 513         otx2_err("Txschq cfg request failed for node %p, rc=%d", tm_node, rc);
 514         return rc;
 515 }
 516
 517
 518 static int
 519 nix_tm_txsch_reg_config(struct otx2_eth_dev *dev)
 520 {
 521         struct otx2_nix_tm_node *tm_node;
 522         uint32_t lvl;
 523         int rc = 0;
 524
 525         if (nix_get_link(dev) == 13)
 526                 return -EPERM;
 527
 528         for (lvl = 0; lvl < (uint32_t)dev->otx2_tm_root_lvl + 1; lvl++) {
 529                 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 530                         if (tm_node->hw_lvl_id == lvl) {
 531                                 rc = populate_tm_registers(dev, tm_node);
 532                                 if (rc)
 533                                         goto exit;
 534                         }
 535                 }
 536         }
 537 exit:
 538         return rc;
 539 }
 540
 541 static struct otx2_nix_tm_node *
 542 nix_tm_node_search(struct otx2_eth_dev *dev,
 543                    uint32_t node_id, bool user)
 544 {
 545         struct otx2_nix_tm_node *tm_node;
 546
 547         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 548                 if (tm_node->id == node_id &&
 549                     (user == !!(tm_node->flags & NIX_TM_NODE_USER)))
 550                         return tm_node;
 551         }
 552         return NULL;
 553 }
 554
 555 static uint32_t
 556 check_rr(struct otx2_eth_dev *dev, uint32_t priority, uint32_t parent_id)
 557 {
 558         struct otx2_nix_tm_node *tm_node;
 559         uint32_t rr_num = 0;
 560
 561         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 562                 if (!tm_node->parent)
 563                         continue;
 564
 565                 if (!(tm_node->parent->id == parent_id))
 566                         continue;
 567
 568                 if (tm_node->priority == priority)
 569                         rr_num++;
 570         }
 571         return rr_num;
 572 }
 573
 574 static int
 575 nix_tm_update_parent_info(struct otx2_eth_dev *dev)
 576 {
 577         struct otx2_nix_tm_node *tm_node_child;
 578         struct otx2_nix_tm_node *tm_node;
 579         struct otx2_nix_tm_node *parent;
 580         uint32_t rr_num = 0;
 581         uint32_t priority;
 582
 583         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 584                 if (!tm_node->parent)
 585                         continue;
 586                 /* Count group of children of same priority i.e are RR */
 587                 parent = tm_node->parent;
 588                 priority = tm_node->priority;
 589                 rr_num = check_rr(dev, priority, parent->id);
 590
 591                 /* Assuming that multiple RR groups are
 592                  * not configured based on capability.
 593                  */
 594                 if (rr_num > 1) {
 595                         parent->rr_prio = priority;
 596                         parent->rr_num = rr_num;
 597                 }
 598
 599                 /* Find out static priority children that are not in RR */
 600                 TAILQ_FOREACH(tm_node_child, &dev->node_list, node) {
 601                         if (!tm_node_child->parent)
 602                                 continue;
 603                         if (parent->id != tm_node_child->parent->id)
 604                                 continue;
 605                         if (parent->max_prio == UINT32_MAX &&
 606                             tm_node_child->priority != parent->rr_prio)
 607                                 parent->max_prio = 0;
 608
 609                         if (parent->max_prio < tm_node_child->priority &&
 610                             parent->rr_prio != tm_node_child->priority)
 611                                 parent->max_prio = tm_node_child->priority;
 612                 }
 613         }
 614
 615         return 0;
 616 }
 617
 618 static int
 619 nix_tm_node_add_to_list(struct otx2_eth_dev *dev, uint32_t node_id,
 620                         uint32_t parent_node_id, uint32_t priority,
 621                         uint32_t weight, uint16_t hw_lvl_id,
 622                         uint16_t level_id, bool user,
 623                         struct rte_tm_node_params *params)
 624 {
 625         struct otx2_nix_tm_shaper_profile *shaper_profile;
 626         struct otx2_nix_tm_node *tm_node, *parent_node;
 627         uint32_t shaper_profile_id;
 628
 629         shaper_profile_id = params->shaper_profile_id;
 630         shaper_profile = nix_tm_shaper_profile_search(dev, shaper_profile_id);
 631
 632         parent_node = nix_tm_node_search(dev, parent_node_id, user);
 633
 634         tm_node = rte_zmalloc("otx2_nix_tm_node",
 635                               sizeof(struct otx2_nix_tm_node), 0);
 636         if (!tm_node)
 637                 return -ENOMEM;
 638
 639         tm_node->level_id = level_id;
 640         tm_node->hw_lvl_id = hw_lvl_id;
 641
 642         tm_node->id = node_id;
 643         tm_node->priority = priority;
 644         tm_node->weight = weight;
 645         tm_node->rr_prio = 0xf;
 646         tm_node->max_prio = UINT32_MAX;
 647         tm_node->hw_id = UINT32_MAX;
 648         tm_node->flags = 0;
 649         if (user)
 650                 tm_node->flags = NIX_TM_NODE_USER;
 651         rte_memcpy(&tm_node->params, params, sizeof(struct rte_tm_node_params));
 652
 653         if (shaper_profile)
 654                 shaper_profile->reference_count++;
 655         tm_node->parent = parent_node;
 656         tm_node->parent_hw_id = UINT32_MAX;
 657
 658         TAILQ_INSERT_TAIL(&dev->node_list, tm_node, node);
 659
 660         return 0;
 661 }
 662
 663 static int
 664 nix_tm_clear_shaper_profiles(struct otx2_eth_dev *dev)
 665 {
 666         struct otx2_nix_tm_shaper_profile *shaper_profile;
 667
 668         while ((shaper_profile = TAILQ_FIRST(&dev->shaper_profile_list))) {
 669                 if (shaper_profile->reference_count)
 670                         otx2_tm_dbg("Shaper profile %u has non zero references",
 671                                     shaper_profile->shaper_profile_id);
 672                 TAILQ_REMOVE(&dev->shaper_profile_list, shaper_profile, shaper);
 673                 rte_free(shaper_profile);
 674         }
 675
 676         return 0;
 677 }
 678
 679 static int
 680 nix_smq_xoff(struct otx2_eth_dev *dev, uint16_t smq, bool enable)
 681 {
 682         struct otx2_mbox *mbox = dev->mbox;
 683         struct nix_txschq_config *req;
 684
 685         req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 686         req->lvl = NIX_TXSCH_LVL_SMQ;
 687         req->num_regs = 1;
 688
 689         req->reg[0] = NIX_AF_SMQX_CFG(smq);
 690         /* Unmodified fields */
 691         req->regval[0] = ((uint64_t)NIX_MAX_VTAG_INS << 36) |
 692                                 (NIX_MAX_HW_FRS << 8) | NIX_MIN_HW_FRS;
 693
 694         if (enable)
 695                 req->regval[0] |= BIT_ULL(50) | BIT_ULL(49);
 696         else
 697                 req->regval[0] |= 0;
 698
 699         return otx2_mbox_process(mbox);
 700 }
 701
 702 int
 703 otx2_nix_sq_sqb_aura_fc(void *__txq, bool enable)
 704 {
 705         struct otx2_eth_txq *txq = __txq;
 706         struct npa_aq_enq_req *req;
 707         struct npa_aq_enq_rsp *rsp;
 708         struct otx2_npa_lf *lf;
 709         struct otx2_mbox *mbox;
 710         uint64_t aura_handle;
 711         int rc;
 712
 713         lf = otx2_npa_lf_obj_get();
 714         if (!lf)
 715                 return -EFAULT;
 716         mbox = lf->mbox;
 717         /* Set/clear sqb aura fc_ena */
 718         aura_handle = txq->sqb_pool->pool_id;
 719         req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
 720
 721         req->aura_id = npa_lf_aura_handle_to_aura(aura_handle);
 722         req->ctype = NPA_AQ_CTYPE_AURA;
 723         req->op = NPA_AQ_INSTOP_WRITE;
 724         /* Below is not needed for aura writes but AF driver needs it */
 725         /* AF will translate to associated poolctx */
 726         req->aura.pool_addr = req->aura_id;
 727
 728         req->aura.fc_ena = enable;
 729         req->aura_mask.fc_ena = 1;
 730
 731         rc = otx2_mbox_process(mbox);
 732         if (rc)
 733                 return rc;
 734
 735         /* Read back npa aura ctx */
 736         req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
 737
 738         req->aura_id = npa_lf_aura_handle_to_aura(aura_handle);
 739         req->ctype = NPA_AQ_CTYPE_AURA;
 740         req->op = NPA_AQ_INSTOP_READ;
 741
 742         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
 743         if (rc)
 744                 return rc;
 745
 746         /* Init when enabled as there might be no triggers */
 747         if (enable)
 748                 *(volatile uint64_t *)txq->fc_mem = rsp->aura.count;
 749         else
 750                 *(volatile uint64_t *)txq->fc_mem = txq->nb_sqb_bufs;
 751         /* Sync write barrier */
 752         rte_wmb();
 753
 754         return 0;
 755 }
 756
 757 static void
 758 nix_txq_flush_sq_spin(struct otx2_eth_txq *txq)
 759 {
 760         uint16_t sqb_cnt, head_off, tail_off;
 761         struct otx2_eth_dev *dev = txq->dev;
 762         uint16_t sq = txq->sq;
 763         uint64_t reg, val;
 764         int64_t *regaddr;
 765
 766         while (true) {
 767                 reg = ((uint64_t)sq << 32);
 768                 regaddr = (int64_t *)(dev->base + NIX_LF_SQ_OP_PKTS);
 769                 val = otx2_atomic64_add_nosync(reg, regaddr);
 770
 771                 regaddr = (int64_t *)(dev->base + NIX_LF_SQ_OP_STATUS);
 772                 val = otx2_atomic64_add_nosync(reg, regaddr);
 773                 sqb_cnt = val & 0xFFFF;
 774                 head_off = (val >> 20) & 0x3F;
 775                 tail_off = (val >> 28) & 0x3F;
 776
 777                 /* SQ reached quiescent state */
 778                 if (sqb_cnt <= 1 && head_off == tail_off &&
 779                     (*txq->fc_mem == txq->nb_sqb_bufs)) {
 780                         break;
 781                 }
 782
 783                 rte_pause();
 784         }
 785 }
 786
 787 int
 788 otx2_nix_tm_sw_xoff(void *__txq, bool dev_started)
 789 {
 790         struct otx2_eth_txq *txq = __txq;
 791         struct otx2_eth_dev *dev = txq->dev;
 792         struct otx2_mbox *mbox = dev->mbox;
 793         struct nix_aq_enq_req *req;
 794         struct nix_aq_enq_rsp *rsp;
 795         uint16_t smq;
 796         int rc;
 797
 798         /* Get smq from sq */
 799         req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
 800         req->qidx = txq->sq;
 801         req->ctype = NIX_AQ_CTYPE_SQ;
 802         req->op = NIX_AQ_INSTOP_READ;
 803         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
 804         if (rc) {
 805                 otx2_err("Failed to get smq, rc=%d", rc);
 806                 return -EIO;
 807         }
 808
 809         /* Check if sq is enabled */
 810         if (!rsp->sq.ena)
 811                 return 0;
 812
 813         smq = rsp->sq.smq;
 814
 815         /* Enable CGX RXTX to drain pkts */
 816         if (!dev_started) {
 817                 rc = otx2_cgx_rxtx_start(dev);
 818                 if (rc)
 819                         return rc;
 820         }
 821
 822         rc = otx2_nix_sq_sqb_aura_fc(txq, false);
 823         if (rc < 0) {
 824                 otx2_err("Failed to disable sqb aura fc, rc=%d", rc);
 825                 goto cleanup;
 826         }
 827
 828         /* Disable smq xoff for case it was enabled earlier */
 829         rc = nix_smq_xoff(dev, smq, false);
 830         if (rc) {
 831                 otx2_err("Failed to enable smq for sq %u, rc=%d", txq->sq, rc);
 832                 goto cleanup;
 833         }
 834
 835         /* Wait for sq entries to be flushed */
 836         nix_txq_flush_sq_spin(txq);
 837
 838         /* Flush and enable smq xoff */
 839         rc = nix_smq_xoff(dev, smq, true);
 840         if (rc) {
 841                 otx2_err("Failed to disable smq for sq %u, rc=%d", txq->sq, rc);
 842                 return rc;
 843         }
 844
 845 cleanup:
 846         /* Restore cgx state */
 847         if (!dev_started)
 848                 rc |= otx2_cgx_rxtx_stop(dev);
 849
 850         return rc;
 851 }
 852
 853 static int
 854 nix_tm_sw_xon(struct otx2_eth_txq *txq,
 855               uint16_t smq, uint32_t rr_quantum)
 856 {
 857         struct otx2_eth_dev *dev = txq->dev;
 858         struct otx2_mbox *mbox = dev->mbox;
 859         struct nix_aq_enq_req *req;
 860         int rc;
 861
 862         otx2_tm_dbg("Enabling sq(%u)->smq(%u), rr_quantum %u",
 863                     txq->sq, txq->sq, rr_quantum);
 864         /* Set smq from sq */
 865         req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
 866         req->qidx = txq->sq;
 867         req->ctype = NIX_AQ_CTYPE_SQ;
 868         req->op = NIX_AQ_INSTOP_WRITE;
 869         req->sq.smq = smq;
 870         req->sq.smq_rr_quantum = rr_quantum;
 871         req->sq_mask.smq = ~req->sq_mask.smq;
 872         req->sq_mask.smq_rr_quantum = ~req->sq_mask.smq_rr_quantum;
 873
 874         rc = otx2_mbox_process(mbox);
 875         if (rc) {
 876                 otx2_err("Failed to set smq, rc=%d", rc);
 877                 return -EIO;
 878         }
 879
 880         /* Enable sqb_aura fc */
 881         rc = otx2_nix_sq_sqb_aura_fc(txq, true);
 882         if (rc < 0) {
 883                 otx2_err("Failed to enable sqb aura fc, rc=%d", rc);
 884                 return rc;
 885         }
 886
 887         /* Disable smq xoff */
 888         rc = nix_smq_xoff(dev, smq, false);
 889         if (rc) {
 890                 otx2_err("Failed to enable smq for sq %u", txq->sq);
 891                 return rc;
 892         }
 893
 894         return 0;
 895 }
 896
 897 static int
 898 nix_tm_free_resources(struct otx2_eth_dev *dev, uint32_t flags_mask,
 899                       uint32_t flags, bool hw_only)
 900 {
 901         struct otx2_nix_tm_shaper_profile *shaper_profile;
 902         struct otx2_nix_tm_node *tm_node, *next_node;
 903         struct otx2_mbox *mbox = dev->mbox;
 904         struct nix_txsch_free_req *req;
 905         uint32_t shaper_profile_id;
 906         bool skip_node = false;
 907         int rc = 0;
 908
 909         next_node = TAILQ_FIRST(&dev->node_list);
 910         while (next_node) {
 911                 tm_node = next_node;
 912                 next_node = TAILQ_NEXT(tm_node, node);
 913
 914                 /* Check for only requested nodes */
 915                 if ((tm_node->flags & flags_mask) != flags)
 916                         continue;
 917
 918                 if (nix_tm_have_tl1_access(dev) &&
 919                     tm_node->hw_lvl_id ==  NIX_TXSCH_LVL_TL1)
 920                         skip_node = true;
 921
 922                 otx2_tm_dbg("Free hwres for node %u, hwlvl %u, hw_id %u (%p)",
 923                             tm_node->id,  tm_node->hw_lvl_id,
 924                             tm_node->hw_id, tm_node);
 925                 /* Free specific HW resource if requested */
 926                 if (!skip_node && flags_mask &&
 927                     tm_node->flags & NIX_TM_NODE_HWRES) {
 928                         req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
 929                         req->flags = 0;
 930                         req->schq_lvl = tm_node->hw_lvl_id;
 931                         req->schq = tm_node->hw_id;
 932                         rc = otx2_mbox_process(mbox);
 933                         if (rc)
 934                                 break;
 935                 } else {
 936                         skip_node = false;
 937                 }
 938                 tm_node->flags &= ~NIX_TM_NODE_HWRES;
 939
 940                 /* Leave software elements if needed */
 941                 if (hw_only)
 942                         continue;
 943
 944                 shaper_profile_id = tm_node->params.shaper_profile_id;
 945                 shaper_profile =
 946                         nix_tm_shaper_profile_search(dev, shaper_profile_id);
 947                 if (shaper_profile)
 948                         shaper_profile->reference_count--;
 949
 950                 TAILQ_REMOVE(&dev->node_list, tm_node, node);
 951                 rte_free(tm_node);
 952         }
 953
 954         if (!flags_mask) {
 955                 /* Free all hw resources */
 956                 req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
 957                 req->flags = TXSCHQ_FREE_ALL;
 958
 959                 return otx2_mbox_process(mbox);
 960         }
 961
 962         return rc;
 963 }
 964
 965 static uint8_t
 966 nix_tm_copy_rsp_to_dev(struct otx2_eth_dev *dev,
 967                        struct nix_txsch_alloc_rsp *rsp)
 968 {
 969         uint16_t schq;
 970         uint8_t lvl;
 971
 972         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
 973                 for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) {
 974                         dev->txschq_list[lvl][schq] = rsp->schq_list[lvl][schq];
 975                         dev->txschq_contig_list[lvl][schq] =
 976                                 rsp->schq_contig_list[lvl][schq];
 977                 }
 978
 979                 dev->txschq[lvl] = rsp->schq[lvl];
 980                 dev->txschq_contig[lvl] = rsp->schq_contig[lvl];
 981         }
 982         return 0;
 983 }
 984
 985 static int
 986 nix_tm_assign_id_to_node(struct otx2_eth_dev *dev,
 987                          struct otx2_nix_tm_node *child,
 988                          struct otx2_nix_tm_node *parent)
 989 {
 990         uint32_t hw_id, schq_con_index, prio_offset;
 991         uint32_t l_id, schq_index;
 992
 993         otx2_tm_dbg("Assign hw id for child node %u, lvl %u, hw_lvl %u (%p)",
 994                     child->id, child->level_id, child->hw_lvl_id, child);
 995
 996         child->flags |= NIX_TM_NODE_HWRES;
 997
 998         /* Process root nodes */
 999         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
1000             child->hw_lvl_id == dev->otx2_tm_root_lvl && !parent) {
1001                 int idx = 0;
1002                 uint32_t tschq_con_index;
1003
1004                 l_id = child->hw_lvl_id;
1005                 tschq_con_index = dev->txschq_contig_index[l_id];
1006                 hw_id = dev->txschq_contig_list[l_id][tschq_con_index];
1007                 child->hw_id = hw_id;
1008                 dev->txschq_contig_index[l_id]++;
1009                 /* Update TL1 hw_id for its parent for config purpose */
1010                 idx = dev->txschq_index[NIX_TXSCH_LVL_TL1]++;
1011                 hw_id = dev->txschq_list[NIX_TXSCH_LVL_TL1][idx];
1012                 child->parent_hw_id = hw_id;
1013                 return 0;
1014         }
1015         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL1 &&
1016             child->hw_lvl_id == dev->otx2_tm_root_lvl && !parent) {
1017                 uint32_t tschq_con_index;
1018
1019                 l_id = child->hw_lvl_id;
1020                 tschq_con_index = dev->txschq_index[l_id];
1021                 hw_id = dev->txschq_list[l_id][tschq_con_index];
1022                 child->hw_id = hw_id;
1023                 dev->txschq_index[l_id]++;
1024                 return 0;
1025         }
1026
1027         /* Process children with parents */
1028         l_id = child->hw_lvl_id;
1029         schq_index = dev->txschq_index[l_id];
1030         schq_con_index = dev->txschq_contig_index[l_id];
1031
1032         if (child->priority == parent->rr_prio) {
1033                 hw_id = dev->txschq_list[l_id][schq_index];
1034                 child->hw_id = hw_id;
1035                 child->parent_hw_id = parent->hw_id;
1036                 dev->txschq_index[l_id]++;
1037         } else {
1038                 prio_offset = schq_con_index + child->priority;
1039                 hw_id = dev->txschq_contig_list[l_id][prio_offset];
1040                 child->hw_id = hw_id;
1041         }
1042         return 0;
1043 }
1044
1045 static int
1046 nix_tm_assign_hw_id(struct otx2_eth_dev *dev)
1047 {
1048         struct otx2_nix_tm_node *parent, *child;
1049         uint32_t child_hw_lvl, con_index_inc, i;
1050
1051         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--) {
1052                 TAILQ_FOREACH(parent, &dev->node_list, node) {
1053                         child_hw_lvl = parent->hw_lvl_id - 1;
1054                         if (parent->hw_lvl_id != i)
1055                                 continue;
1056                         TAILQ_FOREACH(child, &dev->node_list, node) {
1057                                 if (!child->parent)
1058                                         continue;
1059                                 if (child->parent->id != parent->id)
1060                                         continue;
1061                                 nix_tm_assign_id_to_node(dev, child, parent);
1062                         }
1063
1064                         con_index_inc = parent->max_prio + 1;
1065                         dev->txschq_contig_index[child_hw_lvl] += con_index_inc;
1066
1067                         /*
1068                          * Explicitly assign id to parent node if it
1069                          * doesn't have a parent
1070                          */
1071                         if (parent->hw_lvl_id == dev->otx2_tm_root_lvl)
1072                                 nix_tm_assign_id_to_node(dev, parent, NULL);
1073                 }
1074         }
1075         return 0;
1076 }
1077
1078 static uint8_t
1079 nix_tm_count_req_schq(struct otx2_eth_dev *dev,
1080                       struct nix_txsch_alloc_req *req, uint8_t lvl)
1081 {
1082         struct otx2_nix_tm_node *tm_node;
1083         uint8_t contig_count;
1084
1085         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1086                 if (lvl == tm_node->hw_lvl_id) {
1087                         req->schq[lvl - 1] += tm_node->rr_num;
1088                         if (tm_node->max_prio != UINT32_MAX) {
1089                                 contig_count = tm_node->max_prio + 1;
1090                                 req->schq_contig[lvl - 1] += contig_count;
1091                         }
1092                 }
1093                 if (lvl == dev->otx2_tm_root_lvl &&
1094                     dev->otx2_tm_root_lvl && lvl == NIX_TXSCH_LVL_TL2 &&
1095                     tm_node->hw_lvl_id == dev->otx2_tm_root_lvl) {
1096                         req->schq_contig[dev->otx2_tm_root_lvl]++;
1097                 }
1098         }
1099
1100         req->schq[NIX_TXSCH_LVL_TL1] = 1;
1101         req->schq_contig[NIX_TXSCH_LVL_TL1] = 0;
1102
1103         return 0;
1104 }
1105
1106 static int
1107 nix_tm_prepare_txschq_req(struct otx2_eth_dev *dev,
1108                           struct nix_txsch_alloc_req *req)
1109 {
1110         uint8_t i;
1111
1112         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--)
1113                 nix_tm_count_req_schq(dev, req, i);
1114
1115         for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
1116                 dev->txschq_index[i] = 0;
1117                 dev->txschq_contig_index[i] = 0;
1118         }
1119         return 0;
1120 }
1121
1122 static int
1123 nix_tm_send_txsch_alloc_msg(struct otx2_eth_dev *dev)
1124 {
1125         struct otx2_mbox *mbox = dev->mbox;
1126         struct nix_txsch_alloc_req *req;
1127         struct nix_txsch_alloc_rsp *rsp;
1128         int rc;
1129
1130         req = otx2_mbox_alloc_msg_nix_txsch_alloc(mbox);
1131
1132         rc = nix_tm_prepare_txschq_req(dev, req);
1133         if (rc)
1134                 return rc;
1135
1136         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
1137         if (rc)
1138                 return rc;
1139
1140         nix_tm_copy_rsp_to_dev(dev, rsp);
1141
1142         nix_tm_assign_hw_id(dev);
1143         return 0;
1144 }
1145
1146 static int
1147 nix_tm_alloc_resources(struct rte_eth_dev *eth_dev, bool xmit_enable)
1148 {
1149         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1150         struct otx2_nix_tm_node *tm_node;
1151         uint16_t sq, smq, rr_quantum;
1152         struct otx2_eth_txq *txq;
1153         int rc;
1154
1155         nix_tm_update_parent_info(dev);
1156
1157         rc = nix_tm_send_txsch_alloc_msg(dev);
1158         if (rc) {
1159                 otx2_err("TM failed to alloc tm resources=%d", rc);
1160                 return rc;
1161         }
1162
1163         rc = nix_tm_txsch_reg_config(dev);
1164         if (rc) {
1165                 otx2_err("TM failed to configure sched registers=%d", rc);
1166                 return rc;
1167         }
1168
1169         /* Enable xmit as all the topology is ready */
1170         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1171                 if (tm_node->flags & NIX_TM_NODE_ENABLED)
1172                         continue;
1173
1174                 /* Enable xmit on sq */
1175                 if (tm_node->level_id != OTX2_TM_LVL_QUEUE) {
1176                         tm_node->flags |= NIX_TM_NODE_ENABLED;
1177                         continue;
1178                 }
1179
1180                 /* Don't enable SMQ or mark as enable */
1181                 if (!xmit_enable)
1182                         continue;
1183
1184                 sq = tm_node->id;
1185                 if (sq > eth_dev->data->nb_tx_queues) {
1186                         rc = -EFAULT;
1187                         break;
1188                 }
1189
1190                 txq = eth_dev->data->tx_queues[sq];
1191
1192                 smq = tm_node->parent->hw_id;
1193                 rr_quantum = (tm_node->weight *
1194                               NIX_TM_RR_QUANTUM_MAX) / MAX_SCHED_WEIGHT;
1195
1196                 rc = nix_tm_sw_xon(txq, smq, rr_quantum);
1197                 if (rc)
1198                         break;
1199                 tm_node->flags |= NIX_TM_NODE_ENABLED;
1200         }
1201
1202         if (rc)
1203                 otx2_err("TM failed to enable xmit on sq %u, rc=%d", sq, rc);
1204
1205         return rc;
1206 }
1207
1208 static int
1209 nix_tm_prepare_default_tree(struct rte_eth_dev *eth_dev)
1210 {
1211         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1212         uint32_t def = eth_dev->data->nb_tx_queues;
1213         struct rte_tm_node_params params;
1214         uint32_t leaf_parent, i;
1215         int rc = 0;
1216
1217         /* Default params */
1218         memset(&params, 0, sizeof(params));
1219         params.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
1220
1221         if (nix_tm_have_tl1_access(dev)) {
1222                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL1;
1223                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
1224                                              DEFAULT_RR_WEIGHT,
1225                                              NIX_TXSCH_LVL_TL1,
1226                                              OTX2_TM_LVL_ROOT, false, &params);
1227                 if (rc)
1228                         goto exit;
1229                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
1230                                              DEFAULT_RR_WEIGHT,
1231                                              NIX_TXSCH_LVL_TL2,
1232                                              OTX2_TM_LVL_SCH1, false, &params);
1233                 if (rc)
1234                         goto exit;
1235
1236                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
1237                                              DEFAULT_RR_WEIGHT,
1238                                              NIX_TXSCH_LVL_TL3,
1239                                              OTX2_TM_LVL_SCH2, false, &params);
1240                 if (rc)
1241                         goto exit;
1242
1243                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
1244                                              DEFAULT_RR_WEIGHT,
1245                                              NIX_TXSCH_LVL_TL4,
1246                                              OTX2_TM_LVL_SCH3, false, &params);
1247                 if (rc)
1248                         goto exit;
1249
1250                 rc = nix_tm_node_add_to_list(dev, def + 4, def + 3, 0,
1251                                              DEFAULT_RR_WEIGHT,
1252                                              NIX_TXSCH_LVL_SMQ,
1253                                              OTX2_TM_LVL_SCH4, false, &params);
1254                 if (rc)
1255                         goto exit;
1256
1257                 leaf_parent = def + 4;
1258         } else {
1259                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL2;
1260                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
1261                                              DEFAULT_RR_WEIGHT,
1262                                              NIX_TXSCH_LVL_TL2,
1263                                              OTX2_TM_LVL_ROOT, false, &params);
1264                 if (rc)
1265                         goto exit;
1266
1267                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
1268                                              DEFAULT_RR_WEIGHT,
1269                                              NIX_TXSCH_LVL_TL3,
1270                                              OTX2_TM_LVL_SCH1, false, &params);
1271                 if (rc)
1272                         goto exit;
1273
1274                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
1275                                              DEFAULT_RR_WEIGHT,
1276                                              NIX_TXSCH_LVL_TL4,
1277                                              OTX2_TM_LVL_SCH2, false, &params);
1278                 if (rc)
1279                         goto exit;
1280
1281                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
1282                                              DEFAULT_RR_WEIGHT,
1283                                              NIX_TXSCH_LVL_SMQ,
1284                                              OTX2_TM_LVL_SCH3, false, &params);
1285                 if (rc)
1286                         goto exit;
1287
1288                 leaf_parent = def + 3;
1289         }
1290
1291         /* Add leaf nodes */
1292         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
1293                 rc = nix_tm_node_add_to_list(dev, i, leaf_parent, 0,
1294                                              DEFAULT_RR_WEIGHT,
1295                                              NIX_TXSCH_LVL_CNT,
1296                                              OTX2_TM_LVL_QUEUE, false, &params);
1297                 if (rc)
1298                         break;
1299         }
1300
1301 exit:
1302         return rc;
1303 }
1304
1305 void otx2_nix_tm_conf_init(struct rte_eth_dev *eth_dev)
1306 {
1307         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1308
1309         TAILQ_INIT(&dev->node_list);
1310         TAILQ_INIT(&dev->shaper_profile_list);
1311 }
1312
1313 int otx2_nix_tm_init_default(struct rte_eth_dev *eth_dev)
1314 {
1315         struct otx2_eth_dev  *dev = otx2_eth_pmd_priv(eth_dev);
1316         uint16_t sq_cnt = eth_dev->data->nb_tx_queues;
1317         int rc;
1318
1319         /* Free up all resources already held */
1320         rc = nix_tm_free_resources(dev, 0, 0, false);
1321         if (rc) {
1322                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
1323                 return rc;
1324         }
1325
1326         /* Clear shaper profiles */
1327         nix_tm_clear_shaper_profiles(dev);
1328         dev->tm_flags = NIX_TM_DEFAULT_TREE;
1329
1330         rc = nix_tm_prepare_default_tree(eth_dev);
1331         if (rc != 0)
1332                 return rc;
1333
1334         rc = nix_tm_alloc_resources(eth_dev, false);
1335         if (rc != 0)
1336                 return rc;
1337         dev->tm_leaf_cnt = sq_cnt;
1338
1339         return 0;
1340 }
1341
1342 int
1343 otx2_nix_tm_fini(struct rte_eth_dev *eth_dev)
1344 {
1345         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1346         int rc;
1347
1348         /* Xmit is assumed to be disabled */
1349         /* Free up resources already held */
1350         rc = nix_tm_free_resources(dev, 0, 0, false);
1351         if (rc) {
1352                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
1353                 return rc;
1354         }
1355
1356         /* Clear shaper profiles */
1357         nix_tm_clear_shaper_profiles(dev);
1358
1359         dev->tm_flags = 0;
1360         return 0;
1361 }
1362
1363 int
1364 otx2_nix_tm_get_leaf_data(struct otx2_eth_dev *dev, uint16_t sq,
1365                           uint32_t *rr_quantum, uint16_t *smq)
1366 {
1367         struct otx2_nix_tm_node *tm_node;
1368         int rc;
1369
1370         /* 0..sq_cnt-1 are leaf nodes */
1371         if (sq >= dev->tm_leaf_cnt)
1372                 return -EINVAL;
1373
1374         /* Search for internal node first */
1375         tm_node = nix_tm_node_search(dev, sq, false);
1376         if (!tm_node)
1377                 tm_node = nix_tm_node_search(dev, sq, true);
1378
1379         /* Check if we found a valid leaf node */
1380         if (!tm_node || tm_node->level_id != OTX2_TM_LVL_QUEUE ||
1381             !tm_node->parent || tm_node->parent->hw_id == UINT32_MAX) {
1382                 return -EIO;
1383         }
1384
1385         /* Get SMQ Id of leaf node's parent */
1386         *smq = tm_node->parent->hw_id;
1387         *rr_quantum = (tm_node->weight * NIX_TM_RR_QUANTUM_MAX)
1388                 / MAX_SCHED_WEIGHT;
1389
1390         rc = nix_smq_xoff(dev, *smq, false);
1391         if (rc)
1392                 return rc;
1393         tm_node->flags |= NIX_TM_NODE_ENABLED;
1394
1395         return 0;
1396 }