drivers/net/octeontx2/otx2_tm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <rte_malloc.h>
   6
   7 #include "otx2_ethdev.h"
   8 #include "otx2_tm.h"
   9
  10 /* Use last LVL_CNT nodes as default nodes */
  11 #define NIX_DEFAULT_NODE_ID_START (RTE_TM_NODE_ID_NULL - NIX_TXSCH_LVL_CNT)
  12
  13 enum otx2_tm_node_level {
  14         OTX2_TM_LVL_ROOT = 0,
  15         OTX2_TM_LVL_SCH1,
  16         OTX2_TM_LVL_SCH2,
  17         OTX2_TM_LVL_SCH3,
  18         OTX2_TM_LVL_SCH4,
  19         OTX2_TM_LVL_QUEUE,
  20         OTX2_TM_LVL_MAX,
  21 };
  22
  23 static inline
  24 uint64_t shaper2regval(struct shaper_params *shaper)
  25 {
  26         return (shaper->burst_exponent << 37) | (shaper->burst_mantissa << 29) |
  27                 (shaper->div_exp << 13) | (shaper->exponent << 9) |
  28                 (shaper->mantissa << 1);
  29 }
  30
  31 static int
  32 nix_get_link(struct otx2_eth_dev *dev)
  33 {
  34         int link = 13 /* SDP */;
  35         uint16_t lmac_chan;
  36         uint16_t map;
  37
  38         lmac_chan = dev->tx_chan_base;
  39
  40         /* CGX lmac link */
  41         if (lmac_chan >= 0x800) {
  42                 map = lmac_chan & 0x7FF;
  43                 link = 4 * ((map >> 8) & 0xF) + ((map >> 4) & 0xF);
  44         } else if (lmac_chan < 0x700) {
  45                 /* LBK channel */
  46                 link = 12;
  47         }
  48
  49         return link;
  50 }
  51
  52 static uint8_t
  53 nix_get_relchan(struct otx2_eth_dev *dev)
  54 {
  55         return dev->tx_chan_base & 0xff;
  56 }
  57
  58 static bool
  59 nix_tm_have_tl1_access(struct otx2_eth_dev *dev)
  60 {
  61         bool is_lbk = otx2_dev_is_lbk(dev);
  62         return otx2_dev_is_pf(dev) && !otx2_dev_is_A0(dev) &&
  63                 !is_lbk && !dev->maxvf;
  64 }
  65
  66 static int
  67 find_prio_anchor(struct otx2_eth_dev *dev, uint32_t node_id)
  68 {
  69         struct otx2_nix_tm_node *child_node;
  70
  71         TAILQ_FOREACH(child_node, &dev->node_list, node) {
  72                 if (!child_node->parent)
  73                         continue;
  74                 if (!(child_node->parent->id == node_id))
  75                         continue;
  76                 if (child_node->priority == child_node->parent->rr_prio)
  77                         continue;
  78                 return child_node->hw_id - child_node->priority;
  79         }
  80         return 0;
  81 }
  82
  83
  84 static struct otx2_nix_tm_shaper_profile *
  85 nix_tm_shaper_profile_search(struct otx2_eth_dev *dev, uint32_t shaper_id)
  86 {
  87         struct otx2_nix_tm_shaper_profile *tm_shaper_profile;
  88
  89         TAILQ_FOREACH(tm_shaper_profile, &dev->shaper_profile_list, shaper) {
  90                 if (tm_shaper_profile->shaper_profile_id == shaper_id)
  91                         return tm_shaper_profile;
  92         }
  93         return NULL;
  94 }
  95
  96 static inline uint64_t
  97 shaper_rate_to_nix(uint64_t cclk_hz, uint64_t cclk_ticks,
  98                    uint64_t value, uint64_t *exponent_p,
  99                    uint64_t *mantissa_p, uint64_t *div_exp_p)
 100 {
 101         uint64_t div_exp, exponent, mantissa;
 102
 103         /* Boundary checks */
 104         if (value < MIN_SHAPER_RATE(cclk_hz, cclk_ticks) ||
 105             value > MAX_SHAPER_RATE(cclk_hz, cclk_ticks))
 106                 return 0;
 107
 108         if (value <= SHAPER_RATE(cclk_hz, cclk_ticks, 0, 0, 0)) {
 109                 /* Calculate rate div_exp and mantissa using
 110                  * the following formula:
 111                  *
 112                  * value = (cclk_hz * (256 + mantissa)
 113                  *              / ((cclk_ticks << div_exp) * 256)
 114                  */
 115                 div_exp = 0;
 116                 exponent = 0;
 117                 mantissa = MAX_RATE_MANTISSA;
 118
 119                 while (value < (cclk_hz / (cclk_ticks << div_exp)))
 120                         div_exp += 1;
 121
 122                 while (value <
 123                        ((cclk_hz * (256 + mantissa)) /
 124                         ((cclk_ticks << div_exp) * 256)))
 125                         mantissa -= 1;
 126         } else {
 127                 /* Calculate rate exponent and mantissa using
 128                  * the following formula:
 129                  *
 130                  * value = (cclk_hz * ((256 + mantissa) << exponent)
 131                  *              / (cclk_ticks * 256)
 132                  *
 133                  */
 134                 div_exp = 0;
 135                 exponent = MAX_RATE_EXPONENT;
 136                 mantissa = MAX_RATE_MANTISSA;
 137
 138                 while (value < (cclk_hz * (1 << exponent)) / cclk_ticks)
 139                         exponent -= 1;
 140
 141                 while (value < (cclk_hz * ((256 + mantissa) << exponent)) /
 142                        (cclk_ticks * 256))
 143                         mantissa -= 1;
 144         }
 145
 146         if (div_exp > MAX_RATE_DIV_EXP ||
 147             exponent > MAX_RATE_EXPONENT || mantissa > MAX_RATE_MANTISSA)
 148                 return 0;
 149
 150         if (div_exp_p)
 151                 *div_exp_p = div_exp;
 152         if (exponent_p)
 153                 *exponent_p = exponent;
 154         if (mantissa_p)
 155                 *mantissa_p = mantissa;
 156
 157         /* Calculate real rate value */
 158         return SHAPER_RATE(cclk_hz, cclk_ticks, exponent, mantissa, div_exp);
 159 }
 160
 161 static inline uint64_t
 162 lx_shaper_rate_to_nix(uint64_t cclk_hz, uint32_t hw_lvl,
 163                       uint64_t value, uint64_t *exponent,
 164                       uint64_t *mantissa, uint64_t *div_exp)
 165 {
 166         if (hw_lvl == NIX_TXSCH_LVL_TL1)
 167                 return shaper_rate_to_nix(cclk_hz, L1_TIME_WHEEL_CCLK_TICKS,
 168                                           value, exponent, mantissa, div_exp);
 169         else
 170                 return shaper_rate_to_nix(cclk_hz, LX_TIME_WHEEL_CCLK_TICKS,
 171                                           value, exponent, mantissa, div_exp);
 172 }
 173
 174 static inline uint64_t
 175 shaper_burst_to_nix(uint64_t value, uint64_t *exponent_p,
 176                     uint64_t *mantissa_p)
 177 {
 178         uint64_t exponent, mantissa;
 179
 180         if (value < MIN_SHAPER_BURST || value > MAX_SHAPER_BURST)
 181                 return 0;
 182
 183         /* Calculate burst exponent and mantissa using
 184          * the following formula:
 185          *
 186          * value = (((256 + mantissa) << (exponent + 1)
 187          / 256)
 188          *
 189          */
 190         exponent = MAX_BURST_EXPONENT;
 191         mantissa = MAX_BURST_MANTISSA;
 192
 193         while (value < (1ull << (exponent + 1)))
 194                 exponent -= 1;
 195
 196         while (value < ((256 + mantissa) << (exponent + 1)) / 256)
 197                 mantissa -= 1;
 198
 199         if (exponent > MAX_BURST_EXPONENT || mantissa > MAX_BURST_MANTISSA)
 200                 return 0;
 201
 202         if (exponent_p)
 203                 *exponent_p = exponent;
 204         if (mantissa_p)
 205                 *mantissa_p = mantissa;
 206
 207         return SHAPER_BURST(exponent, mantissa);
 208 }
 209
 210 static int
 211 configure_shaper_cir_pir_reg(struct otx2_eth_dev *dev,
 212                              struct otx2_nix_tm_node *tm_node,
 213                              struct shaper_params *cir,
 214                              struct shaper_params *pir)
 215 {
 216         uint32_t shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
 217         struct otx2_nix_tm_shaper_profile *shaper_profile = NULL;
 218         struct rte_tm_shaper_params *param;
 219
 220         shaper_profile_id = tm_node->params.shaper_profile_id;
 221
 222         shaper_profile = nix_tm_shaper_profile_search(dev, shaper_profile_id);
 223         if (shaper_profile) {
 224                 param = &shaper_profile->profile;
 225                 /* Calculate CIR exponent and mantissa */
 226                 if (param->committed.rate)
 227                         cir->rate = lx_shaper_rate_to_nix(CCLK_HZ,
 228                                                           tm_node->hw_lvl_id,
 229                                                           param->committed.rate,
 230                                                           &cir->exponent,
 231                                                           &cir->mantissa,
 232                                                           &cir->div_exp);
 233
 234                 /* Calculate PIR exponent and mantissa */
 235                 if (param->peak.rate)
 236                         pir->rate = lx_shaper_rate_to_nix(CCLK_HZ,
 237                                                           tm_node->hw_lvl_id,
 238                                                           param->peak.rate,
 239                                                           &pir->exponent,
 240                                                           &pir->mantissa,
 241                                                           &pir->div_exp);
 242
 243                 /* Calculate CIR burst exponent and mantissa */
 244                 if (param->committed.size)
 245                         cir->burst = shaper_burst_to_nix(param->committed.size,
 246                                                          &cir->burst_exponent,
 247                                                          &cir->burst_mantissa);
 248
 249                 /* Calculate PIR burst exponent and mantissa */
 250                 if (param->peak.size)
 251                         pir->burst = shaper_burst_to_nix(param->peak.size,
 252                                                          &pir->burst_exponent,
 253                                                          &pir->burst_mantissa);
 254         }
 255
 256         return 0;
 257 }
 258
 259 static int
 260 send_tm_reqval(struct otx2_mbox *mbox, struct nix_txschq_config *req)
 261 {
 262         int rc;
 263
 264         if (req->num_regs > MAX_REGS_PER_MBOX_MSG)
 265                 return -ERANGE;
 266
 267         rc = otx2_mbox_process(mbox);
 268         if (rc)
 269                 return rc;
 270
 271         req->num_regs = 0;
 272         return 0;
 273 }
 274
 275 static int
 276 populate_tm_registers(struct otx2_eth_dev *dev,
 277                       struct otx2_nix_tm_node *tm_node)
 278 {
 279         uint64_t strict_schedul_prio, rr_prio;
 280         struct otx2_mbox *mbox = dev->mbox;
 281         volatile uint64_t *reg, *regval;
 282         uint64_t parent = 0, child = 0;
 283         struct shaper_params cir, pir;
 284         struct nix_txschq_config *req;
 285         uint64_t rr_quantum;
 286         uint32_t hw_lvl;
 287         uint32_t schq;
 288         int rc;
 289
 290         memset(&cir, 0, sizeof(cir));
 291         memset(&pir, 0, sizeof(pir));
 292
 293         /* Skip leaf nodes */
 294         if (tm_node->hw_lvl_id == NIX_TXSCH_LVL_CNT)
 295                 return 0;
 296
 297         /* Root node will not have a parent node */
 298         if (tm_node->hw_lvl_id == dev->otx2_tm_root_lvl)
 299                 parent = tm_node->parent_hw_id;
 300         else
 301                 parent = tm_node->parent->hw_id;
 302
 303         /* Do we need this trigger to configure TL1 */
 304         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
 305             tm_node->hw_lvl_id == dev->otx2_tm_root_lvl) {
 306                 schq = parent;
 307                 /*
 308                  * Default config for TL1.
 309                  * For VF this is always ignored.
 310                  */
 311
 312                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 313                 req->lvl = NIX_TXSCH_LVL_TL1;
 314
 315                 /* Set DWRR quantum */
 316                 req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq);
 317                 req->regval[0] = TXSCH_TL1_DFLT_RR_QTM;
 318                 req->num_regs++;
 319
 320                 req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq);
 321                 req->regval[1] = (TXSCH_TL1_DFLT_RR_PRIO << 1);
 322                 req->num_regs++;
 323
 324                 req->reg[2] = NIX_AF_TL1X_CIR(schq);
 325                 req->regval[2] = 0;
 326                 req->num_regs++;
 327
 328                 rc = send_tm_reqval(mbox, req);
 329                 if (rc)
 330                         goto error;
 331         }
 332
 333         if (tm_node->hw_lvl_id != NIX_TXSCH_LVL_SMQ)
 334                 child = find_prio_anchor(dev, tm_node->id);
 335
 336         rr_prio = tm_node->rr_prio;
 337         hw_lvl = tm_node->hw_lvl_id;
 338         strict_schedul_prio = tm_node->priority;
 339         schq = tm_node->hw_id;
 340         rr_quantum = (tm_node->weight * NIX_TM_RR_QUANTUM_MAX) /
 341                 MAX_SCHED_WEIGHT;
 342
 343         configure_shaper_cir_pir_reg(dev, tm_node, &cir, &pir);
 344
 345         otx2_tm_dbg("Configure node %p, lvl %u hw_lvl %u, id %u, hw_id %u,"
 346                      "parent_hw_id %" PRIx64 ", pir %" PRIx64 ", cir %" PRIx64,
 347                      tm_node, tm_node->level_id, hw_lvl,
 348                      tm_node->id, schq, parent, pir.rate, cir.rate);
 349
 350         rc = -EFAULT;
 351
 352         switch (hw_lvl) {
 353         case NIX_TXSCH_LVL_SMQ:
 354                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 355                 req->lvl = hw_lvl;
 356                 reg = req->reg;
 357                 regval = req->regval;
 358                 req->num_regs = 0;
 359
 360                 /* Set xoff which will be cleared later */
 361                 *reg++ = NIX_AF_SMQX_CFG(schq);
 362                 *regval++ = BIT_ULL(50) | ((uint64_t)NIX_MAX_VTAG_INS << 36) |
 363                                 (NIX_MAX_HW_FRS << 8) | NIX_MIN_HW_FRS;
 364                 req->num_regs++;
 365                 *reg++ = NIX_AF_MDQX_PARENT(schq);
 366                 *regval++ = parent << 16;
 367                 req->num_regs++;
 368                 *reg++ = NIX_AF_MDQX_SCHEDULE(schq);
 369                 *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 370                 req->num_regs++;
 371                 if (pir.rate && pir.burst) {
 372                         *reg++ = NIX_AF_MDQX_PIR(schq);
 373                         *regval++ = shaper2regval(&pir) | 1;
 374                         req->num_regs++;
 375                 }
 376
 377                 if (cir.rate && cir.burst) {
 378                         *reg++ = NIX_AF_MDQX_CIR(schq);
 379                         *regval++ = shaper2regval(&cir) | 1;
 380                         req->num_regs++;
 381                 }
 382
 383                 rc = send_tm_reqval(mbox, req);
 384                 if (rc)
 385                         goto error;
 386                 break;
 387         case NIX_TXSCH_LVL_TL4:
 388                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 389                 req->lvl = hw_lvl;
 390                 req->num_regs = 0;
 391                 reg = req->reg;
 392                 regval = req->regval;
 393
 394                 *reg++ = NIX_AF_TL4X_PARENT(schq);
 395                 *regval++ = parent << 16;
 396                 req->num_regs++;
 397                 *reg++ = NIX_AF_TL4X_TOPOLOGY(schq);
 398                 *regval++ = (child << 32) | (rr_prio << 1);
 399                 req->num_regs++;
 400                 *reg++ = NIX_AF_TL4X_SCHEDULE(schq);
 401                 *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 402                 req->num_regs++;
 403                 if (pir.rate && pir.burst) {
 404                         *reg++ = NIX_AF_TL4X_PIR(schq);
 405                         *regval++ = shaper2regval(&pir) | 1;
 406                         req->num_regs++;
 407                 }
 408                 if (cir.rate && cir.burst) {
 409                         *reg++ = NIX_AF_TL4X_CIR(schq);
 410                         *regval++ = shaper2regval(&cir) | 1;
 411                         req->num_regs++;
 412                 }
 413
 414                 rc = send_tm_reqval(mbox, req);
 415                 if (rc)
 416                         goto error;
 417                 break;
 418         case NIX_TXSCH_LVL_TL3:
 419                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 420                 req->lvl = hw_lvl;
 421                 req->num_regs = 0;
 422                 reg = req->reg;
 423                 regval = req->regval;
 424
 425                 *reg++ = NIX_AF_TL3X_PARENT(schq);
 426                 *regval++ = parent << 16;
 427                 req->num_regs++;
 428                 *reg++ = NIX_AF_TL3X_TOPOLOGY(schq);
 429                 *regval++ = (child << 32) | (rr_prio << 1);
 430                 req->num_regs++;
 431                 *reg++ = NIX_AF_TL3X_SCHEDULE(schq);
 432                 *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 433                 req->num_regs++;
 434                 if (pir.rate && pir.burst) {
 435                         *reg++ = NIX_AF_TL3X_PIR(schq);
 436                         *regval++ = shaper2regval(&pir) | 1;
 437                         req->num_regs++;
 438                 }
 439                 if (cir.rate && cir.burst) {
 440                         *reg++ = NIX_AF_TL3X_CIR(schq);
 441                         *regval++ = shaper2regval(&cir) | 1;
 442                         req->num_regs++;
 443                 }
 444
 445                 rc = send_tm_reqval(mbox, req);
 446                 if (rc)
 447                         goto error;
 448                 break;
 449         case NIX_TXSCH_LVL_TL2:
 450                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 451                 req->lvl = hw_lvl;
 452                 req->num_regs = 0;
 453                 reg = req->reg;
 454                 regval = req->regval;
 455
 456                 *reg++ = NIX_AF_TL2X_PARENT(schq);
 457                 *regval++ = parent << 16;
 458                 req->num_regs++;
 459                 *reg++ = NIX_AF_TL2X_TOPOLOGY(schq);
 460                 *regval++ = (child << 32) | (rr_prio << 1);
 461                 req->num_regs++;
 462                 *reg++ = NIX_AF_TL2X_SCHEDULE(schq);
 463                 if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2)
 464                         *regval++ = (1 << 24) | rr_quantum;
 465                 else
 466                         *regval++ = (strict_schedul_prio << 24) | rr_quantum;
 467                 req->num_regs++;
 468                 *reg++ = NIX_AF_TL3_TL2X_LINKX_CFG(schq, nix_get_link(dev));
 469                 *regval++ = BIT_ULL(12) | nix_get_relchan(dev);
 470                 req->num_regs++;
 471                 if (pir.rate && pir.burst) {
 472                         *reg++ = NIX_AF_TL2X_PIR(schq);
 473                         *regval++ = shaper2regval(&pir) | 1;
 474                         req->num_regs++;
 475                 }
 476                 if (cir.rate && cir.burst) {
 477                         *reg++ = NIX_AF_TL2X_CIR(schq);
 478                         *regval++ = shaper2regval(&cir) | 1;
 479                         req->num_regs++;
 480                 }
 481
 482                 rc = send_tm_reqval(mbox, req);
 483                 if (rc)
 484                         goto error;
 485                 break;
 486         case NIX_TXSCH_LVL_TL1:
 487                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 488                 req->lvl = hw_lvl;
 489                 req->num_regs = 0;
 490                 reg = req->reg;
 491                 regval = req->regval;
 492
 493                 *reg++ = NIX_AF_TL1X_SCHEDULE(schq);
 494                 *regval++ = rr_quantum;
 495                 req->num_regs++;
 496                 *reg++ = NIX_AF_TL1X_TOPOLOGY(schq);
 497                 *regval++ = (child << 32) | (rr_prio << 1 /*RR_PRIO*/);
 498                 req->num_regs++;
 499                 if (cir.rate && cir.burst) {
 500                         *reg++ = NIX_AF_TL1X_CIR(schq);
 501                         *regval++ = shaper2regval(&cir) | 1;
 502                         req->num_regs++;
 503                 }
 504
 505                 rc = send_tm_reqval(mbox, req);
 506                 if (rc)
 507                         goto error;
 508                 break;
 509         }
 510
 511         return 0;
 512 error:
 513         otx2_err("Txschq cfg request failed for node %p, rc=%d", tm_node, rc);
 514         return rc;
 515 }
 516
 517
 518 static int
 519 nix_tm_txsch_reg_config(struct otx2_eth_dev *dev)
 520 {
 521         struct otx2_nix_tm_node *tm_node;
 522         uint32_t lvl;
 523         int rc = 0;
 524
 525         if (nix_get_link(dev) == 13)
 526                 return -EPERM;
 527
 528         for (lvl = 0; lvl < (uint32_t)dev->otx2_tm_root_lvl + 1; lvl++) {
 529                 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 530                         if (tm_node->hw_lvl_id == lvl) {
 531                                 rc = populate_tm_registers(dev, tm_node);
 532                                 if (rc)
 533                                         goto exit;
 534                         }
 535                 }
 536         }
 537 exit:
 538         return rc;
 539 }
 540
 541 static struct otx2_nix_tm_node *
 542 nix_tm_node_search(struct otx2_eth_dev *dev,
 543                    uint32_t node_id, bool user)
 544 {
 545         struct otx2_nix_tm_node *tm_node;
 546
 547         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 548                 if (tm_node->id == node_id &&
 549                     (user == !!(tm_node->flags & NIX_TM_NODE_USER)))
 550                         return tm_node;
 551         }
 552         return NULL;
 553 }
 554
 555 static uint32_t
 556 check_rr(struct otx2_eth_dev *dev, uint32_t priority, uint32_t parent_id)
 557 {
 558         struct otx2_nix_tm_node *tm_node;
 559         uint32_t rr_num = 0;
 560
 561         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 562                 if (!tm_node->parent)
 563                         continue;
 564
 565                 if (!(tm_node->parent->id == parent_id))
 566                         continue;
 567
 568                 if (tm_node->priority == priority)
 569                         rr_num++;
 570         }
 571         return rr_num;
 572 }
 573
 574 static int
 575 nix_tm_update_parent_info(struct otx2_eth_dev *dev)
 576 {
 577         struct otx2_nix_tm_node *tm_node_child;
 578         struct otx2_nix_tm_node *tm_node;
 579         struct otx2_nix_tm_node *parent;
 580         uint32_t rr_num = 0;
 581         uint32_t priority;
 582
 583         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 584                 if (!tm_node->parent)
 585                         continue;
 586                 /* Count group of children of same priority i.e are RR */
 587                 parent = tm_node->parent;
 588                 priority = tm_node->priority;
 589                 rr_num = check_rr(dev, priority, parent->id);
 590
 591                 /* Assuming that multiple RR groups are
 592                  * not configured based on capability.
 593                  */
 594                 if (rr_num > 1) {
 595                         parent->rr_prio = priority;
 596                         parent->rr_num = rr_num;
 597                 }
 598
 599                 /* Find out static priority children that are not in RR */
 600                 TAILQ_FOREACH(tm_node_child, &dev->node_list, node) {
 601                         if (!tm_node_child->parent)
 602                                 continue;
 603                         if (parent->id != tm_node_child->parent->id)
 604                                 continue;
 605                         if (parent->max_prio == UINT32_MAX &&
 606                             tm_node_child->priority != parent->rr_prio)
 607                                 parent->max_prio = 0;
 608
 609                         if (parent->max_prio < tm_node_child->priority &&
 610                             parent->rr_prio != tm_node_child->priority)
 611                                 parent->max_prio = tm_node_child->priority;
 612                 }
 613         }
 614
 615         return 0;
 616 }
 617
 618 static int
 619 nix_tm_node_add_to_list(struct otx2_eth_dev *dev, uint32_t node_id,
 620                         uint32_t parent_node_id, uint32_t priority,
 621                         uint32_t weight, uint16_t hw_lvl_id,
 622                         uint16_t level_id, bool user,
 623                         struct rte_tm_node_params *params)
 624 {
 625         struct otx2_nix_tm_shaper_profile *shaper_profile;
 626         struct otx2_nix_tm_node *tm_node, *parent_node;
 627         uint32_t shaper_profile_id;
 628
 629         shaper_profile_id = params->shaper_profile_id;
 630         shaper_profile = nix_tm_shaper_profile_search(dev, shaper_profile_id);
 631
 632         parent_node = nix_tm_node_search(dev, parent_node_id, user);
 633
 634         tm_node = rte_zmalloc("otx2_nix_tm_node",
 635                               sizeof(struct otx2_nix_tm_node), 0);
 636         if (!tm_node)
 637                 return -ENOMEM;
 638
 639         tm_node->level_id = level_id;
 640         tm_node->hw_lvl_id = hw_lvl_id;
 641
 642         tm_node->id = node_id;
 643         tm_node->priority = priority;
 644         tm_node->weight = weight;
 645         tm_node->rr_prio = 0xf;
 646         tm_node->max_prio = UINT32_MAX;
 647         tm_node->hw_id = UINT32_MAX;
 648         tm_node->flags = 0;
 649         if (user)
 650                 tm_node->flags = NIX_TM_NODE_USER;
 651         rte_memcpy(&tm_node->params, params, sizeof(struct rte_tm_node_params));
 652
 653         if (shaper_profile)
 654                 shaper_profile->reference_count++;
 655         tm_node->parent = parent_node;
 656         tm_node->parent_hw_id = UINT32_MAX;
 657
 658         TAILQ_INSERT_TAIL(&dev->node_list, tm_node, node);
 659
 660         return 0;
 661 }
 662
 663 static int
 664 nix_tm_clear_shaper_profiles(struct otx2_eth_dev *dev)
 665 {
 666         struct otx2_nix_tm_shaper_profile *shaper_profile;
 667
 668         while ((shaper_profile = TAILQ_FIRST(&dev->shaper_profile_list))) {
 669                 if (shaper_profile->reference_count)
 670                         otx2_tm_dbg("Shaper profile %u has non zero references",
 671                                     shaper_profile->shaper_profile_id);
 672                 TAILQ_REMOVE(&dev->shaper_profile_list, shaper_profile, shaper);
 673                 rte_free(shaper_profile);
 674         }
 675
 676         return 0;
 677 }
 678
 679 static int
 680 nix_tm_free_resources(struct otx2_eth_dev *dev, uint32_t flags_mask,
 681                       uint32_t flags, bool hw_only)
 682 {
 683         struct otx2_nix_tm_shaper_profile *shaper_profile;
 684         struct otx2_nix_tm_node *tm_node, *next_node;
 685         struct otx2_mbox *mbox = dev->mbox;
 686         struct nix_txsch_free_req *req;
 687         uint32_t shaper_profile_id;
 688         bool skip_node = false;
 689         int rc = 0;
 690
 691         next_node = TAILQ_FIRST(&dev->node_list);
 692         while (next_node) {
 693                 tm_node = next_node;
 694                 next_node = TAILQ_NEXT(tm_node, node);
 695
 696                 /* Check for only requested nodes */
 697                 if ((tm_node->flags & flags_mask) != flags)
 698                         continue;
 699
 700                 if (nix_tm_have_tl1_access(dev) &&
 701                     tm_node->hw_lvl_id ==  NIX_TXSCH_LVL_TL1)
 702                         skip_node = true;
 703
 704                 otx2_tm_dbg("Free hwres for node %u, hwlvl %u, hw_id %u (%p)",
 705                             tm_node->id,  tm_node->hw_lvl_id,
 706                             tm_node->hw_id, tm_node);
 707                 /* Free specific HW resource if requested */
 708                 if (!skip_node && flags_mask &&
 709                     tm_node->flags & NIX_TM_NODE_HWRES) {
 710                         req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
 711                         req->flags = 0;
 712                         req->schq_lvl = tm_node->hw_lvl_id;
 713                         req->schq = tm_node->hw_id;
 714                         rc = otx2_mbox_process(mbox);
 715                         if (rc)
 716                                 break;
 717                 } else {
 718                         skip_node = false;
 719                 }
 720                 tm_node->flags &= ~NIX_TM_NODE_HWRES;
 721
 722                 /* Leave software elements if needed */
 723                 if (hw_only)
 724                         continue;
 725
 726                 shaper_profile_id = tm_node->params.shaper_profile_id;
 727                 shaper_profile =
 728                         nix_tm_shaper_profile_search(dev, shaper_profile_id);
 729                 if (shaper_profile)
 730                         shaper_profile->reference_count--;
 731
 732                 TAILQ_REMOVE(&dev->node_list, tm_node, node);
 733                 rte_free(tm_node);
 734         }
 735
 736         if (!flags_mask) {
 737                 /* Free all hw resources */
 738                 req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
 739                 req->flags = TXSCHQ_FREE_ALL;
 740
 741                 return otx2_mbox_process(mbox);
 742         }
 743
 744         return rc;
 745 }
 746
 747 static uint8_t
 748 nix_tm_copy_rsp_to_dev(struct otx2_eth_dev *dev,
 749                        struct nix_txsch_alloc_rsp *rsp)
 750 {
 751         uint16_t schq;
 752         uint8_t lvl;
 753
 754         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
 755                 for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) {
 756                         dev->txschq_list[lvl][schq] = rsp->schq_list[lvl][schq];
 757                         dev->txschq_contig_list[lvl][schq] =
 758                                 rsp->schq_contig_list[lvl][schq];
 759                 }
 760
 761                 dev->txschq[lvl] = rsp->schq[lvl];
 762                 dev->txschq_contig[lvl] = rsp->schq_contig[lvl];
 763         }
 764         return 0;
 765 }
 766
 767 static int
 768 nix_tm_assign_id_to_node(struct otx2_eth_dev *dev,
 769                          struct otx2_nix_tm_node *child,
 770                          struct otx2_nix_tm_node *parent)
 771 {
 772         uint32_t hw_id, schq_con_index, prio_offset;
 773         uint32_t l_id, schq_index;
 774
 775         otx2_tm_dbg("Assign hw id for child node %u, lvl %u, hw_lvl %u (%p)",
 776                     child->id, child->level_id, child->hw_lvl_id, child);
 777
 778         child->flags |= NIX_TM_NODE_HWRES;
 779
 780         /* Process root nodes */
 781         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
 782             child->hw_lvl_id == dev->otx2_tm_root_lvl && !parent) {
 783                 int idx = 0;
 784                 uint32_t tschq_con_index;
 785
 786                 l_id = child->hw_lvl_id;
 787                 tschq_con_index = dev->txschq_contig_index[l_id];
 788                 hw_id = dev->txschq_contig_list[l_id][tschq_con_index];
 789                 child->hw_id = hw_id;
 790                 dev->txschq_contig_index[l_id]++;
 791                 /* Update TL1 hw_id for its parent for config purpose */
 792                 idx = dev->txschq_index[NIX_TXSCH_LVL_TL1]++;
 793                 hw_id = dev->txschq_list[NIX_TXSCH_LVL_TL1][idx];
 794                 child->parent_hw_id = hw_id;
 795                 return 0;
 796         }
 797         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL1 &&
 798             child->hw_lvl_id == dev->otx2_tm_root_lvl && !parent) {
 799                 uint32_t tschq_con_index;
 800
 801                 l_id = child->hw_lvl_id;
 802                 tschq_con_index = dev->txschq_index[l_id];
 803                 hw_id = dev->txschq_list[l_id][tschq_con_index];
 804                 child->hw_id = hw_id;
 805                 dev->txschq_index[l_id]++;
 806                 return 0;
 807         }
 808
 809         /* Process children with parents */
 810         l_id = child->hw_lvl_id;
 811         schq_index = dev->txschq_index[l_id];
 812         schq_con_index = dev->txschq_contig_index[l_id];
 813
 814         if (child->priority == parent->rr_prio) {
 815                 hw_id = dev->txschq_list[l_id][schq_index];
 816                 child->hw_id = hw_id;
 817                 child->parent_hw_id = parent->hw_id;
 818                 dev->txschq_index[l_id]++;
 819         } else {
 820                 prio_offset = schq_con_index + child->priority;
 821                 hw_id = dev->txschq_contig_list[l_id][prio_offset];
 822                 child->hw_id = hw_id;
 823         }
 824         return 0;
 825 }
 826
 827 static int
 828 nix_tm_assign_hw_id(struct otx2_eth_dev *dev)
 829 {
 830         struct otx2_nix_tm_node *parent, *child;
 831         uint32_t child_hw_lvl, con_index_inc, i;
 832
 833         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--) {
 834                 TAILQ_FOREACH(parent, &dev->node_list, node) {
 835                         child_hw_lvl = parent->hw_lvl_id - 1;
 836                         if (parent->hw_lvl_id != i)
 837                                 continue;
 838                         TAILQ_FOREACH(child, &dev->node_list, node) {
 839                                 if (!child->parent)
 840                                         continue;
 841                                 if (child->parent->id != parent->id)
 842                                         continue;
 843                                 nix_tm_assign_id_to_node(dev, child, parent);
 844                         }
 845
 846                         con_index_inc = parent->max_prio + 1;
 847                         dev->txschq_contig_index[child_hw_lvl] += con_index_inc;
 848
 849                         /*
 850                          * Explicitly assign id to parent node if it
 851                          * doesn't have a parent
 852                          */
 853                         if (parent->hw_lvl_id == dev->otx2_tm_root_lvl)
 854                                 nix_tm_assign_id_to_node(dev, parent, NULL);
 855                 }
 856         }
 857         return 0;
 858 }
 859
 860 static uint8_t
 861 nix_tm_count_req_schq(struct otx2_eth_dev *dev,
 862                       struct nix_txsch_alloc_req *req, uint8_t lvl)
 863 {
 864         struct otx2_nix_tm_node *tm_node;
 865         uint8_t contig_count;
 866
 867         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 868                 if (lvl == tm_node->hw_lvl_id) {
 869                         req->schq[lvl - 1] += tm_node->rr_num;
 870                         if (tm_node->max_prio != UINT32_MAX) {
 871                                 contig_count = tm_node->max_prio + 1;
 872                                 req->schq_contig[lvl - 1] += contig_count;
 873                         }
 874                 }
 875                 if (lvl == dev->otx2_tm_root_lvl &&
 876                     dev->otx2_tm_root_lvl && lvl == NIX_TXSCH_LVL_TL2 &&
 877                     tm_node->hw_lvl_id == dev->otx2_tm_root_lvl) {
 878                         req->schq_contig[dev->otx2_tm_root_lvl]++;
 879                 }
 880         }
 881
 882         req->schq[NIX_TXSCH_LVL_TL1] = 1;
 883         req->schq_contig[NIX_TXSCH_LVL_TL1] = 0;
 884
 885         return 0;
 886 }
 887
 888 static int
 889 nix_tm_prepare_txschq_req(struct otx2_eth_dev *dev,
 890                           struct nix_txsch_alloc_req *req)
 891 {
 892         uint8_t i;
 893
 894         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--)
 895                 nix_tm_count_req_schq(dev, req, i);
 896
 897         for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
 898                 dev->txschq_index[i] = 0;
 899                 dev->txschq_contig_index[i] = 0;
 900         }
 901         return 0;
 902 }
 903
 904 static int
 905 nix_tm_send_txsch_alloc_msg(struct otx2_eth_dev *dev)
 906 {
 907         struct otx2_mbox *mbox = dev->mbox;
 908         struct nix_txsch_alloc_req *req;
 909         struct nix_txsch_alloc_rsp *rsp;
 910         int rc;
 911
 912         req = otx2_mbox_alloc_msg_nix_txsch_alloc(mbox);
 913
 914         rc = nix_tm_prepare_txschq_req(dev, req);
 915         if (rc)
 916                 return rc;
 917
 918         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
 919         if (rc)
 920                 return rc;
 921
 922         nix_tm_copy_rsp_to_dev(dev, rsp);
 923
 924         nix_tm_assign_hw_id(dev);
 925         return 0;
 926 }
 927
 928 static int
 929 nix_tm_alloc_resources(struct rte_eth_dev *eth_dev, bool xmit_enable)
 930 {
 931         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 932         int rc;
 933
 934         RTE_SET_USED(xmit_enable);
 935
 936         nix_tm_update_parent_info(dev);
 937
 938         rc = nix_tm_send_txsch_alloc_msg(dev);
 939         if (rc) {
 940                 otx2_err("TM failed to alloc tm resources=%d", rc);
 941                 return rc;
 942         }
 943
 944         rc = nix_tm_txsch_reg_config(dev);
 945         if (rc) {
 946                 otx2_err("TM failed to configure sched registers=%d", rc);
 947                 return rc;
 948         }
 949
 950         return 0;
 951 }
 952
 953 static int
 954 nix_tm_prepare_default_tree(struct rte_eth_dev *eth_dev)
 955 {
 956         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 957         uint32_t def = eth_dev->data->nb_tx_queues;
 958         struct rte_tm_node_params params;
 959         uint32_t leaf_parent, i;
 960         int rc = 0;
 961
 962         /* Default params */
 963         memset(&params, 0, sizeof(params));
 964         params.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
 965
 966         if (nix_tm_have_tl1_access(dev)) {
 967                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL1;
 968                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
 969                                              DEFAULT_RR_WEIGHT,
 970                                              NIX_TXSCH_LVL_TL1,
 971                                              OTX2_TM_LVL_ROOT, false, &params);
 972                 if (rc)
 973                         goto exit;
 974                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
 975                                              DEFAULT_RR_WEIGHT,
 976                                              NIX_TXSCH_LVL_TL2,
 977                                              OTX2_TM_LVL_SCH1, false, &params);
 978                 if (rc)
 979                         goto exit;
 980
 981                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
 982                                              DEFAULT_RR_WEIGHT,
 983                                              NIX_TXSCH_LVL_TL3,
 984                                              OTX2_TM_LVL_SCH2, false, &params);
 985                 if (rc)
 986                         goto exit;
 987
 988                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
 989                                              DEFAULT_RR_WEIGHT,
 990                                              NIX_TXSCH_LVL_TL4,
 991                                              OTX2_TM_LVL_SCH3, false, &params);
 992                 if (rc)
 993                         goto exit;
 994
 995                 rc = nix_tm_node_add_to_list(dev, def + 4, def + 3, 0,
 996                                              DEFAULT_RR_WEIGHT,
 997                                              NIX_TXSCH_LVL_SMQ,
 998                                              OTX2_TM_LVL_SCH4, false, &params);
 999                 if (rc)
1000                         goto exit;
1001
1002                 leaf_parent = def + 4;
1003         } else {
1004                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL2;
1005                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
1006                                              DEFAULT_RR_WEIGHT,
1007                                              NIX_TXSCH_LVL_TL2,
1008                                              OTX2_TM_LVL_ROOT, false, &params);
1009                 if (rc)
1010                         goto exit;
1011
1012                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
1013                                              DEFAULT_RR_WEIGHT,
1014                                              NIX_TXSCH_LVL_TL3,
1015                                              OTX2_TM_LVL_SCH1, false, &params);
1016                 if (rc)
1017                         goto exit;
1018
1019                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
1020                                              DEFAULT_RR_WEIGHT,
1021                                              NIX_TXSCH_LVL_TL4,
1022                                              OTX2_TM_LVL_SCH2, false, &params);
1023                 if (rc)
1024                         goto exit;
1025
1026                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
1027                                              DEFAULT_RR_WEIGHT,
1028                                              NIX_TXSCH_LVL_SMQ,
1029                                              OTX2_TM_LVL_SCH3, false, &params);
1030                 if (rc)
1031                         goto exit;
1032
1033                 leaf_parent = def + 3;
1034         }
1035
1036         /* Add leaf nodes */
1037         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
1038                 rc = nix_tm_node_add_to_list(dev, i, leaf_parent, 0,
1039                                              DEFAULT_RR_WEIGHT,
1040                                              NIX_TXSCH_LVL_CNT,
1041                                              OTX2_TM_LVL_QUEUE, false, &params);
1042                 if (rc)
1043                         break;
1044         }
1045
1046 exit:
1047         return rc;
1048 }
1049
1050 void otx2_nix_tm_conf_init(struct rte_eth_dev *eth_dev)
1051 {
1052         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1053
1054         TAILQ_INIT(&dev->node_list);
1055         TAILQ_INIT(&dev->shaper_profile_list);
1056 }
1057
1058 int otx2_nix_tm_init_default(struct rte_eth_dev *eth_dev)
1059 {
1060         struct otx2_eth_dev  *dev = otx2_eth_pmd_priv(eth_dev);
1061         uint16_t sq_cnt = eth_dev->data->nb_tx_queues;
1062         int rc;
1063
1064         /* Free up all resources already held */
1065         rc = nix_tm_free_resources(dev, 0, 0, false);
1066         if (rc) {
1067                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
1068                 return rc;
1069         }
1070
1071         /* Clear shaper profiles */
1072         nix_tm_clear_shaper_profiles(dev);
1073         dev->tm_flags = NIX_TM_DEFAULT_TREE;
1074
1075         rc = nix_tm_prepare_default_tree(eth_dev);
1076         if (rc != 0)
1077                 return rc;
1078
1079         rc = nix_tm_alloc_resources(eth_dev, false);
1080         if (rc != 0)
1081                 return rc;
1082         dev->tm_leaf_cnt = sq_cnt;
1083
1084         return 0;
1085 }
1086
1087 int
1088 otx2_nix_tm_fini(struct rte_eth_dev *eth_dev)
1089 {
1090         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1091         int rc;
1092
1093         /* Xmit is assumed to be disabled */
1094         /* Free up resources already held */
1095         rc = nix_tm_free_resources(dev, 0, 0, false);
1096         if (rc) {
1097                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
1098                 return rc;
1099         }
1100
1101         /* Clear shaper profiles */
1102         nix_tm_clear_shaper_profiles(dev);
1103
1104         dev->tm_flags = 0;
1105         return 0;
1106 }