drivers/net/octeontx2/otx2_tm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <rte_malloc.h>
   6
   7 #include "otx2_ethdev.h"
   8 #include "otx2_tm.h"
   9
  10 /* Use last LVL_CNT nodes as default nodes */
  11 #define NIX_DEFAULT_NODE_ID_START (RTE_TM_NODE_ID_NULL - NIX_TXSCH_LVL_CNT)
  12
  13 enum otx2_tm_node_level {
  14         OTX2_TM_LVL_ROOT = 0,
  15         OTX2_TM_LVL_SCH1,
  16         OTX2_TM_LVL_SCH2,
  17         OTX2_TM_LVL_SCH3,
  18         OTX2_TM_LVL_SCH4,
  19         OTX2_TM_LVL_QUEUE,
  20         OTX2_TM_LVL_MAX,
  21 };
  22
  23 static inline
  24 uint64_t shaper2regval(struct shaper_params *shaper)
  25 {
  26         return (shaper->burst_exponent << 37) | (shaper->burst_mantissa << 29) |
  27                 (shaper->div_exp << 13) | (shaper->exponent << 9) |
  28                 (shaper->mantissa << 1);
  29 }
  30
  31 static int
  32 nix_get_link(struct otx2_eth_dev *dev)
  33 {
  34         int link = 13 /* SDP */;
  35         uint16_t lmac_chan;
  36         uint16_t map;
  37
  38         lmac_chan = dev->tx_chan_base;
  39
  40         /* CGX lmac link */
  41         if (lmac_chan >= 0x800) {
  42                 map = lmac_chan & 0x7FF;
  43                 link = 4 * ((map >> 8) & 0xF) + ((map >> 4) & 0xF);
  44         } else if (lmac_chan < 0x700) {
  45                 /* LBK channel */
  46                 link = 12;
  47         }
  48
  49         return link;
  50 }
  51
  52 static uint8_t
  53 nix_get_relchan(struct otx2_eth_dev *dev)
  54 {
  55         return dev->tx_chan_base & 0xff;
  56 }
  57
  58 static bool
  59 nix_tm_have_tl1_access(struct otx2_eth_dev *dev)
  60 {
  61         bool is_lbk = otx2_dev_is_lbk(dev);
  62         return otx2_dev_is_pf(dev) && !otx2_dev_is_Ax(dev) && !is_lbk;
  63 }
  64
  65 static bool
  66 nix_tm_is_leaf(struct otx2_eth_dev *dev, int lvl)
  67 {
  68         if (nix_tm_have_tl1_access(dev))
  69                 return (lvl == OTX2_TM_LVL_QUEUE);
  70
  71         return (lvl == OTX2_TM_LVL_SCH4);
  72 }
  73
  74 static int
  75 find_prio_anchor(struct otx2_eth_dev *dev, uint32_t node_id)
  76 {
  77         struct otx2_nix_tm_node *child_node;
  78
  79         TAILQ_FOREACH(child_node, &dev->node_list, node) {
  80                 if (!child_node->parent)
  81                         continue;
  82                 if (!(child_node->parent->id == node_id))
  83                         continue;
  84                 if (child_node->priority == child_node->parent->rr_prio)
  85                         continue;
  86                 return child_node->hw_id - child_node->priority;
  87         }
  88         return 0;
  89 }
  90
  91
  92 static struct otx2_nix_tm_shaper_profile *
  93 nix_tm_shaper_profile_search(struct otx2_eth_dev *dev, uint32_t shaper_id)
  94 {
  95         struct otx2_nix_tm_shaper_profile *tm_shaper_profile;
  96
  97         TAILQ_FOREACH(tm_shaper_profile, &dev->shaper_profile_list, shaper) {
  98                 if (tm_shaper_profile->shaper_profile_id == shaper_id)
  99                         return tm_shaper_profile;
 100         }
 101         return NULL;
 102 }
 103
 104 static inline uint64_t
 105 shaper_rate_to_nix(uint64_t value, uint64_t *exponent_p,
 106                    uint64_t *mantissa_p, uint64_t *div_exp_p)
 107 {
 108         uint64_t div_exp, exponent, mantissa;
 109
 110         /* Boundary checks */
 111         if (value < MIN_SHAPER_RATE ||
 112             value > MAX_SHAPER_RATE)
 113                 return 0;
 114
 115         if (value <= SHAPER_RATE(0, 0, 0)) {
 116                 /* Calculate rate div_exp and mantissa using
 117                  * the following formula:
 118                  *
 119                  * value = (2E6 * (256 + mantissa)
 120                  *              / ((1 << div_exp) * 256))
 121                  */
 122                 div_exp = 0;
 123                 exponent = 0;
 124                 mantissa = MAX_RATE_MANTISSA;
 125
 126                 while (value < (NIX_SHAPER_RATE_CONST / (1 << div_exp)))
 127                         div_exp += 1;
 128
 129                 while (value <
 130                        ((NIX_SHAPER_RATE_CONST * (256 + mantissa)) /
 131                         ((1 << div_exp) * 256)))
 132                         mantissa -= 1;
 133         } else {
 134                 /* Calculate rate exponent and mantissa using
 135                  * the following formula:
 136                  *
 137                  * value = (2E6 * ((256 + mantissa) << exponent)) / 256
 138                  *
 139                  */
 140                 div_exp = 0;
 141                 exponent = MAX_RATE_EXPONENT;
 142                 mantissa = MAX_RATE_MANTISSA;
 143
 144                 while (value < (NIX_SHAPER_RATE_CONST * (1 << exponent)))
 145                         exponent -= 1;
 146
 147                 while (value < ((NIX_SHAPER_RATE_CONST *
 148                                 ((256 + mantissa) << exponent)) / 256))
 149                         mantissa -= 1;
 150         }
 151
 152         if (div_exp > MAX_RATE_DIV_EXP ||
 153             exponent > MAX_RATE_EXPONENT || mantissa > MAX_RATE_MANTISSA)
 154                 return 0;
 155
 156         if (div_exp_p)
 157                 *div_exp_p = div_exp;
 158         if (exponent_p)
 159                 *exponent_p = exponent;
 160         if (mantissa_p)
 161                 *mantissa_p = mantissa;
 162
 163         /* Calculate real rate value */
 164         return SHAPER_RATE(exponent, mantissa, div_exp);
 165 }
 166
 167 static inline uint64_t
 168 shaper_burst_to_nix(uint64_t value, uint64_t *exponent_p,
 169                     uint64_t *mantissa_p)
 170 {
 171         uint64_t exponent, mantissa;
 172
 173         if (value < MIN_SHAPER_BURST || value > MAX_SHAPER_BURST)
 174                 return 0;
 175
 176         /* Calculate burst exponent and mantissa using
 177          * the following formula:
 178          *
 179          * value = (((256 + mantissa) << (exponent + 1)
 180          / 256)
 181          *
 182          */
 183         exponent = MAX_BURST_EXPONENT;
 184         mantissa = MAX_BURST_MANTISSA;
 185
 186         while (value < (1ull << (exponent + 1)))
 187                 exponent -= 1;
 188
 189         while (value < ((256 + mantissa) << (exponent + 1)) / 256)
 190                 mantissa -= 1;
 191
 192         if (exponent > MAX_BURST_EXPONENT || mantissa > MAX_BURST_MANTISSA)
 193                 return 0;
 194
 195         if (exponent_p)
 196                 *exponent_p = exponent;
 197         if (mantissa_p)
 198                 *mantissa_p = mantissa;
 199
 200         return SHAPER_BURST(exponent, mantissa);
 201 }
 202
 203 static void
 204 shaper_config_to_nix(struct otx2_nix_tm_shaper_profile *profile,
 205                      struct shaper_params *cir,
 206                      struct shaper_params *pir)
 207 {
 208         struct rte_tm_shaper_params *param = &profile->params;
 209
 210         if (!profile)
 211                 return;
 212
 213         /* Calculate CIR exponent and mantissa */
 214         if (param->committed.rate)
 215                 cir->rate = shaper_rate_to_nix(param->committed.rate,
 216                                                &cir->exponent,
 217                                                &cir->mantissa,
 218                                                &cir->div_exp);
 219
 220         /* Calculate PIR exponent and mantissa */
 221         if (param->peak.rate)
 222                 pir->rate = shaper_rate_to_nix(param->peak.rate,
 223                                                &pir->exponent,
 224                                                &pir->mantissa,
 225                                                &pir->div_exp);
 226
 227         /* Calculate CIR burst exponent and mantissa */
 228         if (param->committed.size)
 229                 cir->burst = shaper_burst_to_nix(param->committed.size,
 230                                                  &cir->burst_exponent,
 231                                                  &cir->burst_mantissa);
 232
 233         /* Calculate PIR burst exponent and mantissa */
 234         if (param->peak.size)
 235                 pir->burst = shaper_burst_to_nix(param->peak.size,
 236                                                  &pir->burst_exponent,
 237                                                  &pir->burst_mantissa);
 238 }
 239
 240 static int
 241 populate_tm_tl1_default(struct otx2_eth_dev *dev, uint32_t schq)
 242 {
 243         struct otx2_mbox *mbox = dev->mbox;
 244         struct nix_txschq_config *req;
 245
 246         /*
 247          * Default config for TL1.
 248          * For VF this is always ignored.
 249          */
 250
 251         req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 252         req->lvl = NIX_TXSCH_LVL_TL1;
 253
 254         /* Set DWRR quantum */
 255         req->reg[0] = NIX_AF_TL1X_SCHEDULE(schq);
 256         req->regval[0] = TXSCH_TL1_DFLT_RR_QTM;
 257         req->num_regs++;
 258
 259         req->reg[1] = NIX_AF_TL1X_TOPOLOGY(schq);
 260         req->regval[1] = (TXSCH_TL1_DFLT_RR_PRIO << 1);
 261         req->num_regs++;
 262
 263         req->reg[2] = NIX_AF_TL1X_CIR(schq);
 264         req->regval[2] = 0;
 265         req->num_regs++;
 266
 267         return otx2_mbox_process(mbox);
 268 }
 269
 270 static uint8_t
 271 prepare_tm_sched_reg(struct otx2_eth_dev *dev,
 272                      struct otx2_nix_tm_node *tm_node,
 273                      volatile uint64_t *reg, volatile uint64_t *regval)
 274 {
 275         uint64_t strict_prio = tm_node->priority;
 276         uint32_t hw_lvl = tm_node->hw_lvl;
 277         uint32_t schq = tm_node->hw_id;
 278         uint64_t rr_quantum;
 279         uint8_t k = 0;
 280
 281         rr_quantum = NIX_TM_WEIGHT_TO_RR_QUANTUM(tm_node->weight);
 282
 283         /* For children to root, strict prio is default if either
 284          * device root is TL2 or TL1 Static Priority is disabled.
 285          */
 286         if (hw_lvl == NIX_TXSCH_LVL_TL2 &&
 287             (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 ||
 288              dev->tm_flags & NIX_TM_TL1_NO_SP))
 289                 strict_prio = TXSCH_TL1_DFLT_RR_PRIO;
 290
 291         otx2_tm_dbg("Schedule config node %s(%u) lvl %u id %u, "
 292                      "prio 0x%" PRIx64 ", rr_quantum 0x%" PRIx64 " (%p)",
 293                      nix_hwlvl2str(tm_node->hw_lvl), schq, tm_node->lvl,
 294                      tm_node->id, strict_prio, rr_quantum, tm_node);
 295
 296         switch (hw_lvl) {
 297         case NIX_TXSCH_LVL_SMQ:
 298                 reg[k] = NIX_AF_MDQX_SCHEDULE(schq);
 299                 regval[k] = (strict_prio << 24) | rr_quantum;
 300                 k++;
 301
 302                 break;
 303         case NIX_TXSCH_LVL_TL4:
 304                 reg[k] = NIX_AF_TL4X_SCHEDULE(schq);
 305                 regval[k] = (strict_prio << 24) | rr_quantum;
 306                 k++;
 307
 308                 break;
 309         case NIX_TXSCH_LVL_TL3:
 310                 reg[k] = NIX_AF_TL3X_SCHEDULE(schq);
 311                 regval[k] = (strict_prio << 24) | rr_quantum;
 312                 k++;
 313
 314                 break;
 315         case NIX_TXSCH_LVL_TL2:
 316                 reg[k] = NIX_AF_TL2X_SCHEDULE(schq);
 317                 regval[k] = (strict_prio << 24) | rr_quantum;
 318                 k++;
 319
 320                 break;
 321         case NIX_TXSCH_LVL_TL1:
 322                 reg[k] = NIX_AF_TL1X_SCHEDULE(schq);
 323                 regval[k] = rr_quantum;
 324                 k++;
 325
 326                 break;
 327         }
 328
 329         return k;
 330 }
 331
 332 static uint8_t
 333 prepare_tm_shaper_reg(struct otx2_nix_tm_node *tm_node,
 334                       struct otx2_nix_tm_shaper_profile *profile,
 335                       volatile uint64_t *reg, volatile uint64_t *regval)
 336 {
 337         struct shaper_params cir, pir;
 338         uint32_t schq = tm_node->hw_id;
 339         uint8_t k = 0;
 340
 341         memset(&cir, 0, sizeof(cir));
 342         memset(&pir, 0, sizeof(pir));
 343         shaper_config_to_nix(profile, &cir, &pir);
 344
 345         otx2_tm_dbg("Shaper config node %s(%u) lvl %u id %u, "
 346                     "pir %" PRIu64 "(%" PRIu64 "B),"
 347                      " cir %" PRIu64 "(%" PRIu64 "B) (%p)",
 348                      nix_hwlvl2str(tm_node->hw_lvl), schq, tm_node->lvl,
 349                      tm_node->id, pir.rate, pir.burst,
 350                      cir.rate, cir.burst, tm_node);
 351
 352         switch (tm_node->hw_lvl) {
 353         case NIX_TXSCH_LVL_SMQ:
 354                 /* Configure PIR, CIR */
 355                 reg[k] = NIX_AF_MDQX_PIR(schq);
 356                 regval[k] = (pir.rate && pir.burst) ?
 357                                 (shaper2regval(&pir) | 1) : 0;
 358                 k++;
 359
 360                 reg[k] = NIX_AF_MDQX_CIR(schq);
 361                 regval[k] = (cir.rate && cir.burst) ?
 362                                 (shaper2regval(&cir) | 1) : 0;
 363                 k++;
 364
 365                 /* Configure RED ALG */
 366                 reg[k] = NIX_AF_MDQX_SHAPE(schq);
 367                 regval[k] = ((uint64_t)tm_node->red_algo << 9);
 368                 k++;
 369                 break;
 370         case NIX_TXSCH_LVL_TL4:
 371                 /* Configure PIR, CIR */
 372                 reg[k] = NIX_AF_TL4X_PIR(schq);
 373                 regval[k] = (pir.rate && pir.burst) ?
 374                                 (shaper2regval(&pir) | 1) : 0;
 375                 k++;
 376
 377                 reg[k] = NIX_AF_TL4X_CIR(schq);
 378                 regval[k] = (cir.rate && cir.burst) ?
 379                                 (shaper2regval(&cir) | 1) : 0;
 380                 k++;
 381
 382                 /* Configure RED algo */
 383                 reg[k] = NIX_AF_TL4X_SHAPE(schq);
 384                 regval[k] = ((uint64_t)tm_node->red_algo << 9);
 385                 k++;
 386                 break;
 387         case NIX_TXSCH_LVL_TL3:
 388                 /* Configure PIR, CIR */
 389                 reg[k] = NIX_AF_TL3X_PIR(schq);
 390                 regval[k] = (pir.rate && pir.burst) ?
 391                                 (shaper2regval(&pir) | 1) : 0;
 392                 k++;
 393
 394                 reg[k] = NIX_AF_TL3X_CIR(schq);
 395                 regval[k] = (cir.rate && cir.burst) ?
 396                                 (shaper2regval(&cir) | 1) : 0;
 397                 k++;
 398
 399                 /* Configure RED algo */
 400                 reg[k] = NIX_AF_TL3X_SHAPE(schq);
 401                 regval[k] = ((uint64_t)tm_node->red_algo << 9);
 402                 k++;
 403
 404                 break;
 405         case NIX_TXSCH_LVL_TL2:
 406                 /* Configure PIR, CIR */
 407                 reg[k] = NIX_AF_TL2X_PIR(schq);
 408                 regval[k] = (pir.rate && pir.burst) ?
 409                                 (shaper2regval(&pir) | 1) : 0;
 410                 k++;
 411
 412                 reg[k] = NIX_AF_TL2X_CIR(schq);
 413                 regval[k] = (cir.rate && cir.burst) ?
 414                                 (shaper2regval(&cir) | 1) : 0;
 415                 k++;
 416
 417                 /* Configure RED algo */
 418                 reg[k] = NIX_AF_TL2X_SHAPE(schq);
 419                 regval[k] = ((uint64_t)tm_node->red_algo << 9);
 420                 k++;
 421
 422                 break;
 423         case NIX_TXSCH_LVL_TL1:
 424                 /* Configure CIR */
 425                 reg[k] = NIX_AF_TL1X_CIR(schq);
 426                 regval[k] = (cir.rate && cir.burst) ?
 427                                 (shaper2regval(&cir) | 1) : 0;
 428                 k++;
 429                 break;
 430         }
 431
 432         return k;
 433 }
 434
 435 static uint8_t
 436 prepare_tm_sw_xoff(struct otx2_nix_tm_node *tm_node, bool enable,
 437                    volatile uint64_t *reg, volatile uint64_t *regval)
 438 {
 439         uint32_t hw_lvl = tm_node->hw_lvl;
 440         uint32_t schq = tm_node->hw_id;
 441         uint8_t k = 0;
 442
 443         otx2_tm_dbg("sw xoff config node %s(%u) lvl %u id %u, enable %u (%p)",
 444                     nix_hwlvl2str(hw_lvl), schq, tm_node->lvl,
 445                     tm_node->id, enable, tm_node);
 446
 447         regval[k] = enable;
 448
 449         switch (hw_lvl) {
 450         case NIX_TXSCH_LVL_MDQ:
 451                 reg[k] = NIX_AF_MDQX_SW_XOFF(schq);
 452                 k++;
 453                 break;
 454         case NIX_TXSCH_LVL_TL4:
 455                 reg[k] = NIX_AF_TL4X_SW_XOFF(schq);
 456                 k++;
 457                 break;
 458         case NIX_TXSCH_LVL_TL3:
 459                 reg[k] = NIX_AF_TL3X_SW_XOFF(schq);
 460                 k++;
 461                 break;
 462         case NIX_TXSCH_LVL_TL2:
 463                 reg[k] = NIX_AF_TL2X_SW_XOFF(schq);
 464                 k++;
 465                 break;
 466         case NIX_TXSCH_LVL_TL1:
 467                 reg[k] = NIX_AF_TL1X_SW_XOFF(schq);
 468                 k++;
 469                 break;
 470         default:
 471                 break;
 472         }
 473
 474         return k;
 475 }
 476
 477 static int
 478 populate_tm_reg(struct otx2_eth_dev *dev,
 479                 struct otx2_nix_tm_node *tm_node)
 480 {
 481         struct otx2_nix_tm_shaper_profile *profile;
 482         uint64_t regval_mask[MAX_REGS_PER_MBOX_MSG];
 483         uint64_t regval[MAX_REGS_PER_MBOX_MSG];
 484         uint64_t reg[MAX_REGS_PER_MBOX_MSG];
 485         struct otx2_mbox *mbox = dev->mbox;
 486         uint64_t parent = 0, child = 0;
 487         uint32_t hw_lvl, rr_prio, schq;
 488         struct nix_txschq_config *req;
 489         int rc = -EFAULT;
 490         uint8_t k = 0;
 491
 492         memset(regval_mask, 0, sizeof(regval_mask));
 493         profile = nix_tm_shaper_profile_search(dev,
 494                                         tm_node->params.shaper_profile_id);
 495         rr_prio = tm_node->rr_prio;
 496         hw_lvl = tm_node->hw_lvl;
 497         schq = tm_node->hw_id;
 498
 499         /* Root node will not have a parent node */
 500         if (hw_lvl == dev->otx2_tm_root_lvl)
 501                 parent = tm_node->parent_hw_id;
 502         else
 503                 parent = tm_node->parent->hw_id;
 504
 505         /* Do we need this trigger to configure TL1 */
 506         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
 507             hw_lvl == dev->otx2_tm_root_lvl) {
 508                 rc = populate_tm_tl1_default(dev, parent);
 509                 if (rc)
 510                         goto error;
 511         }
 512
 513         if (hw_lvl != NIX_TXSCH_LVL_SMQ)
 514                 child = find_prio_anchor(dev, tm_node->id);
 515
 516         /* Override default rr_prio when TL1
 517          * Static Priority is disabled
 518          */
 519         if (hw_lvl == NIX_TXSCH_LVL_TL1 &&
 520             dev->tm_flags & NIX_TM_TL1_NO_SP) {
 521                 rr_prio = TXSCH_TL1_DFLT_RR_PRIO;
 522                 child = 0;
 523         }
 524
 525         otx2_tm_dbg("Topology config node %s(%u)->%s(%"PRIu64") lvl %u, id %u"
 526                     " prio_anchor %"PRIu64" rr_prio %u (%p)",
 527                     nix_hwlvl2str(hw_lvl), schq, nix_hwlvl2str(hw_lvl + 1),
 528                     parent, tm_node->lvl, tm_node->id, child, rr_prio, tm_node);
 529
 530         /* Prepare Topology and Link config */
 531         switch (hw_lvl) {
 532         case NIX_TXSCH_LVL_SMQ:
 533
 534                 /* Set xoff which will be cleared later */
 535                 reg[k] = NIX_AF_SMQX_CFG(schq);
 536                 regval[k] = BIT_ULL(50);
 537                 regval_mask[k] = ~BIT_ULL(50);
 538                 k++;
 539
 540                 /* Parent and schedule conf */
 541                 reg[k] = NIX_AF_MDQX_PARENT(schq);
 542                 regval[k] = parent << 16;
 543                 k++;
 544
 545                 break;
 546         case NIX_TXSCH_LVL_TL4:
 547                 /* Parent and schedule conf */
 548                 reg[k] = NIX_AF_TL4X_PARENT(schq);
 549                 regval[k] = parent << 16;
 550                 k++;
 551
 552                 reg[k] = NIX_AF_TL4X_TOPOLOGY(schq);
 553                 regval[k] = (child << 32) | (rr_prio << 1);
 554                 k++;
 555
 556                 /* Configure TL4 to send to SDP channel instead of CGX/LBK */
 557                 if (otx2_dev_is_sdp(dev)) {
 558                         reg[k] = NIX_AF_TL4X_SDP_LINK_CFG(schq);
 559                         regval[k] = BIT_ULL(12);
 560                         k++;
 561                 }
 562                 break;
 563         case NIX_TXSCH_LVL_TL3:
 564                 /* Parent and schedule conf */
 565                 reg[k] = NIX_AF_TL3X_PARENT(schq);
 566                 regval[k] = parent << 16;
 567                 k++;
 568
 569                 reg[k] = NIX_AF_TL3X_TOPOLOGY(schq);
 570                 regval[k] = (child << 32) | (rr_prio << 1);
 571                 k++;
 572
 573                 /* Link configuration */
 574                 if (!otx2_dev_is_sdp(dev) &&
 575                     dev->link_cfg_lvl == NIX_TXSCH_LVL_TL3) {
 576                         reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(schq,
 577                                                 nix_get_link(dev));
 578                         regval[k] = BIT_ULL(12) | nix_get_relchan(dev);
 579                         k++;
 580                 }
 581
 582                 break;
 583         case NIX_TXSCH_LVL_TL2:
 584                 /* Parent and schedule conf */
 585                 reg[k] = NIX_AF_TL2X_PARENT(schq);
 586                 regval[k] = parent << 16;
 587                 k++;
 588
 589                 reg[k] = NIX_AF_TL2X_TOPOLOGY(schq);
 590                 regval[k] = (child << 32) | (rr_prio << 1);
 591                 k++;
 592
 593                 /* Link configuration */
 594                 if (!otx2_dev_is_sdp(dev) &&
 595                     dev->link_cfg_lvl == NIX_TXSCH_LVL_TL2) {
 596                         reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(schq,
 597                                                 nix_get_link(dev));
 598                         regval[k] = BIT_ULL(12) | nix_get_relchan(dev);
 599                         k++;
 600                 }
 601
 602                 break;
 603         case NIX_TXSCH_LVL_TL1:
 604                 reg[k] = NIX_AF_TL1X_TOPOLOGY(schq);
 605                 regval[k] = (child << 32) | (rr_prio << 1 /*RR_PRIO*/);
 606                 k++;
 607
 608                 break;
 609         }
 610
 611         /* Prepare schedule config */
 612         k += prepare_tm_sched_reg(dev, tm_node, &reg[k], &regval[k]);
 613
 614         /* Prepare shaping config */
 615         k += prepare_tm_shaper_reg(tm_node, profile, &reg[k], &regval[k]);
 616
 617         if (!k)
 618                 return 0;
 619
 620         /* Copy and send config mbox */
 621         req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 622         req->lvl = hw_lvl;
 623         req->num_regs = k;
 624
 625         otx2_mbox_memcpy(req->reg, reg, sizeof(uint64_t) * k);
 626         otx2_mbox_memcpy(req->regval, regval, sizeof(uint64_t) * k);
 627         otx2_mbox_memcpy(req->regval_mask, regval_mask, sizeof(uint64_t) * k);
 628
 629         rc = otx2_mbox_process(mbox);
 630         if (rc)
 631                 goto error;
 632
 633         return 0;
 634 error:
 635         otx2_err("Txschq cfg request failed for node %p, rc=%d", tm_node, rc);
 636         return rc;
 637 }
 638
 639
 640 static int
 641 nix_tm_txsch_reg_config(struct otx2_eth_dev *dev)
 642 {
 643         struct otx2_nix_tm_node *tm_node;
 644         uint32_t hw_lvl;
 645         int rc = 0;
 646
 647         for (hw_lvl = 0; hw_lvl <= dev->otx2_tm_root_lvl; hw_lvl++) {
 648                 TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 649                         if (tm_node->hw_lvl == hw_lvl &&
 650                             tm_node->hw_lvl != NIX_TXSCH_LVL_CNT) {
 651                                 rc = populate_tm_reg(dev, tm_node);
 652                                 if (rc)
 653                                         goto exit;
 654                         }
 655                 }
 656         }
 657 exit:
 658         return rc;
 659 }
 660
 661 static struct otx2_nix_tm_node *
 662 nix_tm_node_search(struct otx2_eth_dev *dev,
 663                    uint32_t node_id, bool user)
 664 {
 665         struct otx2_nix_tm_node *tm_node;
 666
 667         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 668                 if (tm_node->id == node_id &&
 669                     (user == !!(tm_node->flags & NIX_TM_NODE_USER)))
 670                         return tm_node;
 671         }
 672         return NULL;
 673 }
 674
 675 static uint32_t
 676 check_rr(struct otx2_eth_dev *dev, uint32_t priority, uint32_t parent_id)
 677 {
 678         struct otx2_nix_tm_node *tm_node;
 679         uint32_t rr_num = 0;
 680
 681         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 682                 if (!tm_node->parent)
 683                         continue;
 684
 685                 if (!(tm_node->parent->id == parent_id))
 686                         continue;
 687
 688                 if (tm_node->priority == priority)
 689                         rr_num++;
 690         }
 691         return rr_num;
 692 }
 693
 694 static int
 695 nix_tm_update_parent_info(struct otx2_eth_dev *dev)
 696 {
 697         struct otx2_nix_tm_node *tm_node_child;
 698         struct otx2_nix_tm_node *tm_node;
 699         struct otx2_nix_tm_node *parent;
 700         uint32_t rr_num = 0;
 701         uint32_t priority;
 702
 703         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 704                 if (!tm_node->parent)
 705                         continue;
 706                 /* Count group of children of same priority i.e are RR */
 707                 parent = tm_node->parent;
 708                 priority = tm_node->priority;
 709                 rr_num = check_rr(dev, priority, parent->id);
 710
 711                 /* Assuming that multiple RR groups are
 712                  * not configured based on capability.
 713                  */
 714                 if (rr_num > 1) {
 715                         parent->rr_prio = priority;
 716                         parent->rr_num = rr_num;
 717                 }
 718
 719                 /* Find out static priority children that are not in RR */
 720                 TAILQ_FOREACH(tm_node_child, &dev->node_list, node) {
 721                         if (!tm_node_child->parent)
 722                                 continue;
 723                         if (parent->id != tm_node_child->parent->id)
 724                                 continue;
 725                         if (parent->max_prio == UINT32_MAX &&
 726                             tm_node_child->priority != parent->rr_prio)
 727                                 parent->max_prio = 0;
 728
 729                         if (parent->max_prio < tm_node_child->priority &&
 730                             parent->rr_prio != tm_node_child->priority)
 731                                 parent->max_prio = tm_node_child->priority;
 732                 }
 733         }
 734
 735         return 0;
 736 }
 737
 738 static int
 739 nix_tm_node_add_to_list(struct otx2_eth_dev *dev, uint32_t node_id,
 740                         uint32_t parent_node_id, uint32_t priority,
 741                         uint32_t weight, uint16_t hw_lvl,
 742                         uint16_t lvl, bool user,
 743                         struct rte_tm_node_params *params)
 744 {
 745         struct otx2_nix_tm_shaper_profile *profile;
 746         struct otx2_nix_tm_node *tm_node, *parent_node;
 747         struct shaper_params cir, pir;
 748         uint32_t profile_id;
 749
 750         profile_id = params->shaper_profile_id;
 751         profile = nix_tm_shaper_profile_search(dev, profile_id);
 752
 753         parent_node = nix_tm_node_search(dev, parent_node_id, user);
 754
 755         tm_node = rte_zmalloc("otx2_nix_tm_node",
 756                               sizeof(struct otx2_nix_tm_node), 0);
 757         if (!tm_node)
 758                 return -ENOMEM;
 759
 760         tm_node->lvl = lvl;
 761         tm_node->hw_lvl = hw_lvl;
 762
 763         /* Maintain minimum weight */
 764         if (!weight)
 765                 weight = 1;
 766
 767         tm_node->id = node_id;
 768         tm_node->priority = priority;
 769         tm_node->weight = weight;
 770         tm_node->rr_prio = 0xf;
 771         tm_node->max_prio = UINT32_MAX;
 772         tm_node->hw_id = UINT32_MAX;
 773         tm_node->flags = 0;
 774         if (user)
 775                 tm_node->flags = NIX_TM_NODE_USER;
 776         rte_memcpy(&tm_node->params, params, sizeof(struct rte_tm_node_params));
 777
 778         if (profile)
 779                 profile->reference_count++;
 780
 781         memset(&cir, 0, sizeof(cir));
 782         memset(&pir, 0, sizeof(pir));
 783         shaper_config_to_nix(profile, &cir, &pir);
 784
 785         tm_node->parent = parent_node;
 786         tm_node->parent_hw_id = UINT32_MAX;
 787         /* C0 doesn't support STALL when both PIR & CIR are enabled */
 788         if (lvl < OTX2_TM_LVL_QUEUE &&
 789             otx2_dev_is_96xx_Cx(dev) &&
 790             pir.rate && cir.rate)
 791                 tm_node->red_algo = NIX_REDALG_DISCARD;
 792         else
 793                 tm_node->red_algo = NIX_REDALG_STD;
 794
 795         TAILQ_INSERT_TAIL(&dev->node_list, tm_node, node);
 796
 797         return 0;
 798 }
 799
 800 static int
 801 nix_tm_clear_shaper_profiles(struct otx2_eth_dev *dev)
 802 {
 803         struct otx2_nix_tm_shaper_profile *shaper_profile;
 804
 805         while ((shaper_profile = TAILQ_FIRST(&dev->shaper_profile_list))) {
 806                 if (shaper_profile->reference_count)
 807                         otx2_tm_dbg("Shaper profile %u has non zero references",
 808                                     shaper_profile->shaper_profile_id);
 809                 TAILQ_REMOVE(&dev->shaper_profile_list, shaper_profile, shaper);
 810                 rte_free(shaper_profile);
 811         }
 812
 813         return 0;
 814 }
 815
 816 static int
 817 nix_clear_path_xoff(struct otx2_eth_dev *dev,
 818                     struct otx2_nix_tm_node *tm_node)
 819 {
 820         struct nix_txschq_config *req;
 821         struct otx2_nix_tm_node *p;
 822         int rc;
 823
 824         /* Manipulating SW_XOFF not supported on Ax */
 825         if (otx2_dev_is_Ax(dev))
 826                 return 0;
 827
 828         /* Enable nodes in path for flush to succeed */
 829         if (!nix_tm_is_leaf(dev, tm_node->lvl))
 830                 p = tm_node;
 831         else
 832                 p = tm_node->parent;
 833         while (p) {
 834                 if (!(p->flags & NIX_TM_NODE_ENABLED) &&
 835                     (p->flags & NIX_TM_NODE_HWRES)) {
 836                         req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
 837                         req->lvl = p->hw_lvl;
 838                         req->num_regs = prepare_tm_sw_xoff(p, false, req->reg,
 839                                                            req->regval);
 840                         rc = otx2_mbox_process(dev->mbox);
 841                         if (rc)
 842                                 return rc;
 843
 844                         p->flags |= NIX_TM_NODE_ENABLED;
 845                 }
 846                 p = p->parent;
 847         }
 848
 849         return 0;
 850 }
 851
 852 static int
 853 nix_smq_xoff(struct otx2_eth_dev *dev,
 854              struct otx2_nix_tm_node *tm_node,
 855              bool enable)
 856 {
 857         struct otx2_mbox *mbox = dev->mbox;
 858         struct nix_txschq_config *req;
 859         uint16_t smq;
 860         int rc;
 861
 862         smq = tm_node->hw_id;
 863         otx2_tm_dbg("Setting SMQ %u XOFF/FLUSH to %s", smq,
 864                     enable ? "enable" : "disable");
 865
 866         rc = nix_clear_path_xoff(dev, tm_node);
 867         if (rc)
 868                 return rc;
 869
 870         req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
 871         req->lvl = NIX_TXSCH_LVL_SMQ;
 872         req->num_regs = 1;
 873
 874         req->reg[0] = NIX_AF_SMQX_CFG(smq);
 875         req->regval[0] = enable ? (BIT_ULL(50) | BIT_ULL(49)) : 0;
 876         req->regval_mask[0] = enable ?
 877                                 ~(BIT_ULL(50) | BIT_ULL(49)) : ~BIT_ULL(50);
 878
 879         return otx2_mbox_process(mbox);
 880 }
 881
 882 int
 883 otx2_nix_sq_sqb_aura_fc(void *__txq, bool enable)
 884 {
 885         struct otx2_eth_txq *txq = __txq;
 886         struct npa_aq_enq_req *req;
 887         struct npa_aq_enq_rsp *rsp;
 888         struct otx2_npa_lf *lf;
 889         struct otx2_mbox *mbox;
 890         uint64_t aura_handle;
 891         int rc;
 892
 893         otx2_tm_dbg("Setting SQ %u SQB aura FC to %s", txq->sq,
 894                     enable ? "enable" : "disable");
 895
 896         lf = otx2_npa_lf_obj_get();
 897         if (!lf)
 898                 return -EFAULT;
 899         mbox = lf->mbox;
 900         /* Set/clear sqb aura fc_ena */
 901         aura_handle = txq->sqb_pool->pool_id;
 902         req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
 903
 904         req->aura_id = npa_lf_aura_handle_to_aura(aura_handle);
 905         req->ctype = NPA_AQ_CTYPE_AURA;
 906         req->op = NPA_AQ_INSTOP_WRITE;
 907         /* Below is not needed for aura writes but AF driver needs it */
 908         /* AF will translate to associated poolctx */
 909         req->aura.pool_addr = req->aura_id;
 910
 911         req->aura.fc_ena = enable;
 912         req->aura_mask.fc_ena = 1;
 913
 914         rc = otx2_mbox_process(mbox);
 915         if (rc)
 916                 return rc;
 917
 918         /* Read back npa aura ctx */
 919         req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
 920
 921         req->aura_id = npa_lf_aura_handle_to_aura(aura_handle);
 922         req->ctype = NPA_AQ_CTYPE_AURA;
 923         req->op = NPA_AQ_INSTOP_READ;
 924
 925         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
 926         if (rc)
 927                 return rc;
 928
 929         /* Init when enabled as there might be no triggers */
 930         if (enable)
 931                 *(volatile uint64_t *)txq->fc_mem = rsp->aura.count;
 932         else
 933                 *(volatile uint64_t *)txq->fc_mem = txq->nb_sqb_bufs;
 934         /* Sync write barrier */
 935         rte_wmb();
 936
 937         return 0;
 938 }
 939
 940 static int
 941 nix_txq_flush_sq_spin(struct otx2_eth_txq *txq)
 942 {
 943         uint16_t sqb_cnt, head_off, tail_off;
 944         struct otx2_eth_dev *dev = txq->dev;
 945         uint64_t wdata, val, prev;
 946         uint16_t sq = txq->sq;
 947         int64_t *regaddr;
 948         uint64_t timeout;/* 10's of usec */
 949
 950         /* Wait for enough time based on shaper min rate */
 951         timeout = (txq->qconf.nb_desc * NIX_MAX_HW_FRS * 8 * 1E5);
 952         timeout = timeout / dev->tm_rate_min;
 953         if (!timeout)
 954                 timeout = 10000;
 955
 956         wdata = ((uint64_t)sq << 32);
 957         regaddr = (int64_t *)(dev->base + NIX_LF_SQ_OP_STATUS);
 958         val = otx2_atomic64_add_nosync(wdata, regaddr);
 959
 960         /* Spin multiple iterations as "txq->fc_cache_pkts" can still
 961          * have space to send pkts even though fc_mem is disabled
 962          */
 963
 964         while (true) {
 965                 prev = val;
 966                 rte_delay_us(10);
 967                 val = otx2_atomic64_add_nosync(wdata, regaddr);
 968                 /* Continue on error */
 969                 if (val & BIT_ULL(63))
 970                         continue;
 971
 972                 if (prev != val)
 973                         continue;
 974
 975                 sqb_cnt = val & 0xFFFF;
 976                 head_off = (val >> 20) & 0x3F;
 977                 tail_off = (val >> 28) & 0x3F;
 978
 979                 /* SQ reached quiescent state */
 980                 if (sqb_cnt <= 1 && head_off == tail_off &&
 981                     (*txq->fc_mem == txq->nb_sqb_bufs)) {
 982                         break;
 983                 }
 984
 985                 /* Timeout */
 986                 if (!timeout)
 987                         goto exit;
 988                 timeout--;
 989         }
 990
 991         return 0;
 992 exit:
 993         return -EFAULT;
 994 }
 995
 996 /* Flush and disable tx queue and its parent SMQ */
 997 int otx2_nix_sq_flush_pre(void *_txq, bool dev_started)
 998 {
 999         struct otx2_nix_tm_node *tm_node, *sibling;
1000         struct otx2_eth_txq *txq;
1001         struct otx2_eth_dev *dev;
1002         uint16_t sq;
1003         bool user;
1004         int rc;
1005
1006         txq = _txq;
1007         dev = txq->dev;
1008         sq = txq->sq;
1009
1010         user = !!(dev->tm_flags & NIX_TM_COMMITTED);
1011
1012         /* Find the node for this SQ */
1013         tm_node = nix_tm_node_search(dev, sq, user);
1014         if (!tm_node || !(tm_node->flags & NIX_TM_NODE_ENABLED)) {
1015                 otx2_err("Invalid node/state for sq %u", sq);
1016                 return -EFAULT;
1017         }
1018
1019         /* Enable CGX RXTX to drain pkts */
1020         if (!dev_started) {
1021                 /* Though it enables both RX MCAM Entries and CGX Link
1022                  * we assume all the rx queues are stopped way back.
1023                  */
1024                 otx2_mbox_alloc_msg_nix_lf_start_rx(dev->mbox);
1025                 rc = otx2_mbox_process(dev->mbox);
1026                 if (rc) {
1027                         otx2_err("cgx start failed, rc=%d", rc);
1028                         return rc;
1029                 }
1030         }
1031
1032         /* Disable smq xoff for case it was enabled earlier */
1033         rc = nix_smq_xoff(dev, tm_node->parent, false);
1034         if (rc) {
1035                 otx2_err("Failed to enable smq %u, rc=%d",
1036                          tm_node->parent->hw_id, rc);
1037                 return rc;
1038         }
1039
1040         /* As per HRM, to disable an SQ, all other SQ's
1041          * that feed to same SMQ must be paused before SMQ flush.
1042          */
1043         TAILQ_FOREACH(sibling, &dev->node_list, node) {
1044                 if (sibling->parent != tm_node->parent)
1045                         continue;
1046                 if (!(sibling->flags & NIX_TM_NODE_ENABLED))
1047                         continue;
1048
1049                 sq = sibling->id;
1050                 txq = dev->eth_dev->data->tx_queues[sq];
1051                 if (!txq)
1052                         continue;
1053
1054                 rc = otx2_nix_sq_sqb_aura_fc(txq, false);
1055                 if (rc) {
1056                         otx2_err("Failed to disable sqb aura fc, rc=%d", rc);
1057                         goto cleanup;
1058                 }
1059
1060                 /* Wait for sq entries to be flushed */
1061                 rc = nix_txq_flush_sq_spin(txq);
1062                 if (rc) {
1063                         otx2_err("Failed to drain sq %u, rc=%d\n", txq->sq, rc);
1064                         return rc;
1065                 }
1066         }
1067
1068         tm_node->flags &= ~NIX_TM_NODE_ENABLED;
1069
1070         /* Disable and flush */
1071         rc = nix_smq_xoff(dev, tm_node->parent, true);
1072         if (rc) {
1073                 otx2_err("Failed to disable smq %u, rc=%d",
1074                          tm_node->parent->hw_id, rc);
1075                 goto cleanup;
1076         }
1077 cleanup:
1078         /* Restore cgx state */
1079         if (!dev_started) {
1080                 otx2_mbox_alloc_msg_nix_lf_stop_rx(dev->mbox);
1081                 rc |= otx2_mbox_process(dev->mbox);
1082         }
1083
1084         return rc;
1085 }
1086
1087 int otx2_nix_sq_flush_post(void *_txq)
1088 {
1089         struct otx2_nix_tm_node *tm_node, *sibling;
1090         struct otx2_eth_txq *txq = _txq;
1091         struct otx2_eth_txq *s_txq;
1092         struct otx2_eth_dev *dev;
1093         bool once = false;
1094         uint16_t sq, s_sq;
1095         bool user;
1096         int rc;
1097
1098         dev = txq->dev;
1099         sq = txq->sq;
1100         user = !!(dev->tm_flags & NIX_TM_COMMITTED);
1101
1102         /* Find the node for this SQ */
1103         tm_node = nix_tm_node_search(dev, sq, user);
1104         if (!tm_node) {
1105                 otx2_err("Invalid node for sq %u", sq);
1106                 return -EFAULT;
1107         }
1108
1109         /* Enable all the siblings back */
1110         TAILQ_FOREACH(sibling, &dev->node_list, node) {
1111                 if (sibling->parent != tm_node->parent)
1112                         continue;
1113
1114                 if (sibling->id == sq)
1115                         continue;
1116
1117                 if (!(sibling->flags & NIX_TM_NODE_ENABLED))
1118                         continue;
1119
1120                 s_sq = sibling->id;
1121                 s_txq = dev->eth_dev->data->tx_queues[s_sq];
1122                 if (!s_txq)
1123                         continue;
1124
1125                 if (!once) {
1126                         /* Enable back if any SQ is still present */
1127                         rc = nix_smq_xoff(dev, tm_node->parent, false);
1128                         if (rc) {
1129                                 otx2_err("Failed to enable smq %u, rc=%d",
1130                                          tm_node->parent->hw_id, rc);
1131                                 return rc;
1132                         }
1133                         once = true;
1134                 }
1135
1136                 rc = otx2_nix_sq_sqb_aura_fc(s_txq, true);
1137                 if (rc) {
1138                         otx2_err("Failed to enable sqb aura fc, rc=%d", rc);
1139                         return rc;
1140                 }
1141         }
1142
1143         return 0;
1144 }
1145
1146 static int
1147 nix_sq_sched_data(struct otx2_eth_dev *dev,
1148                   struct otx2_nix_tm_node *tm_node,
1149                   bool rr_quantum_only)
1150 {
1151         struct rte_eth_dev *eth_dev = dev->eth_dev;
1152         struct otx2_mbox *mbox = dev->mbox;
1153         uint16_t sq = tm_node->id, smq;
1154         struct nix_aq_enq_req *req;
1155         uint64_t rr_quantum;
1156         int rc;
1157
1158         smq = tm_node->parent->hw_id;
1159         rr_quantum = NIX_TM_WEIGHT_TO_RR_QUANTUM(tm_node->weight);
1160
1161         if (rr_quantum_only)
1162                 otx2_tm_dbg("Update sq(%u) rr_quantum 0x%"PRIx64, sq, rr_quantum);
1163         else
1164                 otx2_tm_dbg("Enabling sq(%u)->smq(%u), rr_quantum 0x%"PRIx64,
1165                             sq, smq, rr_quantum);
1166
1167         if (sq > eth_dev->data->nb_tx_queues)
1168                 return -EFAULT;
1169
1170         req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
1171         req->qidx = sq;
1172         req->ctype = NIX_AQ_CTYPE_SQ;
1173         req->op = NIX_AQ_INSTOP_WRITE;
1174
1175         /* smq update only when needed */
1176         if (!rr_quantum_only) {
1177                 req->sq.smq = smq;
1178                 req->sq_mask.smq = ~req->sq_mask.smq;
1179         }
1180         req->sq.smq_rr_quantum = rr_quantum;
1181         req->sq_mask.smq_rr_quantum = ~req->sq_mask.smq_rr_quantum;
1182
1183         rc = otx2_mbox_process(mbox);
1184         if (rc)
1185                 otx2_err("Failed to set smq, rc=%d", rc);
1186         return rc;
1187 }
1188
1189 int otx2_nix_sq_enable(void *_txq)
1190 {
1191         struct otx2_eth_txq *txq = _txq;
1192         int rc;
1193
1194         /* Enable sqb_aura fc */
1195         rc = otx2_nix_sq_sqb_aura_fc(txq, true);
1196         if (rc) {
1197                 otx2_err("Failed to enable sqb aura fc, rc=%d", rc);
1198                 return rc;
1199         }
1200
1201         return 0;
1202 }
1203
1204 static int
1205 nix_tm_free_resources(struct otx2_eth_dev *dev, uint32_t flags_mask,
1206                       uint32_t flags, bool hw_only)
1207 {
1208         struct otx2_nix_tm_shaper_profile *profile;
1209         struct otx2_nix_tm_node *tm_node, *next_node;
1210         struct otx2_mbox *mbox = dev->mbox;
1211         struct nix_txsch_free_req *req;
1212         uint32_t profile_id;
1213         int rc = 0;
1214
1215         next_node = TAILQ_FIRST(&dev->node_list);
1216         while (next_node) {
1217                 tm_node = next_node;
1218                 next_node = TAILQ_NEXT(tm_node, node);
1219
1220                 /* Check for only requested nodes */
1221                 if ((tm_node->flags & flags_mask) != flags)
1222                         continue;
1223
1224                 if (!nix_tm_is_leaf(dev, tm_node->lvl) &&
1225                     tm_node->hw_lvl != NIX_TXSCH_LVL_TL1 &&
1226                     tm_node->flags & NIX_TM_NODE_HWRES) {
1227                         /* Free specific HW resource */
1228                         otx2_tm_dbg("Free hwres %s(%u) lvl %u id %u (%p)",
1229                                     nix_hwlvl2str(tm_node->hw_lvl),
1230                                     tm_node->hw_id, tm_node->lvl,
1231                                     tm_node->id, tm_node);
1232
1233                         rc = nix_clear_path_xoff(dev, tm_node);
1234                         if (rc)
1235                                 return rc;
1236
1237                         req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
1238                         req->flags = 0;
1239                         req->schq_lvl = tm_node->hw_lvl;
1240                         req->schq = tm_node->hw_id;
1241                         rc = otx2_mbox_process(mbox);
1242                         if (rc)
1243                                 return rc;
1244                         tm_node->flags &= ~NIX_TM_NODE_HWRES;
1245                 }
1246
1247                 /* Leave software elements if needed */
1248                 if (hw_only)
1249                         continue;
1250
1251                 otx2_tm_dbg("Free node lvl %u id %u (%p)",
1252                             tm_node->lvl, tm_node->id, tm_node);
1253
1254                 profile_id = tm_node->params.shaper_profile_id;
1255                 profile = nix_tm_shaper_profile_search(dev, profile_id);
1256                 if (profile)
1257                         profile->reference_count--;
1258
1259                 TAILQ_REMOVE(&dev->node_list, tm_node, node);
1260                 rte_free(tm_node);
1261         }
1262
1263         if (!flags_mask) {
1264                 /* Free all hw resources */
1265                 req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
1266                 req->flags = TXSCHQ_FREE_ALL;
1267
1268                 return otx2_mbox_process(mbox);
1269         }
1270
1271         return rc;
1272 }
1273
1274 static uint8_t
1275 nix_tm_copy_rsp_to_dev(struct otx2_eth_dev *dev,
1276                        struct nix_txsch_alloc_rsp *rsp)
1277 {
1278         uint16_t schq;
1279         uint8_t lvl;
1280
1281         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1282                 for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) {
1283                         dev->txschq_list[lvl][schq] = rsp->schq_list[lvl][schq];
1284                         dev->txschq_contig_list[lvl][schq] =
1285                                 rsp->schq_contig_list[lvl][schq];
1286                 }
1287
1288                 dev->txschq[lvl] = rsp->schq[lvl];
1289                 dev->txschq_contig[lvl] = rsp->schq_contig[lvl];
1290         }
1291         return 0;
1292 }
1293
1294 static int
1295 nix_tm_assign_id_to_node(struct otx2_eth_dev *dev,
1296                          struct otx2_nix_tm_node *child,
1297                          struct otx2_nix_tm_node *parent)
1298 {
1299         uint32_t hw_id, schq_con_index, prio_offset;
1300         uint32_t l_id, schq_index;
1301
1302         otx2_tm_dbg("Assign hw id for child node %s lvl %u id %u (%p)",
1303                     nix_hwlvl2str(child->hw_lvl), child->lvl, child->id, child);
1304
1305         child->flags |= NIX_TM_NODE_HWRES;
1306
1307         /* Process root nodes */
1308         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
1309             child->hw_lvl == dev->otx2_tm_root_lvl && !parent) {
1310                 int idx = 0;
1311                 uint32_t tschq_con_index;
1312
1313                 l_id = child->hw_lvl;
1314                 tschq_con_index = dev->txschq_contig_index[l_id];
1315                 hw_id = dev->txschq_contig_list[l_id][tschq_con_index];
1316                 child->hw_id = hw_id;
1317                 dev->txschq_contig_index[l_id]++;
1318                 /* Update TL1 hw_id for its parent for config purpose */
1319                 idx = dev->txschq_index[NIX_TXSCH_LVL_TL1]++;
1320                 hw_id = dev->txschq_list[NIX_TXSCH_LVL_TL1][idx];
1321                 child->parent_hw_id = hw_id;
1322                 return 0;
1323         }
1324         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL1 &&
1325             child->hw_lvl == dev->otx2_tm_root_lvl && !parent) {
1326                 uint32_t tschq_con_index;
1327
1328                 l_id = child->hw_lvl;
1329                 tschq_con_index = dev->txschq_index[l_id];
1330                 hw_id = dev->txschq_list[l_id][tschq_con_index];
1331                 child->hw_id = hw_id;
1332                 dev->txschq_index[l_id]++;
1333                 return 0;
1334         }
1335
1336         /* Process children with parents */
1337         l_id = child->hw_lvl;
1338         schq_index = dev->txschq_index[l_id];
1339         schq_con_index = dev->txschq_contig_index[l_id];
1340
1341         if (child->priority == parent->rr_prio) {
1342                 hw_id = dev->txschq_list[l_id][schq_index];
1343                 child->hw_id = hw_id;
1344                 child->parent_hw_id = parent->hw_id;
1345                 dev->txschq_index[l_id]++;
1346         } else {
1347                 prio_offset = schq_con_index + child->priority;
1348                 hw_id = dev->txschq_contig_list[l_id][prio_offset];
1349                 child->hw_id = hw_id;
1350         }
1351         return 0;
1352 }
1353
1354 static int
1355 nix_tm_assign_hw_id(struct otx2_eth_dev *dev)
1356 {
1357         struct otx2_nix_tm_node *parent, *child;
1358         uint32_t child_hw_lvl, con_index_inc, i;
1359
1360         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--) {
1361                 TAILQ_FOREACH(parent, &dev->node_list, node) {
1362                         child_hw_lvl = parent->hw_lvl - 1;
1363                         if (parent->hw_lvl != i)
1364                                 continue;
1365                         TAILQ_FOREACH(child, &dev->node_list, node) {
1366                                 if (!child->parent)
1367                                         continue;
1368                                 if (child->parent->id != parent->id)
1369                                         continue;
1370                                 nix_tm_assign_id_to_node(dev, child, parent);
1371                         }
1372
1373                         con_index_inc = parent->max_prio + 1;
1374                         dev->txschq_contig_index[child_hw_lvl] += con_index_inc;
1375
1376                         /*
1377                          * Explicitly assign id to parent node if it
1378                          * doesn't have a parent
1379                          */
1380                         if (parent->hw_lvl == dev->otx2_tm_root_lvl)
1381                                 nix_tm_assign_id_to_node(dev, parent, NULL);
1382                 }
1383         }
1384         return 0;
1385 }
1386
1387 static uint8_t
1388 nix_tm_count_req_schq(struct otx2_eth_dev *dev,
1389                       struct nix_txsch_alloc_req *req, uint8_t lvl)
1390 {
1391         struct otx2_nix_tm_node *tm_node;
1392         uint8_t contig_count;
1393
1394         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1395                 if (lvl == tm_node->hw_lvl) {
1396                         req->schq[lvl - 1] += tm_node->rr_num;
1397                         if (tm_node->max_prio != UINT32_MAX) {
1398                                 contig_count = tm_node->max_prio + 1;
1399                                 req->schq_contig[lvl - 1] += contig_count;
1400                         }
1401                 }
1402                 if (lvl == dev->otx2_tm_root_lvl &&
1403                     dev->otx2_tm_root_lvl && lvl == NIX_TXSCH_LVL_TL2 &&
1404                     tm_node->hw_lvl == dev->otx2_tm_root_lvl) {
1405                         req->schq_contig[dev->otx2_tm_root_lvl]++;
1406                 }
1407         }
1408
1409         req->schq[NIX_TXSCH_LVL_TL1] = 1;
1410         req->schq_contig[NIX_TXSCH_LVL_TL1] = 0;
1411
1412         return 0;
1413 }
1414
1415 static int
1416 nix_tm_prepare_txschq_req(struct otx2_eth_dev *dev,
1417                           struct nix_txsch_alloc_req *req)
1418 {
1419         uint8_t i;
1420
1421         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--)
1422                 nix_tm_count_req_schq(dev, req, i);
1423
1424         for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
1425                 dev->txschq_index[i] = 0;
1426                 dev->txschq_contig_index[i] = 0;
1427         }
1428         return 0;
1429 }
1430
1431 static int
1432 nix_tm_send_txsch_alloc_msg(struct otx2_eth_dev *dev)
1433 {
1434         struct otx2_mbox *mbox = dev->mbox;
1435         struct nix_txsch_alloc_req *req;
1436         struct nix_txsch_alloc_rsp *rsp;
1437         int rc;
1438
1439         req = otx2_mbox_alloc_msg_nix_txsch_alloc(mbox);
1440
1441         rc = nix_tm_prepare_txschq_req(dev, req);
1442         if (rc)
1443                 return rc;
1444
1445         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
1446         if (rc)
1447                 return rc;
1448
1449         nix_tm_copy_rsp_to_dev(dev, rsp);
1450         dev->link_cfg_lvl = rsp->link_cfg_lvl;
1451
1452         nix_tm_assign_hw_id(dev);
1453         return 0;
1454 }
1455
1456 static int
1457 nix_tm_alloc_resources(struct rte_eth_dev *eth_dev, bool xmit_enable)
1458 {
1459         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1460         struct otx2_nix_tm_node *tm_node;
1461         struct otx2_eth_txq *txq;
1462         uint16_t sq;
1463         int rc;
1464
1465         nix_tm_update_parent_info(dev);
1466
1467         rc = nix_tm_send_txsch_alloc_msg(dev);
1468         if (rc) {
1469                 otx2_err("TM failed to alloc tm resources=%d", rc);
1470                 return rc;
1471         }
1472
1473         rc = nix_tm_txsch_reg_config(dev);
1474         if (rc) {
1475                 otx2_err("TM failed to configure sched registers=%d", rc);
1476                 return rc;
1477         }
1478
1479         /* Trigger MTU recalculate as SMQ needs MTU conf */
1480         if (eth_dev->data->dev_started && eth_dev->data->nb_rx_queues) {
1481                 rc = otx2_nix_recalc_mtu(eth_dev);
1482                 if (rc) {
1483                         otx2_err("TM MTU update failed, rc=%d", rc);
1484                         return rc;
1485                 }
1486         }
1487
1488         /* Mark all non-leaf's as enabled */
1489         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1490                 if (!nix_tm_is_leaf(dev, tm_node->lvl))
1491                         tm_node->flags |= NIX_TM_NODE_ENABLED;
1492         }
1493
1494         if (!xmit_enable)
1495                 return 0;
1496
1497         /* Update SQ Sched Data while SQ is idle */
1498         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1499                 if (!nix_tm_is_leaf(dev, tm_node->lvl))
1500                         continue;
1501
1502                 rc = nix_sq_sched_data(dev, tm_node, false);
1503                 if (rc) {
1504                         otx2_err("SQ %u sched update failed, rc=%d",
1505                                  tm_node->id, rc);
1506                         return rc;
1507                 }
1508         }
1509
1510         /* Finally XON all SMQ's */
1511         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1512                 if (tm_node->hw_lvl != NIX_TXSCH_LVL_SMQ)
1513                         continue;
1514
1515                 rc = nix_smq_xoff(dev, tm_node, false);
1516                 if (rc) {
1517                         otx2_err("Failed to enable smq %u, rc=%d",
1518                                  tm_node->hw_id, rc);
1519                         return rc;
1520                 }
1521         }
1522
1523         /* Enable xmit as all the topology is ready */
1524         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1525                 if (!nix_tm_is_leaf(dev, tm_node->lvl))
1526                         continue;
1527
1528                 sq = tm_node->id;
1529                 txq = eth_dev->data->tx_queues[sq];
1530
1531                 rc = otx2_nix_sq_enable(txq);
1532                 if (rc) {
1533                         otx2_err("TM sw xon failed on SQ %u, rc=%d",
1534                                  tm_node->id, rc);
1535                         return rc;
1536                 }
1537                 tm_node->flags |= NIX_TM_NODE_ENABLED;
1538         }
1539
1540         return 0;
1541 }
1542
1543 static int
1544 send_tm_reqval(struct otx2_mbox *mbox,
1545                struct nix_txschq_config *req,
1546                struct rte_tm_error *error)
1547 {
1548         int rc;
1549
1550         if (!req->num_regs ||
1551             req->num_regs > MAX_REGS_PER_MBOX_MSG) {
1552                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
1553                 error->message = "invalid config";
1554                 return -EIO;
1555         }
1556
1557         rc = otx2_mbox_process(mbox);
1558         if (rc) {
1559                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
1560                 error->message = "unexpected fatal error";
1561         }
1562         return rc;
1563 }
1564
1565 static uint16_t
1566 nix_tm_lvl2nix(struct otx2_eth_dev *dev, uint32_t lvl)
1567 {
1568         if (nix_tm_have_tl1_access(dev)) {
1569                 switch (lvl) {
1570                 case OTX2_TM_LVL_ROOT:
1571                         return NIX_TXSCH_LVL_TL1;
1572                 case OTX2_TM_LVL_SCH1:
1573                         return NIX_TXSCH_LVL_TL2;
1574                 case OTX2_TM_LVL_SCH2:
1575                         return NIX_TXSCH_LVL_TL3;
1576                 case OTX2_TM_LVL_SCH3:
1577                         return NIX_TXSCH_LVL_TL4;
1578                 case OTX2_TM_LVL_SCH4:
1579                         return NIX_TXSCH_LVL_SMQ;
1580                 default:
1581                         return NIX_TXSCH_LVL_CNT;
1582                 }
1583         } else {
1584                 switch (lvl) {
1585                 case OTX2_TM_LVL_ROOT:
1586                         return NIX_TXSCH_LVL_TL2;
1587                 case OTX2_TM_LVL_SCH1:
1588                         return NIX_TXSCH_LVL_TL3;
1589                 case OTX2_TM_LVL_SCH2:
1590                         return NIX_TXSCH_LVL_TL4;
1591                 case OTX2_TM_LVL_SCH3:
1592                         return NIX_TXSCH_LVL_SMQ;
1593                 default:
1594                         return NIX_TXSCH_LVL_CNT;
1595                 }
1596         }
1597 }
1598
1599 static uint16_t
1600 nix_max_prio(struct otx2_eth_dev *dev, uint16_t hw_lvl)
1601 {
1602         if (hw_lvl >= NIX_TXSCH_LVL_CNT)
1603                 return 0;
1604
1605         /* MDQ doesn't support SP */
1606         if (hw_lvl == NIX_TXSCH_LVL_MDQ)
1607                 return 0;
1608
1609         /* PF's TL1 with VF's enabled doesn't support SP */
1610         if (hw_lvl == NIX_TXSCH_LVL_TL1 &&
1611             (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 ||
1612              (dev->tm_flags & NIX_TM_TL1_NO_SP)))
1613                 return 0;
1614
1615         return TXSCH_TLX_SP_PRIO_MAX - 1;
1616 }
1617
1618
1619 static int
1620 validate_prio(struct otx2_eth_dev *dev, uint32_t lvl,
1621               uint32_t parent_id, uint32_t priority,
1622               struct rte_tm_error *error)
1623 {
1624         uint8_t priorities[TXSCH_TLX_SP_PRIO_MAX];
1625         struct otx2_nix_tm_node *tm_node;
1626         uint32_t rr_num = 0;
1627         int i;
1628
1629         /* Validate priority against max */
1630         if (priority > nix_max_prio(dev, nix_tm_lvl2nix(dev, lvl - 1))) {
1631                 error->type = RTE_TM_ERROR_TYPE_CAPABILITIES;
1632                 error->message = "unsupported priority value";
1633                 return -EINVAL;
1634         }
1635
1636         if (parent_id == RTE_TM_NODE_ID_NULL)
1637                 return 0;
1638
1639         memset(priorities, 0, TXSCH_TLX_SP_PRIO_MAX);
1640         priorities[priority] = 1;
1641
1642         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1643                 if (!tm_node->parent)
1644                         continue;
1645
1646                 if (!(tm_node->flags & NIX_TM_NODE_USER))
1647                         continue;
1648
1649                 if (tm_node->parent->id != parent_id)
1650                         continue;
1651
1652                 priorities[tm_node->priority]++;
1653         }
1654
1655         for (i = 0; i < TXSCH_TLX_SP_PRIO_MAX; i++)
1656                 if (priorities[i] > 1)
1657                         rr_num++;
1658
1659         /* At max, one rr groups per parent */
1660         if (rr_num > 1) {
1661                 error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY;
1662                 error->message = "multiple DWRR node priority";
1663                 return -EINVAL;
1664         }
1665
1666         /* Check for previous priority to avoid holes in priorities */
1667         if (priority && !priorities[priority - 1]) {
1668                 error->type = RTE_TM_ERROR_TYPE_NODE_PRIORITY;
1669                 error->message = "priority not in order";
1670                 return -EINVAL;
1671         }
1672
1673         return 0;
1674 }
1675
1676 static int
1677 read_tm_reg(struct otx2_mbox *mbox, uint64_t reg,
1678             uint64_t *regval, uint32_t hw_lvl)
1679 {
1680         volatile struct nix_txschq_config *req;
1681         struct nix_txschq_config *rsp;
1682         int rc;
1683
1684         req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
1685         req->read = 1;
1686         req->lvl = hw_lvl;
1687         req->reg[0] = reg;
1688         req->num_regs = 1;
1689
1690         rc = otx2_mbox_process_msg(mbox, (void **)&rsp);
1691         if (rc)
1692                 return rc;
1693         *regval = rsp->regval[0];
1694         return 0;
1695 }
1696
1697 /* Search for min rate in topology */
1698 static void
1699 nix_tm_shaper_profile_update_min(struct otx2_eth_dev *dev)
1700 {
1701         struct otx2_nix_tm_shaper_profile *profile;
1702         uint64_t rate_min = 1E9; /* 1 Gbps */
1703
1704         TAILQ_FOREACH(profile, &dev->shaper_profile_list, shaper) {
1705                 if (profile->params.peak.rate &&
1706                     profile->params.peak.rate < rate_min)
1707                         rate_min = profile->params.peak.rate;
1708
1709                 if (profile->params.committed.rate &&
1710                     profile->params.committed.rate < rate_min)
1711                         rate_min = profile->params.committed.rate;
1712         }
1713
1714         dev->tm_rate_min = rate_min;
1715 }
1716
1717 static int
1718 nix_xmit_disable(struct rte_eth_dev *eth_dev)
1719 {
1720         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1721         uint16_t sq_cnt = eth_dev->data->nb_tx_queues;
1722         uint16_t sqb_cnt, head_off, tail_off;
1723         struct otx2_nix_tm_node *tm_node;
1724         struct otx2_eth_txq *txq;
1725         uint64_t wdata, val;
1726         int i, rc;
1727
1728         otx2_tm_dbg("Disabling xmit on %s", eth_dev->data->name);
1729
1730         /* Enable CGX RXTX to drain pkts */
1731         if (!eth_dev->data->dev_started) {
1732                 otx2_mbox_alloc_msg_nix_lf_start_rx(dev->mbox);
1733                 rc = otx2_mbox_process(dev->mbox);
1734                 if (rc)
1735                         return rc;
1736         }
1737
1738         /* XON all SMQ's */
1739         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1740                 if (tm_node->hw_lvl != NIX_TXSCH_LVL_SMQ)
1741                         continue;
1742                 if (!(tm_node->flags & NIX_TM_NODE_HWRES))
1743                         continue;
1744
1745                 rc = nix_smq_xoff(dev, tm_node, false);
1746                 if (rc) {
1747                         otx2_err("Failed to enable smq %u, rc=%d",
1748                                  tm_node->hw_id, rc);
1749                         goto cleanup;
1750                 }
1751         }
1752
1753         /* Flush all tx queues */
1754         for (i = 0; i < sq_cnt; i++) {
1755                 txq = eth_dev->data->tx_queues[i];
1756
1757                 rc = otx2_nix_sq_sqb_aura_fc(txq, false);
1758                 if (rc) {
1759                         otx2_err("Failed to disable sqb aura fc, rc=%d", rc);
1760                         goto cleanup;
1761                 }
1762
1763                 /* Wait for sq entries to be flushed */
1764                 rc = nix_txq_flush_sq_spin(txq);
1765                 if (rc) {
1766                         otx2_err("Failed to drain sq, rc=%d\n", rc);
1767                         goto cleanup;
1768                 }
1769         }
1770
1771         /* XOFF & Flush all SMQ's. HRM mandates
1772          * all SQ's empty before SMQ flush is issued.
1773          */
1774         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
1775                 if (tm_node->hw_lvl != NIX_TXSCH_LVL_SMQ)
1776                         continue;
1777                 if (!(tm_node->flags & NIX_TM_NODE_HWRES))
1778                         continue;
1779
1780                 rc = nix_smq_xoff(dev, tm_node, true);
1781                 if (rc) {
1782                         otx2_err("Failed to enable smq %u, rc=%d",
1783                                  tm_node->hw_id, rc);
1784                         goto cleanup;
1785                 }
1786         }
1787
1788         /* Verify sanity of all tx queues */
1789         for (i = 0; i < sq_cnt; i++) {
1790                 txq = eth_dev->data->tx_queues[i];
1791
1792                 wdata = ((uint64_t)txq->sq << 32);
1793                 val = otx2_atomic64_add_nosync(wdata,
1794                                (int64_t *)(dev->base + NIX_LF_SQ_OP_STATUS));
1795
1796                 sqb_cnt = val & 0xFFFF;
1797                 head_off = (val >> 20) & 0x3F;
1798                 tail_off = (val >> 28) & 0x3F;
1799
1800                 if (sqb_cnt > 1 || head_off != tail_off ||
1801                     (*txq->fc_mem != txq->nb_sqb_bufs))
1802                         otx2_err("Failed to gracefully flush sq %u", txq->sq);
1803         }
1804
1805 cleanup:
1806         /* restore cgx state */
1807         if (!eth_dev->data->dev_started) {
1808                 otx2_mbox_alloc_msg_nix_lf_stop_rx(dev->mbox);
1809                 rc |= otx2_mbox_process(dev->mbox);
1810         }
1811
1812         return rc;
1813 }
1814
1815 static int
1816 otx2_nix_tm_node_type_get(struct rte_eth_dev *eth_dev, uint32_t node_id,
1817                           int *is_leaf, struct rte_tm_error *error)
1818 {
1819         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1820         struct otx2_nix_tm_node *tm_node;
1821
1822         if (is_leaf == NULL) {
1823                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
1824                 return -EINVAL;
1825         }
1826
1827         tm_node = nix_tm_node_search(dev, node_id, true);
1828         if (node_id == RTE_TM_NODE_ID_NULL || !tm_node) {
1829                 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
1830                 return -EINVAL;
1831         }
1832         if (nix_tm_is_leaf(dev, tm_node->lvl))
1833                 *is_leaf = true;
1834         else
1835                 *is_leaf = false;
1836
1837         return 0;
1838 }
1839
1840 static int
1841 otx2_nix_tm_shaper_profile_add(struct rte_eth_dev *eth_dev,
1842                                uint32_t profile_id,
1843                                struct rte_tm_shaper_params *params,
1844                                struct rte_tm_error *error)
1845 {
1846         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1847         struct otx2_nix_tm_shaper_profile *profile;
1848
1849         profile = nix_tm_shaper_profile_search(dev, profile_id);
1850         if (profile) {
1851                 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
1852                 error->message = "shaper profile ID exist";
1853                 return -EINVAL;
1854         }
1855
1856         /* Committed rate and burst size can be enabled/disabled */
1857         if (params->committed.size || params->committed.rate) {
1858                 if (params->committed.size < MIN_SHAPER_BURST ||
1859                     params->committed.size > MAX_SHAPER_BURST) {
1860                         error->type =
1861                                 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_SIZE;
1862                         return -EINVAL;
1863                 } else if (!shaper_rate_to_nix(params->committed.rate * 8,
1864                                                NULL, NULL, NULL)) {
1865                         error->type =
1866                                 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE;
1867                         error->message = "shaper committed rate invalid";
1868                         return -EINVAL;
1869                 }
1870         }
1871
1872         /* Peak rate and burst size can be enabled/disabled */
1873         if (params->peak.size || params->peak.rate) {
1874                 if (params->peak.size < MIN_SHAPER_BURST ||
1875                     params->peak.size > MAX_SHAPER_BURST) {
1876                         error->type =
1877                                 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE;
1878                         return -EINVAL;
1879                 } else if (!shaper_rate_to_nix(params->peak.rate * 8,
1880                                                NULL, NULL, NULL)) {
1881                         error->type =
1882                                 RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE;
1883                         error->message = "shaper peak rate invalid";
1884                         return -EINVAL;
1885                 }
1886         }
1887
1888         profile = rte_zmalloc("otx2_nix_tm_shaper_profile",
1889                               sizeof(struct otx2_nix_tm_shaper_profile), 0);
1890         if (!profile)
1891                 return -ENOMEM;
1892
1893         profile->shaper_profile_id = profile_id;
1894         rte_memcpy(&profile->params, params,
1895                    sizeof(struct rte_tm_shaper_params));
1896         TAILQ_INSERT_TAIL(&dev->shaper_profile_list, profile, shaper);
1897
1898         otx2_tm_dbg("Added TM shaper profile %u, "
1899                     " pir %" PRIu64 " , pbs %" PRIu64 ", cir %" PRIu64
1900                     ", cbs %" PRIu64 " , adj %u",
1901                     profile_id,
1902                     params->peak.rate * 8,
1903                     params->peak.size,
1904                     params->committed.rate * 8,
1905                     params->committed.size,
1906                     params->pkt_length_adjust);
1907
1908         /* Translate rate as bits per second */
1909         profile->params.peak.rate = profile->params.peak.rate * 8;
1910         profile->params.committed.rate = profile->params.committed.rate * 8;
1911         /* Always use PIR for single rate shaping */
1912         if (!params->peak.rate && params->committed.rate) {
1913                 profile->params.peak = profile->params.committed;
1914                 memset(&profile->params.committed, 0,
1915                        sizeof(profile->params.committed));
1916         }
1917
1918         /* update min rate */
1919         nix_tm_shaper_profile_update_min(dev);
1920         return 0;
1921 }
1922
1923 static int
1924 otx2_nix_tm_shaper_profile_delete(struct rte_eth_dev *eth_dev,
1925                                   uint32_t profile_id,
1926                                   struct rte_tm_error *error)
1927 {
1928         struct otx2_nix_tm_shaper_profile *profile;
1929         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1930
1931         profile = nix_tm_shaper_profile_search(dev, profile_id);
1932
1933         if (!profile) {
1934                 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
1935                 error->message = "shaper profile ID not exist";
1936                 return -EINVAL;
1937         }
1938
1939         if (profile->reference_count) {
1940                 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE;
1941                 error->message = "shaper profile in use";
1942                 return -EINVAL;
1943         }
1944
1945         otx2_tm_dbg("Removing TM shaper profile %u", profile_id);
1946         TAILQ_REMOVE(&dev->shaper_profile_list, profile, shaper);
1947         rte_free(profile);
1948
1949         /* update min rate */
1950         nix_tm_shaper_profile_update_min(dev);
1951         return 0;
1952 }
1953
1954 static int
1955 otx2_nix_tm_node_add(struct rte_eth_dev *eth_dev, uint32_t node_id,
1956                      uint32_t parent_node_id, uint32_t priority,
1957                      uint32_t weight, uint32_t lvl,
1958                      struct rte_tm_node_params *params,
1959                      struct rte_tm_error *error)
1960 {
1961         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
1962         struct otx2_nix_tm_node *parent_node;
1963         int rc, clear_on_fail = 0;
1964         uint32_t exp_next_lvl;
1965         uint16_t hw_lvl;
1966
1967         /* we don't support dynamic updates */
1968         if (dev->tm_flags & NIX_TM_COMMITTED) {
1969                 error->type = RTE_TM_ERROR_TYPE_CAPABILITIES;
1970                 error->message = "dynamic update not supported";
1971                 return -EIO;
1972         }
1973
1974         /* Leaf nodes have to be same priority */
1975         if (nix_tm_is_leaf(dev, lvl) && priority != 0) {
1976                 error->type = RTE_TM_ERROR_TYPE_CAPABILITIES;
1977                 error->message = "queue shapers must be priority 0";
1978                 return -EIO;
1979         }
1980
1981         parent_node = nix_tm_node_search(dev, parent_node_id, true);
1982
1983         /* find the right level */
1984         if (lvl == RTE_TM_NODE_LEVEL_ID_ANY) {
1985                 if (parent_node_id == RTE_TM_NODE_ID_NULL) {
1986                         lvl = OTX2_TM_LVL_ROOT;
1987                 } else if (parent_node) {
1988                         lvl = parent_node->lvl + 1;
1989                 } else {
1990                         /* Neigher proper parent nor proper level id given */
1991                         error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
1992                         error->message = "invalid parent node id";
1993                         return -ERANGE;
1994                 }
1995         }
1996
1997         /* Translate rte_tm level id's to nix hw level id's */
1998         hw_lvl = nix_tm_lvl2nix(dev, lvl);
1999         if (hw_lvl == NIX_TXSCH_LVL_CNT &&
2000             !nix_tm_is_leaf(dev, lvl)) {
2001                 error->type = RTE_TM_ERROR_TYPE_LEVEL_ID;
2002                 error->message = "invalid level id";
2003                 return -ERANGE;
2004         }
2005
2006         if (node_id < dev->tm_leaf_cnt)
2007                 exp_next_lvl = NIX_TXSCH_LVL_SMQ;
2008         else
2009                 exp_next_lvl = hw_lvl + 1;
2010
2011         /* Check if there is no parent node yet */
2012         if (hw_lvl != dev->otx2_tm_root_lvl &&
2013             (!parent_node || parent_node->hw_lvl != exp_next_lvl)) {
2014                 error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
2015                 error->message = "invalid parent node id";
2016                 return -EINVAL;
2017         }
2018
2019         /* Check if a node already exists */
2020         if (nix_tm_node_search(dev, node_id, true)) {
2021                 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2022                 error->message = "node already exists";
2023                 return -EINVAL;
2024         }
2025
2026         /* Check if shaper profile exists for non leaf node */
2027         if (!nix_tm_is_leaf(dev, lvl) &&
2028             params->shaper_profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE &&
2029             !nix_tm_shaper_profile_search(dev, params->shaper_profile_id)) {
2030                 error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
2031                 error->message = "invalid shaper profile";
2032                 return -EINVAL;
2033         }
2034
2035         /* Check if there is second DWRR already in siblings or holes in prio */
2036         if (validate_prio(dev, lvl, parent_node_id, priority, error))
2037                 return -EINVAL;
2038
2039         if (weight > MAX_SCHED_WEIGHT) {
2040                 error->type = RTE_TM_ERROR_TYPE_NODE_WEIGHT;
2041                 error->message = "max weight exceeded";
2042                 return -EINVAL;
2043         }
2044
2045         rc = nix_tm_node_add_to_list(dev, node_id, parent_node_id,
2046                                      priority, weight, hw_lvl,
2047                                      lvl, true, params);
2048         if (rc) {
2049                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2050                 /* cleanup user added nodes */
2051                 if (clear_on_fail)
2052                         nix_tm_free_resources(dev, NIX_TM_NODE_USER,
2053                                               NIX_TM_NODE_USER, false);
2054                 error->message = "failed to add node";
2055                 return rc;
2056         }
2057         error->type = RTE_TM_ERROR_TYPE_NONE;
2058         return 0;
2059 }
2060
2061 static int
2062 otx2_nix_tm_node_delete(struct rte_eth_dev *eth_dev, uint32_t node_id,
2063                         struct rte_tm_error *error)
2064 {
2065         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2066         struct otx2_nix_tm_node *tm_node, *child_node;
2067         struct otx2_nix_tm_shaper_profile *profile;
2068         uint32_t profile_id;
2069
2070         /* we don't support dynamic updates yet */
2071         if (dev->tm_flags & NIX_TM_COMMITTED) {
2072                 error->type = RTE_TM_ERROR_TYPE_CAPABILITIES;
2073                 error->message = "hierarchy exists";
2074                 return -EIO;
2075         }
2076
2077         if (node_id == RTE_TM_NODE_ID_NULL) {
2078                 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2079                 error->message = "invalid node id";
2080                 return -EINVAL;
2081         }
2082
2083         tm_node = nix_tm_node_search(dev, node_id, true);
2084         if (!tm_node) {
2085                 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2086                 error->message = "no such node";
2087                 return -EINVAL;
2088         }
2089
2090         /* Check for any existing children */
2091         TAILQ_FOREACH(child_node, &dev->node_list, node) {
2092                 if (child_node->parent == tm_node) {
2093                         error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2094                         error->message = "children exist";
2095                         return -EINVAL;
2096                 }
2097         }
2098
2099         /* Remove shaper profile reference */
2100         profile_id = tm_node->params.shaper_profile_id;
2101         profile = nix_tm_shaper_profile_search(dev, profile_id);
2102         profile->reference_count--;
2103
2104         TAILQ_REMOVE(&dev->node_list, tm_node, node);
2105         rte_free(tm_node);
2106         return 0;
2107 }
2108
2109 static int
2110 nix_tm_node_suspend_resume(struct rte_eth_dev *eth_dev, uint32_t node_id,
2111                            struct rte_tm_error *error, bool suspend)
2112 {
2113         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2114         struct otx2_mbox *mbox = dev->mbox;
2115         struct otx2_nix_tm_node *tm_node;
2116         struct nix_txschq_config *req;
2117         uint16_t flags;
2118         int rc;
2119
2120         tm_node = nix_tm_node_search(dev, node_id, true);
2121         if (!tm_node) {
2122                 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2123                 error->message = "no such node";
2124                 return -EINVAL;
2125         }
2126
2127         if (!(dev->tm_flags & NIX_TM_COMMITTED)) {
2128                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2129                 error->message = "hierarchy doesn't exist";
2130                 return -EINVAL;
2131         }
2132
2133         flags = tm_node->flags;
2134         flags = suspend ? (flags & ~NIX_TM_NODE_ENABLED) :
2135                 (flags | NIX_TM_NODE_ENABLED);
2136
2137         if (tm_node->flags == flags)
2138                 return 0;
2139
2140         /* send mbox for state change */
2141         req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
2142
2143         req->lvl = tm_node->hw_lvl;
2144         req->num_regs = prepare_tm_sw_xoff(tm_node, suspend,
2145                                            req->reg, req->regval);
2146         rc = send_tm_reqval(mbox, req, error);
2147         if (!rc)
2148                 tm_node->flags = flags;
2149         return rc;
2150 }
2151
2152 static int
2153 otx2_nix_tm_node_suspend(struct rte_eth_dev *eth_dev, uint32_t node_id,
2154                          struct rte_tm_error *error)
2155 {
2156         return nix_tm_node_suspend_resume(eth_dev, node_id, error, true);
2157 }
2158
2159 static int
2160 otx2_nix_tm_node_resume(struct rte_eth_dev *eth_dev, uint32_t node_id,
2161                         struct rte_tm_error *error)
2162 {
2163         return nix_tm_node_suspend_resume(eth_dev, node_id, error, false);
2164 }
2165
2166 static int
2167 otx2_nix_tm_hierarchy_commit(struct rte_eth_dev *eth_dev,
2168                              int clear_on_fail,
2169                              struct rte_tm_error *error)
2170 {
2171         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2172         struct otx2_nix_tm_node *tm_node;
2173         uint32_t leaf_cnt = 0;
2174         int rc;
2175
2176         if (dev->tm_flags & NIX_TM_COMMITTED) {
2177                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2178                 error->message = "hierarchy exists";
2179                 return -EINVAL;
2180         }
2181
2182         /* Check if we have all the leaf nodes */
2183         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
2184                 if (tm_node->flags & NIX_TM_NODE_USER &&
2185                     tm_node->id < dev->tm_leaf_cnt)
2186                         leaf_cnt++;
2187         }
2188
2189         if (leaf_cnt != dev->tm_leaf_cnt) {
2190                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2191                 error->message = "incomplete hierarchy";
2192                 return -EINVAL;
2193         }
2194
2195         /*
2196          * Disable xmit will be enabled when
2197          * new topology is available.
2198          */
2199         rc = nix_xmit_disable(eth_dev);
2200         if (rc) {
2201                 otx2_err("failed to disable TX, rc=%d", rc);
2202                 return -EIO;
2203         }
2204
2205         /* Delete default/ratelimit tree */
2206         if (dev->tm_flags & (NIX_TM_DEFAULT_TREE)) {
2207                 rc = nix_tm_free_resources(dev, NIX_TM_NODE_USER, 0, false);
2208                 if (rc) {
2209                         error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2210                         error->message = "failed to free default resources";
2211                         return rc;
2212                 }
2213                 dev->tm_flags &= ~(NIX_TM_DEFAULT_TREE);
2214         }
2215
2216         /* Free up user alloc'ed resources */
2217         rc = nix_tm_free_resources(dev, NIX_TM_NODE_USER,
2218                                    NIX_TM_NODE_USER, true);
2219         if (rc) {
2220                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2221                 error->message = "failed to free user resources";
2222                 return rc;
2223         }
2224
2225         rc = nix_tm_alloc_resources(eth_dev, true);
2226         if (rc) {
2227                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2228                 error->message = "alloc resources failed";
2229                 /* TODO should we restore default config ? */
2230                 if (clear_on_fail)
2231                         nix_tm_free_resources(dev, 0, 0, false);
2232                 return rc;
2233         }
2234
2235         error->type = RTE_TM_ERROR_TYPE_NONE;
2236         dev->tm_flags |= NIX_TM_COMMITTED;
2237         return 0;
2238 }
2239
2240 static int
2241 otx2_nix_tm_node_shaper_update(struct rte_eth_dev *eth_dev,
2242                                uint32_t node_id,
2243                                uint32_t profile_id,
2244                                struct rte_tm_error *error)
2245 {
2246         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2247         struct otx2_nix_tm_shaper_profile *profile = NULL;
2248         struct otx2_mbox *mbox = dev->mbox;
2249         struct otx2_nix_tm_node *tm_node;
2250         struct nix_txschq_config *req;
2251         uint8_t k;
2252         int rc;
2253
2254         tm_node = nix_tm_node_search(dev, node_id, true);
2255         if (!tm_node || nix_tm_is_leaf(dev, tm_node->lvl)) {
2256                 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2257                 error->message = "invalid node";
2258                 return -EINVAL;
2259         }
2260
2261         if (profile_id == tm_node->params.shaper_profile_id)
2262                 return 0;
2263
2264         if (profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE) {
2265                 profile = nix_tm_shaper_profile_search(dev, profile_id);
2266                 if (!profile) {
2267                         error->type = RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID;
2268                         error->message = "shaper profile ID not exist";
2269                         return -EINVAL;
2270                 }
2271         }
2272
2273         tm_node->params.shaper_profile_id = profile_id;
2274
2275         /* Nothing to do if not yet committed */
2276         if (!(dev->tm_flags & NIX_TM_COMMITTED))
2277                 return 0;
2278
2279         tm_node->flags &= ~NIX_TM_NODE_ENABLED;
2280
2281         /* Flush the specific node with SW_XOFF */
2282         req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
2283         req->lvl = tm_node->hw_lvl;
2284         k = prepare_tm_sw_xoff(tm_node, true, req->reg, req->regval);
2285         req->num_regs = k;
2286
2287         rc = send_tm_reqval(mbox, req, error);
2288         if (rc)
2289                 return rc;
2290
2291         /* Update the PIR/CIR and clear SW XOFF */
2292         req = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox);
2293         req->lvl = tm_node->hw_lvl;
2294
2295         k = prepare_tm_shaper_reg(tm_node, profile, req->reg, req->regval);
2296
2297         k += prepare_tm_sw_xoff(tm_node, false, &req->reg[k], &req->regval[k]);
2298
2299         req->num_regs = k;
2300         rc = send_tm_reqval(mbox, req, error);
2301         if (!rc)
2302                 tm_node->flags |= NIX_TM_NODE_ENABLED;
2303         return rc;
2304 }
2305
2306 static int
2307 otx2_nix_tm_node_parent_update(struct rte_eth_dev *eth_dev,
2308                                uint32_t node_id, uint32_t new_parent_id,
2309                                uint32_t priority, uint32_t weight,
2310                                struct rte_tm_error *error)
2311 {
2312         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2313         struct otx2_nix_tm_node *tm_node, *sibling;
2314         struct otx2_nix_tm_node *new_parent;
2315         struct nix_txschq_config *req;
2316         uint8_t k;
2317         int rc;
2318
2319         if (!(dev->tm_flags & NIX_TM_COMMITTED)) {
2320                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2321                 error->message = "hierarchy doesn't exist";
2322                 return -EINVAL;
2323         }
2324
2325         tm_node = nix_tm_node_search(dev, node_id, true);
2326         if (!tm_node) {
2327                 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2328                 error->message = "no such node";
2329                 return -EINVAL;
2330         }
2331
2332         /* Parent id valid only for non root nodes */
2333         if (tm_node->hw_lvl != dev->otx2_tm_root_lvl) {
2334                 new_parent = nix_tm_node_search(dev, new_parent_id, true);
2335                 if (!new_parent) {
2336                         error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
2337                         error->message = "no such parent node";
2338                         return -EINVAL;
2339                 }
2340
2341                 /* Current support is only for dynamic weight update */
2342                 if (tm_node->parent != new_parent ||
2343                     tm_node->priority != priority) {
2344                         error->type = RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID;
2345                         error->message = "only weight update supported";
2346                         return -EINVAL;
2347                 }
2348         }
2349
2350         /* Skip if no change */
2351         if (tm_node->weight == weight)
2352                 return 0;
2353
2354         tm_node->weight = weight;
2355
2356         /* For leaf nodes, SQ CTX needs update */
2357         if (nix_tm_is_leaf(dev, tm_node->lvl)) {
2358                 /* Update SQ quantum data on the fly */
2359                 rc = nix_sq_sched_data(dev, tm_node, true);
2360                 if (rc) {
2361                         error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2362                         error->message = "sq sched data update failed";
2363                         return rc;
2364                 }
2365         } else {
2366                 /* XOFF Parent node */
2367                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2368                 req->lvl = tm_node->parent->hw_lvl;
2369                 req->num_regs = prepare_tm_sw_xoff(tm_node->parent, true,
2370                                                    req->reg, req->regval);
2371                 rc = send_tm_reqval(dev->mbox, req, error);
2372                 if (rc)
2373                         return rc;
2374
2375                 /* XOFF this node and all other siblings */
2376                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2377                 req->lvl = tm_node->hw_lvl;
2378
2379                 k = 0;
2380                 TAILQ_FOREACH(sibling, &dev->node_list, node) {
2381                         if (sibling->parent != tm_node->parent)
2382                                 continue;
2383                         k += prepare_tm_sw_xoff(sibling, true, &req->reg[k],
2384                                                 &req->regval[k]);
2385                 }
2386                 req->num_regs = k;
2387                 rc = send_tm_reqval(dev->mbox, req, error);
2388                 if (rc)
2389                         return rc;
2390
2391                 /* Update new weight for current node */
2392                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2393                 req->lvl = tm_node->hw_lvl;
2394                 req->num_regs = prepare_tm_sched_reg(dev, tm_node,
2395                                                      req->reg, req->regval);
2396                 rc = send_tm_reqval(dev->mbox, req, error);
2397                 if (rc)
2398                         return rc;
2399
2400                 /* XON this node and all other siblings */
2401                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2402                 req->lvl = tm_node->hw_lvl;
2403
2404                 k = 0;
2405                 TAILQ_FOREACH(sibling, &dev->node_list, node) {
2406                         if (sibling->parent != tm_node->parent)
2407                                 continue;
2408                         k += prepare_tm_sw_xoff(sibling, false, &req->reg[k],
2409                                                 &req->regval[k]);
2410                 }
2411                 req->num_regs = k;
2412                 rc = send_tm_reqval(dev->mbox, req, error);
2413                 if (rc)
2414                         return rc;
2415
2416                 /* XON Parent node */
2417                 req = otx2_mbox_alloc_msg_nix_txschq_cfg(dev->mbox);
2418                 req->lvl = tm_node->parent->hw_lvl;
2419                 req->num_regs = prepare_tm_sw_xoff(tm_node->parent, false,
2420                                                    req->reg, req->regval);
2421                 rc = send_tm_reqval(dev->mbox, req, error);
2422                 if (rc)
2423                         return rc;
2424         }
2425         return 0;
2426 }
2427
2428 static int
2429 otx2_nix_tm_node_stats_read(struct rte_eth_dev *eth_dev, uint32_t node_id,
2430                             struct rte_tm_node_stats *stats,
2431                             uint64_t *stats_mask, int clear,
2432                             struct rte_tm_error *error)
2433 {
2434         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2435         struct otx2_nix_tm_node *tm_node;
2436         uint64_t reg, val;
2437         int64_t *addr;
2438         int rc = 0;
2439
2440         tm_node = nix_tm_node_search(dev, node_id, true);
2441         if (!tm_node) {
2442                 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2443                 error->message = "no such node";
2444                 return -EINVAL;
2445         }
2446
2447         /* Stats support only for leaf node or TL1 root */
2448         if (nix_tm_is_leaf(dev, tm_node->lvl)) {
2449                 reg = (((uint64_t)tm_node->id) << 32);
2450
2451                 /* Packets */
2452                 addr = (int64_t *)(dev->base + NIX_LF_SQ_OP_PKTS);
2453                 val = otx2_atomic64_add_nosync(reg, addr);
2454                 if (val & OP_ERR)
2455                         val = 0;
2456                 stats->n_pkts = val - tm_node->last_pkts;
2457
2458                 /* Bytes */
2459                 addr = (int64_t *)(dev->base + NIX_LF_SQ_OP_OCTS);
2460                 val = otx2_atomic64_add_nosync(reg, addr);
2461                 if (val & OP_ERR)
2462                         val = 0;
2463                 stats->n_bytes = val - tm_node->last_bytes;
2464
2465                 if (clear) {
2466                         tm_node->last_pkts = stats->n_pkts;
2467                         tm_node->last_bytes = stats->n_bytes;
2468                 }
2469
2470                 *stats_mask = RTE_TM_STATS_N_PKTS | RTE_TM_STATS_N_BYTES;
2471
2472         } else if (tm_node->hw_lvl == NIX_TXSCH_LVL_TL1) {
2473                 error->type = RTE_TM_ERROR_TYPE_UNSPECIFIED;
2474                 error->message = "stats read error";
2475
2476                 /* RED Drop packets */
2477                 reg = NIX_AF_TL1X_DROPPED_PACKETS(tm_node->hw_id);
2478                 rc = read_tm_reg(dev->mbox, reg, &val, NIX_TXSCH_LVL_TL1);
2479                 if (rc)
2480                         goto exit;
2481                 stats->leaf.n_pkts_dropped[RTE_COLOR_RED] =
2482                                                 val - tm_node->last_pkts;
2483
2484                 /* RED Drop bytes */
2485                 reg = NIX_AF_TL1X_DROPPED_BYTES(tm_node->hw_id);
2486                 rc = read_tm_reg(dev->mbox, reg, &val, NIX_TXSCH_LVL_TL1);
2487                 if (rc)
2488                         goto exit;
2489                 stats->leaf.n_bytes_dropped[RTE_COLOR_RED] =
2490                                                 val - tm_node->last_bytes;
2491
2492                 /* Clear stats */
2493                 if (clear) {
2494                         tm_node->last_pkts =
2495                                 stats->leaf.n_pkts_dropped[RTE_COLOR_RED];
2496                         tm_node->last_bytes =
2497                                 stats->leaf.n_bytes_dropped[RTE_COLOR_RED];
2498                 }
2499
2500                 *stats_mask = RTE_TM_STATS_N_PKTS_RED_DROPPED |
2501                         RTE_TM_STATS_N_BYTES_RED_DROPPED;
2502
2503         } else {
2504                 error->type = RTE_TM_ERROR_TYPE_NODE_ID;
2505                 error->message = "unsupported node";
2506                 rc = -EINVAL;
2507         }
2508
2509 exit:
2510         return rc;
2511 }
2512
2513 const struct rte_tm_ops otx2_tm_ops = {
2514         .node_type_get = otx2_nix_tm_node_type_get,
2515
2516         .shaper_profile_add = otx2_nix_tm_shaper_profile_add,
2517         .shaper_profile_delete = otx2_nix_tm_shaper_profile_delete,
2518
2519         .node_add = otx2_nix_tm_node_add,
2520         .node_delete = otx2_nix_tm_node_delete,
2521         .node_suspend = otx2_nix_tm_node_suspend,
2522         .node_resume = otx2_nix_tm_node_resume,
2523         .hierarchy_commit = otx2_nix_tm_hierarchy_commit,
2524
2525         .node_shaper_update = otx2_nix_tm_node_shaper_update,
2526         .node_parent_update = otx2_nix_tm_node_parent_update,
2527         .node_stats_read = otx2_nix_tm_node_stats_read,
2528 };
2529
2530 static int
2531 nix_tm_prepare_default_tree(struct rte_eth_dev *eth_dev)
2532 {
2533         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2534         uint32_t def = eth_dev->data->nb_tx_queues;
2535         struct rte_tm_node_params params;
2536         uint32_t leaf_parent, i;
2537         int rc = 0, leaf_level;
2538
2539         /* Default params */
2540         memset(&params, 0, sizeof(params));
2541         params.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
2542
2543         if (nix_tm_have_tl1_access(dev)) {
2544                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL1;
2545                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
2546                                              DEFAULT_RR_WEIGHT,
2547                                              NIX_TXSCH_LVL_TL1,
2548                                              OTX2_TM_LVL_ROOT, false, &params);
2549                 if (rc)
2550                         goto exit;
2551                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
2552                                              DEFAULT_RR_WEIGHT,
2553                                              NIX_TXSCH_LVL_TL2,
2554                                              OTX2_TM_LVL_SCH1, false, &params);
2555                 if (rc)
2556                         goto exit;
2557
2558                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
2559                                              DEFAULT_RR_WEIGHT,
2560                                              NIX_TXSCH_LVL_TL3,
2561                                              OTX2_TM_LVL_SCH2, false, &params);
2562                 if (rc)
2563                         goto exit;
2564
2565                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
2566                                              DEFAULT_RR_WEIGHT,
2567                                              NIX_TXSCH_LVL_TL4,
2568                                              OTX2_TM_LVL_SCH3, false, &params);
2569                 if (rc)
2570                         goto exit;
2571
2572                 rc = nix_tm_node_add_to_list(dev, def + 4, def + 3, 0,
2573                                              DEFAULT_RR_WEIGHT,
2574                                              NIX_TXSCH_LVL_SMQ,
2575                                              OTX2_TM_LVL_SCH4, false, &params);
2576                 if (rc)
2577                         goto exit;
2578
2579                 leaf_parent = def + 4;
2580                 leaf_level = OTX2_TM_LVL_QUEUE;
2581         } else {
2582                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL2;
2583                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
2584                                              DEFAULT_RR_WEIGHT,
2585                                              NIX_TXSCH_LVL_TL2,
2586                                              OTX2_TM_LVL_ROOT, false, &params);
2587                 if (rc)
2588                         goto exit;
2589
2590                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
2591                                              DEFAULT_RR_WEIGHT,
2592                                              NIX_TXSCH_LVL_TL3,
2593                                              OTX2_TM_LVL_SCH1, false, &params);
2594                 if (rc)
2595                         goto exit;
2596
2597                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
2598                                              DEFAULT_RR_WEIGHT,
2599                                              NIX_TXSCH_LVL_TL4,
2600                                              OTX2_TM_LVL_SCH2, false, &params);
2601                 if (rc)
2602                         goto exit;
2603
2604                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
2605                                              DEFAULT_RR_WEIGHT,
2606                                              NIX_TXSCH_LVL_SMQ,
2607                                              OTX2_TM_LVL_SCH3, false, &params);
2608                 if (rc)
2609                         goto exit;
2610
2611                 leaf_parent = def + 3;
2612                 leaf_level = OTX2_TM_LVL_SCH4;
2613         }
2614
2615         /* Add leaf nodes */
2616         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
2617                 rc = nix_tm_node_add_to_list(dev, i, leaf_parent, 0,
2618                                              DEFAULT_RR_WEIGHT,
2619                                              NIX_TXSCH_LVL_CNT,
2620                                              leaf_level, false, &params);
2621                 if (rc)
2622                         break;
2623         }
2624
2625 exit:
2626         return rc;
2627 }
2628
2629 void otx2_nix_tm_conf_init(struct rte_eth_dev *eth_dev)
2630 {
2631         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2632
2633         TAILQ_INIT(&dev->node_list);
2634         TAILQ_INIT(&dev->shaper_profile_list);
2635         dev->tm_rate_min = 1E9; /* 1Gbps */
2636 }
2637
2638 int otx2_nix_tm_init_default(struct rte_eth_dev *eth_dev)
2639 {
2640         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2641         struct otx2_eth_dev  *dev = otx2_eth_pmd_priv(eth_dev);
2642         uint16_t sq_cnt = eth_dev->data->nb_tx_queues;
2643         int rc;
2644
2645         /* Free up all resources already held */
2646         rc = nix_tm_free_resources(dev, 0, 0, false);
2647         if (rc) {
2648                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
2649                 return rc;
2650         }
2651
2652         /* Clear shaper profiles */
2653         nix_tm_clear_shaper_profiles(dev);
2654         dev->tm_flags = NIX_TM_DEFAULT_TREE;
2655
2656         /* Disable TL1 Static Priority when VF's are enabled
2657          * as otherwise VF's TL2 reallocation will be needed
2658          * runtime to support a specific topology of PF.
2659          */
2660         if (pci_dev->max_vfs)
2661                 dev->tm_flags |= NIX_TM_TL1_NO_SP;
2662
2663         rc = nix_tm_prepare_default_tree(eth_dev);
2664         if (rc != 0)
2665                 return rc;
2666
2667         rc = nix_tm_alloc_resources(eth_dev, false);
2668         if (rc != 0)
2669                 return rc;
2670         dev->tm_leaf_cnt = sq_cnt;
2671
2672         return 0;
2673 }
2674
2675 int
2676 otx2_nix_tm_fini(struct rte_eth_dev *eth_dev)
2677 {
2678         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
2679         int rc;
2680
2681         /* Xmit is assumed to be disabled */
2682         /* Free up resources already held */
2683         rc = nix_tm_free_resources(dev, 0, 0, false);
2684         if (rc) {
2685                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
2686                 return rc;
2687         }
2688
2689         /* Clear shaper profiles */
2690         nix_tm_clear_shaper_profiles(dev);
2691
2692         dev->tm_flags = 0;
2693         return 0;
2694 }
2695
2696 int
2697 otx2_nix_tm_get_leaf_data(struct otx2_eth_dev *dev, uint16_t sq,
2698                           uint32_t *rr_quantum, uint16_t *smq)
2699 {
2700         struct otx2_nix_tm_node *tm_node;
2701         int rc;
2702
2703         /* 0..sq_cnt-1 are leaf nodes */
2704         if (sq >= dev->tm_leaf_cnt)
2705                 return -EINVAL;
2706
2707         /* Search for internal node first */
2708         tm_node = nix_tm_node_search(dev, sq, false);
2709         if (!tm_node)
2710                 tm_node = nix_tm_node_search(dev, sq, true);
2711
2712         /* Check if we found a valid leaf node */
2713         if (!tm_node || !nix_tm_is_leaf(dev, tm_node->lvl) ||
2714             !tm_node->parent || tm_node->parent->hw_id == UINT32_MAX) {
2715                 return -EIO;
2716         }
2717
2718         /* Get SMQ Id of leaf node's parent */
2719         *smq = tm_node->parent->hw_id;
2720         *rr_quantum = NIX_TM_WEIGHT_TO_RR_QUANTUM(tm_node->weight);
2721
2722         rc = nix_smq_xoff(dev, tm_node->parent, false);
2723         if (rc)
2724                 return rc;
2725         tm_node->flags |= NIX_TM_NODE_ENABLED;
2726
2727         return 0;
2728 }