drivers/common/cnxk/roc_nix_tm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include "roc_api.h"
   6 #include "roc_priv.h"
   7
   8 static inline int
   9 bitmap_ctzll(uint64_t slab)
  10 {
  11         if (slab == 0)
  12                 return 0;
  13
  14         return __builtin_ctzll(slab);
  15 }
  16
  17 void
  18 nix_tm_clear_shaper_profiles(struct nix *nix)
  19 {
  20         struct nix_tm_shaper_profile *shaper_profile, *tmp;
  21         struct nix_tm_shaper_profile_list *list;
  22
  23         list = &nix->shaper_profile_list;
  24         PLT_TAILQ_FOREACH_SAFE(shaper_profile, list, shaper, tmp) {
  25                 if (shaper_profile->ref_cnt)
  26                         plt_warn("Shaper profile %u has non zero references",
  27                                  shaper_profile->id);
  28                 TAILQ_REMOVE(&nix->shaper_profile_list, shaper_profile, shaper);
  29                 nix_tm_shaper_profile_free(shaper_profile);
  30         }
  31 }
  32
  33 static int
  34 nix_tm_node_reg_conf(struct nix *nix, struct nix_tm_node *node)
  35 {
  36         uint64_t regval_mask[MAX_REGS_PER_MBOX_MSG];
  37         uint64_t regval[MAX_REGS_PER_MBOX_MSG];
  38         struct nix_tm_shaper_profile *profile;
  39         uint64_t reg[MAX_REGS_PER_MBOX_MSG];
  40         struct mbox *mbox = (&nix->dev)->mbox;
  41         struct nix_txschq_config *req;
  42         int rc = -EFAULT;
  43         uint32_t hw_lvl;
  44         uint8_t k = 0;
  45
  46         memset(regval, 0, sizeof(regval));
  47         memset(regval_mask, 0, sizeof(regval_mask));
  48
  49         profile = nix_tm_shaper_profile_search(nix, node->shaper_profile_id);
  50         hw_lvl = node->hw_lvl;
  51
  52         /* Need this trigger to configure TL1 */
  53         if (!nix_tm_have_tl1_access(nix) && hw_lvl == NIX_TXSCH_LVL_TL2) {
  54                 /* Prepare default conf for TL1 */
  55                 req = mbox_alloc_msg_nix_txschq_cfg(mbox);
  56                 req->lvl = NIX_TXSCH_LVL_TL1;
  57
  58                 k = nix_tm_tl1_default_prep(node->parent_hw_id, req->reg,
  59                                             req->regval);
  60                 req->num_regs = k;
  61                 rc = mbox_process(mbox);
  62                 if (rc)
  63                         goto error;
  64         }
  65
  66         /* Prepare topology config */
  67         k = nix_tm_topology_reg_prep(nix, node, reg, regval, regval_mask);
  68
  69         /* Prepare schedule config */
  70         k += nix_tm_sched_reg_prep(nix, node, &reg[k], &regval[k]);
  71
  72         /* Prepare shaping config */
  73         k += nix_tm_shaper_reg_prep(node, profile, &reg[k], &regval[k]);
  74
  75         if (!k)
  76                 return 0;
  77
  78         /* Copy and send config mbox */
  79         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
  80         req->lvl = hw_lvl;
  81         req->num_regs = k;
  82
  83         mbox_memcpy(req->reg, reg, sizeof(uint64_t) * k);
  84         mbox_memcpy(req->regval, regval, sizeof(uint64_t) * k);
  85         mbox_memcpy(req->regval_mask, regval_mask, sizeof(uint64_t) * k);
  86
  87         rc = mbox_process(mbox);
  88         if (rc)
  89                 goto error;
  90
  91         return 0;
  92 error:
  93         plt_err("Txschq conf failed for node %p, rc=%d", node, rc);
  94         return rc;
  95 }
  96
  97 int
  98 nix_tm_txsch_reg_config(struct nix *nix, enum roc_nix_tm_tree tree)
  99 {
 100         struct nix_tm_node_list *list;
 101         bool is_pf_or_lbk = false;
 102         struct nix_tm_node *node;
 103         bool skip_bp = false;
 104         uint32_t hw_lvl;
 105         int rc = 0;
 106
 107         list = nix_tm_node_list(nix, tree);
 108
 109         if ((!dev_is_vf(&nix->dev) || nix->lbk_link) && !nix->sdp_link)
 110                 is_pf_or_lbk = true;
 111
 112         for (hw_lvl = 0; hw_lvl <= nix->tm_root_lvl; hw_lvl++) {
 113                 TAILQ_FOREACH(node, list, node) {
 114                         if (node->hw_lvl != hw_lvl)
 115                                 continue;
 116
 117                         /* Only one TL3/TL2 Link config should have BP enable
 118                          * set per channel only for PF or lbk vf.
 119                          */
 120                         node->bp_capa = 0;
 121                         if (is_pf_or_lbk && !skip_bp &&
 122                             node->hw_lvl == nix->tm_link_cfg_lvl) {
 123                                 node->bp_capa = 1;
 124                                 skip_bp = true;
 125                         }
 126
 127                         rc = nix_tm_node_reg_conf(nix, node);
 128                         if (rc)
 129                                 goto exit;
 130                 }
 131         }
 132 exit:
 133         return rc;
 134 }
 135
 136 int
 137 nix_tm_update_parent_info(struct nix *nix, enum roc_nix_tm_tree tree)
 138 {
 139         struct nix_tm_node *child, *parent;
 140         struct nix_tm_node_list *list;
 141         uint32_t rr_prio, max_prio;
 142         uint32_t rr_num = 0;
 143
 144         list = nix_tm_node_list(nix, tree);
 145
 146         /* Release all the node hw resources locally
 147          * if parent marked as dirty and resource exists.
 148          */
 149         TAILQ_FOREACH(child, list, node) {
 150                 /* Release resource only if parent direct hierarchy changed */
 151                 if (child->flags & NIX_TM_NODE_HWRES && child->parent &&
 152                     child->parent->child_realloc) {
 153                         nix_tm_free_node_resource(nix, child);
 154                 }
 155                 child->max_prio = UINT32_MAX;
 156         }
 157
 158         TAILQ_FOREACH(parent, list, node) {
 159                 /* Count group of children of same priority i.e are RR */
 160                 rr_num = nix_tm_check_rr(nix, parent->id, tree, &rr_prio,
 161                                          &max_prio);
 162
 163                 /* Assuming that multiple RR groups are
 164                  * not configured based on capability.
 165                  */
 166                 parent->rr_prio = rr_prio;
 167                 parent->rr_num = rr_num;
 168                 parent->max_prio = max_prio;
 169         }
 170
 171         return 0;
 172 }
 173
 174 static int
 175 nix_tm_root_node_get(struct nix *nix, int tree)
 176 {
 177         struct nix_tm_node_list *list = nix_tm_node_list(nix, tree);
 178         struct nix_tm_node *tm_node;
 179
 180         TAILQ_FOREACH(tm_node, list, node) {
 181                 if (tm_node->hw_lvl == nix->tm_root_lvl)
 182                         return 1;
 183         }
 184
 185         return 0;
 186 }
 187
 188 int
 189 nix_tm_node_add(struct roc_nix *roc_nix, struct nix_tm_node *node)
 190 {
 191         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
 192         struct nix_tm_shaper_profile *profile;
 193         uint32_t node_id, parent_id, lvl;
 194         struct nix_tm_node *parent_node;
 195         uint32_t priority, profile_id;
 196         uint8_t hw_lvl, exp_next_lvl;
 197         enum roc_nix_tm_tree tree;
 198         int rc;
 199
 200         node_id = node->id;
 201         priority = node->priority;
 202         parent_id = node->parent_id;
 203         profile_id = node->shaper_profile_id;
 204         lvl = node->lvl;
 205         tree = node->tree;
 206
 207         plt_tm_dbg("Add node %s lvl %u id %u, prio 0x%x weight 0x%x "
 208                    "parent %u profile 0x%x tree %u",
 209                    nix_tm_hwlvl2str(nix_tm_lvl2nix(nix, lvl)), lvl, node_id,
 210                    priority, node->weight, parent_id, profile_id, tree);
 211
 212         if (tree >= ROC_NIX_TM_TREE_MAX)
 213                 return NIX_ERR_PARAM;
 214
 215         /* Translate sw level id's to nix hw level id's */
 216         hw_lvl = nix_tm_lvl2nix(nix, lvl);
 217         if (hw_lvl == NIX_TXSCH_LVL_CNT && !nix_tm_is_leaf(nix, lvl))
 218                 return NIX_ERR_TM_INVALID_LVL;
 219
 220         /* Leaf nodes have to be same priority */
 221         if (nix_tm_is_leaf(nix, lvl) && priority != 0)
 222                 return NIX_ERR_TM_INVALID_PRIO;
 223
 224         parent_node = nix_tm_node_search(nix, parent_id, tree);
 225
 226         if (node_id < nix->nb_tx_queues)
 227                 exp_next_lvl = NIX_TXSCH_LVL_SMQ;
 228         else
 229                 exp_next_lvl = hw_lvl + 1;
 230
 231         /* Check if there is no parent node yet */
 232         if (hw_lvl != nix->tm_root_lvl &&
 233             (!parent_node || parent_node->hw_lvl != exp_next_lvl))
 234                 return NIX_ERR_TM_INVALID_PARENT;
 235
 236         /* Check if a node already exists */
 237         if (nix_tm_node_search(nix, node_id, tree))
 238                 return NIX_ERR_TM_NODE_EXISTS;
 239
 240         /* Check if root node exists */
 241         if (hw_lvl == nix->tm_root_lvl && nix_tm_root_node_get(nix, tree))
 242                 return NIX_ERR_TM_NODE_EXISTS;
 243
 244         profile = nix_tm_shaper_profile_search(nix, profile_id);
 245         if (!nix_tm_is_leaf(nix, lvl)) {
 246                 /* Check if shaper profile exists for non leaf node */
 247                 if (!profile && profile_id != ROC_NIX_TM_SHAPER_PROFILE_NONE)
 248                         return NIX_ERR_TM_INVALID_SHAPER_PROFILE;
 249
 250                 /* Packet mode in profile should match with that of tm node */
 251                 if (profile && profile->pkt_mode != node->pkt_mode)
 252                         return NIX_ERR_TM_PKT_MODE_MISMATCH;
 253         }
 254
 255         /* Check if there is second DWRR already in siblings or holes in prio */
 256         rc = nix_tm_validate_prio(nix, lvl, parent_id, priority, tree);
 257         if (rc)
 258                 return rc;
 259
 260         if (node->weight > roc_nix_tm_max_sched_wt_get())
 261                 return NIX_ERR_TM_WEIGHT_EXCEED;
 262
 263         /* Maintain minimum weight */
 264         if (!node->weight)
 265                 node->weight = 1;
 266
 267         node->hw_lvl = nix_tm_lvl2nix(nix, lvl);
 268         node->rr_prio = 0xF;
 269         node->max_prio = UINT32_MAX;
 270         node->hw_id = NIX_TM_HW_ID_INVALID;
 271         node->flags = 0;
 272
 273         if (profile)
 274                 profile->ref_cnt++;
 275
 276         node->parent = parent_node;
 277         if (parent_node)
 278                 parent_node->child_realloc = true;
 279         node->parent_hw_id = NIX_TM_HW_ID_INVALID;
 280
 281         TAILQ_INSERT_TAIL(&nix->trees[tree], node, node);
 282         plt_tm_dbg("Added node %s lvl %u id %u (%p)",
 283                    nix_tm_hwlvl2str(node->hw_lvl), lvl, node_id, node);
 284         return 0;
 285 }
 286
 287 int
 288 nix_tm_clear_path_xoff(struct nix *nix, struct nix_tm_node *node)
 289 {
 290         struct mbox *mbox = (&nix->dev)->mbox;
 291         struct nix_txschq_config *req;
 292         struct nix_tm_node *p;
 293         int rc;
 294
 295         /* Enable nodes in path for flush to succeed */
 296         if (!nix_tm_is_leaf(nix, node->lvl))
 297                 p = node;
 298         else
 299                 p = node->parent;
 300         while (p) {
 301                 if (!(p->flags & NIX_TM_NODE_ENABLED) &&
 302                     (p->flags & NIX_TM_NODE_HWRES)) {
 303                         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
 304                         req->lvl = p->hw_lvl;
 305                         req->num_regs = nix_tm_sw_xoff_prep(p, false, req->reg,
 306                                                             req->regval);
 307                         rc = mbox_process(mbox);
 308                         if (rc)
 309                                 return rc;
 310
 311                         p->flags |= NIX_TM_NODE_ENABLED;
 312                 }
 313                 p = p->parent;
 314         }
 315
 316         return 0;
 317 }
 318
 319 int
 320 nix_tm_bp_config_set(struct roc_nix *roc_nix, bool enable)
 321 {
 322         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
 323         enum roc_nix_tm_tree tree = nix->tm_tree;
 324         struct mbox *mbox = (&nix->dev)->mbox;
 325         struct nix_txschq_config *req = NULL;
 326         struct nix_tm_node_list *list;
 327         struct nix_tm_node *node;
 328         uint8_t k = 0;
 329         uint16_t link;
 330         int rc = 0;
 331
 332         list = nix_tm_node_list(nix, tree);
 333         link = nix->tx_link;
 334
 335         TAILQ_FOREACH(node, list, node) {
 336                 if (node->hw_lvl != nix->tm_link_cfg_lvl)
 337                         continue;
 338
 339                 if (!(node->flags & NIX_TM_NODE_HWRES) || !node->bp_capa)
 340                         continue;
 341
 342                 if (!req) {
 343                         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
 344                         req->lvl = nix->tm_link_cfg_lvl;
 345                         k = 0;
 346                 }
 347
 348                 req->reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(node->hw_id, link);
 349                 req->regval[k] = enable ? BIT_ULL(13) : 0;
 350                 req->regval_mask[k] = ~BIT_ULL(13);
 351                 k++;
 352
 353                 if (k >= MAX_REGS_PER_MBOX_MSG) {
 354                         req->num_regs = k;
 355                         rc = mbox_process(mbox);
 356                         if (rc)
 357                                 goto err;
 358                         req = NULL;
 359                 }
 360         }
 361
 362         if (req) {
 363                 req->num_regs = k;
 364                 rc = mbox_process(mbox);
 365                 if (rc)
 366                         goto err;
 367         }
 368
 369         return 0;
 370 err:
 371         plt_err("Failed to %s bp on link %u, rc=%d(%s)",
 372                 enable ? "enable" : "disable", link, rc, roc_error_msg_get(rc));
 373         return rc;
 374 }
 375
 376 int
 377 nix_tm_bp_config_get(struct roc_nix *roc_nix, bool *is_enabled)
 378 {
 379         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
 380         struct nix_txschq_config *req = NULL, *rsp;
 381         enum roc_nix_tm_tree tree = nix->tm_tree;
 382         struct mbox *mbox = (&nix->dev)->mbox;
 383         struct nix_tm_node_list *list;
 384         struct nix_tm_node *node;
 385         bool found = false;
 386         uint8_t enable = 1;
 387         uint8_t k = 0, i;
 388         uint16_t link;
 389         int rc = 0;
 390
 391         list = nix_tm_node_list(nix, tree);
 392         link = nix->tx_link;
 393
 394         TAILQ_FOREACH(node, list, node) {
 395                 if (node->hw_lvl != nix->tm_link_cfg_lvl)
 396                         continue;
 397
 398                 if (!(node->flags & NIX_TM_NODE_HWRES) || !node->bp_capa)
 399                         continue;
 400
 401                 found = true;
 402                 if (!req) {
 403                         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
 404                         req->read = 1;
 405                         req->lvl = nix->tm_link_cfg_lvl;
 406                         k = 0;
 407                 }
 408
 409                 req->reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(node->hw_id, link);
 410                 k++;
 411
 412                 if (k >= MAX_REGS_PER_MBOX_MSG) {
 413                         req->num_regs = k;
 414                         rc = mbox_process_msg(mbox, (void **)&rsp);
 415                         if (rc || rsp->num_regs != k)
 416                                 goto err;
 417                         req = NULL;
 418
 419                         /* Report it as enabled only if enabled or all */
 420                         for (i = 0; i < k; i++)
 421                                 enable &= !!(rsp->regval[i] & BIT_ULL(13));
 422                 }
 423         }
 424
 425         if (req) {
 426                 req->num_regs = k;
 427                 rc = mbox_process(mbox);
 428                 if (rc)
 429                         goto err;
 430                 /* Report it as enabled only if enabled or all */
 431                 for (i = 0; i < k; i++)
 432                         enable &= !!(rsp->regval[i] & BIT_ULL(13));
 433         }
 434
 435         *is_enabled = found ? !!enable : false;
 436         return 0;
 437 err:
 438         plt_err("Failed to get bp status on link %u, rc=%d(%s)", link, rc,
 439                 roc_error_msg_get(rc));
 440         return rc;
 441 }
 442
 443 int
 444 nix_tm_smq_xoff(struct nix *nix, struct nix_tm_node *node, bool enable)
 445 {
 446         struct mbox *mbox = (&nix->dev)->mbox;
 447         struct nix_txschq_config *req;
 448         uint16_t smq;
 449         int rc;
 450
 451         smq = node->hw_id;
 452         plt_tm_dbg("Setting SMQ %u XOFF/FLUSH to %s", smq,
 453                    enable ? "enable" : "disable");
 454
 455         rc = nix_tm_clear_path_xoff(nix, node);
 456         if (rc)
 457                 return rc;
 458
 459         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
 460         req->lvl = NIX_TXSCH_LVL_SMQ;
 461         req->num_regs = 1;
 462
 463         req->reg[0] = NIX_AF_SMQX_CFG(smq);
 464         req->regval[0] = enable ? (BIT_ULL(50) | BIT_ULL(49)) : 0;
 465         req->regval_mask[0] =
 466                 enable ? ~(BIT_ULL(50) | BIT_ULL(49)) : ~BIT_ULL(50);
 467
 468         return mbox_process(mbox);
 469 }
 470
 471 int
 472 nix_tm_leaf_data_get(struct nix *nix, uint16_t sq, uint32_t *rr_quantum,
 473                      uint16_t *smq)
 474 {
 475         struct nix_tm_node *node;
 476         int rc;
 477
 478         node = nix_tm_node_search(nix, sq, nix->tm_tree);
 479
 480         /* Check if we found a valid leaf node */
 481         if (!node || !nix_tm_is_leaf(nix, node->lvl) || !node->parent ||
 482             node->parent->hw_id == NIX_TM_HW_ID_INVALID) {
 483                 return -EIO;
 484         }
 485
 486         /* Get SMQ Id of leaf node's parent */
 487         *smq = node->parent->hw_id;
 488         *rr_quantum = nix_tm_weight_to_rr_quantum(node->weight);
 489
 490         rc = nix_tm_smq_xoff(nix, node->parent, false);
 491         if (rc)
 492                 return rc;
 493         node->flags |= NIX_TM_NODE_ENABLED;
 494         return 0;
 495 }
 496
 497 int
 498 roc_nix_tm_sq_flush_spin(struct roc_nix_sq *sq)
 499 {
 500         struct nix *nix = roc_nix_to_nix_priv(sq->roc_nix);
 501         uint16_t sqb_cnt, head_off, tail_off;
 502         uint64_t wdata, val, prev;
 503         uint16_t qid = sq->qid;
 504         int64_t *regaddr;
 505         uint64_t timeout; /* 10's of usec */
 506
 507         /* Wait for enough time based on shaper min rate */
 508         timeout = (sq->nb_desc * roc_nix_max_pkt_len(sq->roc_nix) * 8 * 1E5);
 509         /* Wait for worst case scenario of this SQ being last priority
 510          * and so have to wait for all other SQ's drain out by their own.
 511          */
 512         timeout = timeout * nix->nb_tx_queues;
 513         timeout = timeout / nix->tm_rate_min;
 514         if (!timeout)
 515                 timeout = 10000;
 516
 517         wdata = ((uint64_t)qid << 32);
 518         regaddr = (int64_t *)(nix->base + NIX_LF_SQ_OP_STATUS);
 519         val = roc_atomic64_add_nosync(wdata, regaddr);
 520
 521         /* Spin multiple iterations as "sq->fc_cache_pkts" can still
 522          * have space to send pkts even though fc_mem is disabled
 523          */
 524
 525         while (true) {
 526                 prev = val;
 527                 plt_delay_us(10);
 528                 val = roc_atomic64_add_nosync(wdata, regaddr);
 529                 /* Continue on error */
 530                 if (val & BIT_ULL(63))
 531                         continue;
 532
 533                 if (prev != val)
 534                         continue;
 535
 536                 sqb_cnt = val & 0xFFFF;
 537                 head_off = (val >> 20) & 0x3F;
 538                 tail_off = (val >> 28) & 0x3F;
 539
 540                 /* SQ reached quiescent state */
 541                 if (sqb_cnt <= 1 && head_off == tail_off &&
 542                     (*(volatile uint64_t *)sq->fc == sq->nb_sqb_bufs)) {
 543                         break;
 544                 }
 545
 546                 /* Timeout */
 547                 if (!timeout)
 548                         goto exit;
 549                 timeout--;
 550         }
 551
 552         return 0;
 553 exit:
 554         roc_nix_tm_dump(sq->roc_nix);
 555         roc_nix_queues_ctx_dump(sq->roc_nix);
 556         return -EFAULT;
 557 }
 558
 559 /* Flush and disable tx queue and its parent SMQ */
 560 int
 561 nix_tm_sq_flush_pre(struct roc_nix_sq *sq)
 562 {
 563         struct roc_nix *roc_nix = sq->roc_nix;
 564         struct nix_tm_node *node, *sibling;
 565         struct nix_tm_node_list *list;
 566         enum roc_nix_tm_tree tree;
 567         struct mbox *mbox;
 568         struct nix *nix;
 569         uint16_t qid;
 570         int rc;
 571
 572         nix = roc_nix_to_nix_priv(roc_nix);
 573
 574         /* Need not do anything if tree is in disabled state */
 575         if (!(nix->tm_flags & NIX_TM_HIERARCHY_ENA))
 576                 return 0;
 577
 578         mbox = (&nix->dev)->mbox;
 579         qid = sq->qid;
 580
 581         tree = nix->tm_tree;
 582         list = nix_tm_node_list(nix, tree);
 583
 584         /* Find the node for this SQ */
 585         node = nix_tm_node_search(nix, qid, tree);
 586         if (!node || !(node->flags & NIX_TM_NODE_ENABLED)) {
 587                 plt_err("Invalid node/state for sq %u", qid);
 588                 return -EFAULT;
 589         }
 590
 591         /* Enable CGX RXTX to drain pkts */
 592         if (!roc_nix->io_enabled) {
 593                 /* Though it enables both RX MCAM Entries and CGX Link
 594                  * we assume all the rx queues are stopped way back.
 595                  */
 596                 mbox_alloc_msg_nix_lf_start_rx(mbox);
 597                 rc = mbox_process(mbox);
 598                 if (rc) {
 599                         plt_err("cgx start failed, rc=%d", rc);
 600                         return rc;
 601                 }
 602         }
 603
 604         /* Disable backpressure */
 605         rc = nix_tm_bp_config_set(roc_nix, false);
 606         if (rc) {
 607                 plt_err("Failed to disable backpressure for flush, rc=%d", rc);
 608                 return rc;
 609         }
 610
 611         /* Disable smq xoff for case it was enabled earlier */
 612         rc = nix_tm_smq_xoff(nix, node->parent, false);
 613         if (rc) {
 614                 plt_err("Failed to enable smq %u, rc=%d", node->parent->hw_id,
 615                         rc);
 616                 return rc;
 617         }
 618
 619         /* As per HRM, to disable an SQ, all other SQ's
 620          * that feed to same SMQ must be paused before SMQ flush.
 621          */
 622         TAILQ_FOREACH(sibling, list, node) {
 623                 if (sibling->parent != node->parent)
 624                         continue;
 625                 if (!(sibling->flags & NIX_TM_NODE_ENABLED))
 626                         continue;
 627
 628                 qid = sibling->id;
 629                 sq = nix->sqs[qid];
 630                 if (!sq)
 631                         continue;
 632
 633                 rc = roc_nix_tm_sq_aura_fc(sq, false);
 634                 if (rc) {
 635                         plt_err("Failed to disable sqb aura fc, rc=%d", rc);
 636                         goto cleanup;
 637                 }
 638
 639                 /* Wait for sq entries to be flushed */
 640                 rc = roc_nix_tm_sq_flush_spin(sq);
 641                 if (rc) {
 642                         plt_err("Failed to drain sq %u, rc=%d\n", sq->qid, rc);
 643                         return rc;
 644                 }
 645         }
 646
 647         node->flags &= ~NIX_TM_NODE_ENABLED;
 648
 649         /* Disable and flush */
 650         rc = nix_tm_smq_xoff(nix, node->parent, true);
 651         if (rc) {
 652                 plt_err("Failed to disable smq %u, rc=%d", node->parent->hw_id,
 653                         rc);
 654                 goto cleanup;
 655         }
 656 cleanup:
 657         /* Restore cgx state */
 658         if (!roc_nix->io_enabled) {
 659                 mbox_alloc_msg_nix_lf_stop_rx(mbox);
 660                 rc |= mbox_process(mbox);
 661         }
 662
 663         return rc;
 664 }
 665
 666 int
 667 nix_tm_sq_flush_post(struct roc_nix_sq *sq)
 668 {
 669         struct roc_nix *roc_nix = sq->roc_nix;
 670         struct nix_tm_node *node, *sibling;
 671         struct nix_tm_node_list *list;
 672         enum roc_nix_tm_tree tree;
 673         struct roc_nix_sq *s_sq;
 674         bool once = false;
 675         uint16_t qid, s_qid;
 676         struct nix *nix;
 677         int rc;
 678
 679         nix = roc_nix_to_nix_priv(roc_nix);
 680
 681         /* Need not do anything if tree is in disabled state */
 682         if (!(nix->tm_flags & NIX_TM_HIERARCHY_ENA))
 683                 return 0;
 684
 685         qid = sq->qid;
 686         tree = nix->tm_tree;
 687         list = nix_tm_node_list(nix, tree);
 688
 689         /* Find the node for this SQ */
 690         node = nix_tm_node_search(nix, qid, tree);
 691         if (!node) {
 692                 plt_err("Invalid node for sq %u", qid);
 693                 return -EFAULT;
 694         }
 695
 696         /* Enable all the siblings back */
 697         TAILQ_FOREACH(sibling, list, node) {
 698                 if (sibling->parent != node->parent)
 699                         continue;
 700
 701                 if (sibling->id == qid)
 702                         continue;
 703
 704                 if (!(sibling->flags & NIX_TM_NODE_ENABLED))
 705                         continue;
 706
 707                 s_qid = sibling->id;
 708                 s_sq = nix->sqs[s_qid];
 709                 if (!s_sq)
 710                         continue;
 711
 712                 if (!once) {
 713                         /* Enable back if any SQ is still present */
 714                         rc = nix_tm_smq_xoff(nix, node->parent, false);
 715                         if (rc) {
 716                                 plt_err("Failed to enable smq %u, rc=%d",
 717                                         node->parent->hw_id, rc);
 718                                 return rc;
 719                         }
 720                         once = true;
 721                 }
 722
 723                 rc = roc_nix_tm_sq_aura_fc(s_sq, true);
 724                 if (rc) {
 725                         plt_err("Failed to enable sqb aura fc, rc=%d", rc);
 726                         return rc;
 727                 }
 728         }
 729
 730         if (!nix->rx_pause)
 731                 return 0;
 732
 733         /* Restore backpressure */
 734         rc = nix_tm_bp_config_set(roc_nix, true);
 735         if (rc) {
 736                 plt_err("Failed to restore backpressure, rc=%d", rc);
 737                 return rc;
 738         }
 739
 740         return 0;
 741 }
 742
 743 int
 744 nix_tm_sq_sched_conf(struct nix *nix, struct nix_tm_node *node,
 745                      bool rr_quantum_only)
 746 {
 747         struct mbox *mbox = (&nix->dev)->mbox;
 748         uint16_t qid = node->id, smq;
 749         uint64_t rr_quantum;
 750         int rc;
 751
 752         smq = node->parent->hw_id;
 753         rr_quantum = nix_tm_weight_to_rr_quantum(node->weight);
 754
 755         if (rr_quantum_only)
 756                 plt_tm_dbg("Update sq(%u) rr_quantum 0x%" PRIx64, qid,
 757                            rr_quantum);
 758         else
 759                 plt_tm_dbg("Enabling sq(%u)->smq(%u), rr_quantum 0x%" PRIx64,
 760                            qid, smq, rr_quantum);
 761
 762         if (qid > nix->nb_tx_queues)
 763                 return -EFAULT;
 764
 765         if (roc_model_is_cn9k()) {
 766                 struct nix_aq_enq_req *aq;
 767
 768                 aq = mbox_alloc_msg_nix_aq_enq(mbox);
 769                 aq->qidx = qid;
 770                 aq->ctype = NIX_AQ_CTYPE_SQ;
 771                 aq->op = NIX_AQ_INSTOP_WRITE;
 772
 773                 /* smq update only when needed */
 774                 if (!rr_quantum_only) {
 775                         aq->sq.smq = smq;
 776                         aq->sq_mask.smq = ~aq->sq_mask.smq;
 777                 }
 778                 aq->sq.smq_rr_quantum = rr_quantum;
 779                 aq->sq_mask.smq_rr_quantum = ~aq->sq_mask.smq_rr_quantum;
 780         } else {
 781                 struct nix_cn10k_aq_enq_req *aq;
 782
 783                 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
 784                 aq->qidx = qid;
 785                 aq->ctype = NIX_AQ_CTYPE_SQ;
 786                 aq->op = NIX_AQ_INSTOP_WRITE;
 787
 788                 /* smq update only when needed */
 789                 if (!rr_quantum_only) {
 790                         aq->sq.smq = smq;
 791                         aq->sq_mask.smq = ~aq->sq_mask.smq;
 792                 }
 793                 aq->sq.smq_rr_weight = rr_quantum;
 794                 aq->sq_mask.smq_rr_weight = ~aq->sq_mask.smq_rr_weight;
 795         }
 796
 797         rc = mbox_process(mbox);
 798         if (rc)
 799                 plt_err("Failed to set smq, rc=%d", rc);
 800         return rc;
 801 }
 802
 803 int
 804 nix_tm_release_resources(struct nix *nix, uint8_t hw_lvl, bool contig,
 805                          bool above_thresh)
 806 {
 807         uint16_t avail, thresh, to_free = 0, schq;
 808         struct mbox *mbox = (&nix->dev)->mbox;
 809         struct nix_txsch_free_req *req;
 810         struct plt_bitmap *bmp;
 811         uint64_t slab = 0;
 812         uint32_t pos = 0;
 813         int rc = -ENOSPC;
 814
 815         bmp = contig ? nix->schq_contig_bmp[hw_lvl] : nix->schq_bmp[hw_lvl];
 816         thresh =
 817                 contig ? nix->contig_rsvd[hw_lvl] : nix->discontig_rsvd[hw_lvl];
 818         plt_bitmap_scan_init(bmp);
 819
 820         avail = nix_tm_resource_avail(nix, hw_lvl, contig);
 821
 822         if (above_thresh) {
 823                 /* Release only above threshold */
 824                 if (avail > thresh)
 825                         to_free = avail - thresh;
 826         } else {
 827                 /* Release everything */
 828                 to_free = avail;
 829         }
 830
 831         /* Now release resources to AF */
 832         while (to_free) {
 833                 if (!slab && !plt_bitmap_scan(bmp, &pos, &slab))
 834                         break;
 835
 836                 schq = bitmap_ctzll(slab);
 837                 slab &= ~(1ULL << schq);
 838                 schq += pos;
 839
 840                 /* Free to AF */
 841                 req = mbox_alloc_msg_nix_txsch_free(mbox);
 842                 if (req == NULL)
 843                         return rc;
 844                 req->flags = 0;
 845                 req->schq_lvl = hw_lvl;
 846                 req->schq = schq;
 847                 rc = mbox_process(mbox);
 848                 if (rc) {
 849                         plt_err("failed to release hwres %s(%u) rc %d",
 850                                 nix_tm_hwlvl2str(hw_lvl), schq, rc);
 851                         return rc;
 852                 }
 853
 854                 plt_tm_dbg("Released hwres %s(%u)", nix_tm_hwlvl2str(hw_lvl),
 855                            schq);
 856                 plt_bitmap_clear(bmp, schq);
 857                 to_free--;
 858         }
 859
 860         if (to_free) {
 861                 plt_err("resource inconsistency for %s(%u)",
 862                         nix_tm_hwlvl2str(hw_lvl), contig);
 863                 return -EFAULT;
 864         }
 865         return 0;
 866 }
 867
 868 int
 869 nix_tm_free_node_resource(struct nix *nix, struct nix_tm_node *node)
 870 {
 871         struct mbox *mbox = (&nix->dev)->mbox;
 872         struct nix_txsch_free_req *req;
 873         struct plt_bitmap *bmp;
 874         uint16_t avail, hw_id;
 875         uint8_t hw_lvl;
 876         int rc = -ENOSPC;
 877
 878         hw_lvl = node->hw_lvl;
 879         hw_id = node->hw_id;
 880         bmp = nix->schq_bmp[hw_lvl];
 881         /* Free specific HW resource */
 882         plt_tm_dbg("Free hwres %s(%u) lvl %u id %u (%p)",
 883                    nix_tm_hwlvl2str(node->hw_lvl), hw_id, node->lvl, node->id,
 884                    node);
 885
 886         avail = nix_tm_resource_avail(nix, hw_lvl, false);
 887         /* Always for now free to discontiguous queue when avail
 888          * is not sufficient.
 889          */
 890         if (nix->discontig_rsvd[hw_lvl] &&
 891             avail < nix->discontig_rsvd[hw_lvl]) {
 892                 PLT_ASSERT(hw_id < NIX_TM_MAX_HW_TXSCHQ);
 893                 PLT_ASSERT(plt_bitmap_get(bmp, hw_id) == 0);
 894                 plt_bitmap_set(bmp, hw_id);
 895                 node->hw_id = NIX_TM_HW_ID_INVALID;
 896                 node->flags &= ~NIX_TM_NODE_HWRES;
 897                 return 0;
 898         }
 899
 900         /* Free to AF */
 901         req = mbox_alloc_msg_nix_txsch_free(mbox);
 902         if (req == NULL)
 903                 return rc;
 904         req->flags = 0;
 905         req->schq_lvl = node->hw_lvl;
 906         req->schq = hw_id;
 907         rc = mbox_process(mbox);
 908         if (rc) {
 909                 plt_err("failed to release hwres %s(%u) rc %d",
 910                         nix_tm_hwlvl2str(node->hw_lvl), hw_id, rc);
 911                 return rc;
 912         }
 913
 914         /* Mark parent as dirty for reallocing it's children */
 915         if (node->parent)
 916                 node->parent->child_realloc = true;
 917
 918         node->hw_id = NIX_TM_HW_ID_INVALID;
 919         node->flags &= ~NIX_TM_NODE_HWRES;
 920         plt_tm_dbg("Released hwres %s(%u) to af",
 921                    nix_tm_hwlvl2str(node->hw_lvl), hw_id);
 922         return 0;
 923 }
 924
 925 int
 926 nix_tm_node_delete(struct roc_nix *roc_nix, uint32_t node_id,
 927                    enum roc_nix_tm_tree tree, bool free)
 928 {
 929         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
 930         struct nix_tm_shaper_profile *profile;
 931         struct nix_tm_node *node, *child;
 932         struct nix_tm_node_list *list;
 933         uint32_t profile_id;
 934         int rc;
 935
 936         plt_tm_dbg("Delete node id %u tree %u", node_id, tree);
 937
 938         node = nix_tm_node_search(nix, node_id, tree);
 939         if (!node)
 940                 return NIX_ERR_TM_INVALID_NODE;
 941
 942         list = nix_tm_node_list(nix, tree);
 943         /* Check for any existing children */
 944         TAILQ_FOREACH(child, list, node) {
 945                 if (child->parent == node)
 946                         return NIX_ERR_TM_CHILD_EXISTS;
 947         }
 948
 949         /* Remove shaper profile reference */
 950         profile_id = node->shaper_profile_id;
 951         profile = nix_tm_shaper_profile_search(nix, profile_id);
 952
 953         /* Free hw resource locally */
 954         if (node->flags & NIX_TM_NODE_HWRES) {
 955                 rc = nix_tm_free_node_resource(nix, node);
 956                 if (rc)
 957                         return rc;
 958         }
 959
 960         if (profile)
 961                 profile->ref_cnt--;
 962
 963         TAILQ_REMOVE(list, node, node);
 964
 965         plt_tm_dbg("Deleted node %s lvl %u id %u, prio 0x%x weight 0x%x "
 966                    "parent %u profile 0x%x tree %u (%p)",
 967                    nix_tm_hwlvl2str(node->hw_lvl), node->lvl, node->id,
 968                    node->priority, node->weight,
 969                    node->parent ? node->parent->id : UINT32_MAX,
 970                    node->shaper_profile_id, tree, node);
 971         /* Free only if requested */
 972         if (free)
 973                 nix_tm_node_free(node);
 974         return 0;
 975 }
 976
 977 static int
 978 nix_tm_assign_hw_id(struct nix *nix, struct nix_tm_node *parent,
 979                     uint16_t *contig_id, int *contig_cnt,
 980                     struct nix_tm_node_list *list)
 981 {
 982         struct nix_tm_node *child;
 983         struct plt_bitmap *bmp;
 984         uint8_t child_hw_lvl;
 985         int spare_schq = -1;
 986         uint32_t pos = 0;
 987         uint64_t slab;
 988         uint16_t schq;
 989
 990         child_hw_lvl = parent->hw_lvl - 1;
 991         bmp = nix->schq_bmp[child_hw_lvl];
 992         plt_bitmap_scan_init(bmp);
 993         slab = 0;
 994
 995         /* Save spare schq if it is case of RR + SP */
 996         if (parent->rr_prio != 0xf && *contig_cnt > 1)
 997                 spare_schq = *contig_id + parent->rr_prio;
 998
 999         TAILQ_FOREACH(child, list, node) {
1000                 if (!child->parent)
1001                         continue;
1002                 if (child->parent->id != parent->id)
1003                         continue;
1004
1005                 /* Resource never expected to be present */
1006                 if (child->flags & NIX_TM_NODE_HWRES) {
1007                         plt_err("Resource exists for child (%s)%u, id %u (%p)",
1008                                 nix_tm_hwlvl2str(child->hw_lvl), child->hw_id,
1009                                 child->id, child);
1010                         return -EFAULT;
1011                 }
1012
1013                 if (!slab)
1014                         plt_bitmap_scan(bmp, &pos, &slab);
1015
1016                 if (child->priority == parent->rr_prio && spare_schq != -1) {
1017                         /* Use spare schq first if present */
1018                         schq = spare_schq;
1019                         spare_schq = -1;
1020                         *contig_cnt = *contig_cnt - 1;
1021
1022                 } else if (child->priority == parent->rr_prio) {
1023                         /* Assign a discontiguous queue */
1024                         if (!slab) {
1025                                 plt_err("Schq not found for Child %u "
1026                                         "lvl %u (%p)",
1027                                         child->id, child->lvl, child);
1028                                 return -ENOENT;
1029                         }
1030
1031                         schq = bitmap_ctzll(slab);
1032                         slab &= ~(1ULL << schq);
1033                         schq += pos;
1034                         plt_bitmap_clear(bmp, schq);
1035                 } else {
1036                         /* Assign a contiguous queue */
1037                         schq = *contig_id + child->priority;
1038                         *contig_cnt = *contig_cnt - 1;
1039                 }
1040
1041                 plt_tm_dbg("Resource %s(%u), for lvl %u id %u(%p)",
1042                            nix_tm_hwlvl2str(child->hw_lvl), schq, child->lvl,
1043                            child->id, child);
1044
1045                 child->hw_id = schq;
1046                 child->parent_hw_id = parent->hw_id;
1047                 child->flags |= NIX_TM_NODE_HWRES;
1048         }
1049
1050         return 0;
1051 }
1052
1053 int
1054 nix_tm_assign_resources(struct nix *nix, enum roc_nix_tm_tree tree)
1055 {
1056         struct nix_tm_node *parent, *root = NULL;
1057         struct plt_bitmap *bmp, *bmp_contig;
1058         struct nix_tm_node_list *list;
1059         uint8_t child_hw_lvl, hw_lvl;
1060         uint16_t contig_id, j;
1061         uint64_t slab = 0;
1062         uint32_t pos = 0;
1063         int cnt, rc;
1064
1065         list = nix_tm_node_list(nix, tree);
1066         /* Walk from TL1 to TL4 parents */
1067         for (hw_lvl = NIX_TXSCH_LVL_TL1; hw_lvl > 0; hw_lvl--) {
1068                 TAILQ_FOREACH(parent, list, node) {
1069                         child_hw_lvl = parent->hw_lvl - 1;
1070                         if (parent->hw_lvl != hw_lvl)
1071                                 continue;
1072
1073                         /* Remember root for future */
1074                         if (parent->hw_lvl == nix->tm_root_lvl)
1075                                 root = parent;
1076
1077                         if (!parent->child_realloc) {
1078                                 /* Skip when parent is not dirty */
1079                                 if (nix_tm_child_res_valid(list, parent))
1080                                         continue;
1081                                 plt_err("Parent not dirty but invalid "
1082                                         "child res parent id %u(lvl %u)",
1083                                         parent->id, parent->lvl);
1084                                 return -EFAULT;
1085                         }
1086
1087                         bmp_contig = nix->schq_contig_bmp[child_hw_lvl];
1088
1089                         /* Prealloc contiguous indices for a parent */
1090                         contig_id = NIX_TM_MAX_HW_TXSCHQ;
1091                         cnt = (int)parent->max_prio + 1;
1092                         if (cnt > 0) {
1093                                 plt_bitmap_scan_init(bmp_contig);
1094                                 if (!plt_bitmap_scan(bmp_contig, &pos, &slab)) {
1095                                         plt_err("Contig schq not found");
1096                                         return -ENOENT;
1097                                 }
1098                                 contig_id = pos + bitmap_ctzll(slab);
1099
1100                                 /* Check if we have enough */
1101                                 for (j = contig_id; j < contig_id + cnt; j++) {
1102                                         if (!plt_bitmap_get(bmp_contig, j))
1103                                                 break;
1104                                 }
1105
1106                                 if (j != contig_id + cnt) {
1107                                         plt_err("Contig schq not sufficient");
1108                                         return -ENOENT;
1109                                 }
1110
1111                                 for (j = contig_id; j < contig_id + cnt; j++)
1112                                         plt_bitmap_clear(bmp_contig, j);
1113                         }
1114
1115                         /* Assign hw id to all children */
1116                         rc = nix_tm_assign_hw_id(nix, parent, &contig_id, &cnt,
1117                                                  list);
1118                         if (cnt || rc) {
1119                                 plt_err("Unexpected err, contig res alloc, "
1120                                         "parent %u, of %s, rc=%d, cnt=%d",
1121                                         parent->id, nix_tm_hwlvl2str(hw_lvl),
1122                                         rc, cnt);
1123                                 return -EFAULT;
1124                         }
1125
1126                         /* Clear the dirty bit as children's
1127                          * resources are reallocated.
1128                          */
1129                         parent->child_realloc = false;
1130                 }
1131         }
1132
1133         /* Root is always expected to be there */
1134         if (!root)
1135                 return -EFAULT;
1136
1137         if (root->flags & NIX_TM_NODE_HWRES)
1138                 return 0;
1139
1140         /* Process root node */
1141         bmp = nix->schq_bmp[nix->tm_root_lvl];
1142         plt_bitmap_scan_init(bmp);
1143         if (!plt_bitmap_scan(bmp, &pos, &slab)) {
1144                 plt_err("Resource not allocated for root");
1145                 return -EIO;
1146         }
1147
1148         root->hw_id = pos + bitmap_ctzll(slab);
1149         root->flags |= NIX_TM_NODE_HWRES;
1150         plt_bitmap_clear(bmp, root->hw_id);
1151
1152         /* Get TL1 id as well when root is not TL1 */
1153         if (!nix_tm_have_tl1_access(nix)) {
1154                 bmp = nix->schq_bmp[NIX_TXSCH_LVL_TL1];
1155
1156                 plt_bitmap_scan_init(bmp);
1157                 if (!plt_bitmap_scan(bmp, &pos, &slab)) {
1158                         plt_err("Resource not found for TL1");
1159                         return -EIO;
1160                 }
1161                 root->parent_hw_id = pos + bitmap_ctzll(slab);
1162                 plt_bitmap_clear(bmp, root->parent_hw_id);
1163         }
1164
1165         plt_tm_dbg("Resource %s(%u) for root(id %u) (%p)",
1166                    nix_tm_hwlvl2str(root->hw_lvl), root->hw_id, root->id, root);
1167
1168         return 0;
1169 }
1170
1171 void
1172 nix_tm_copy_rsp_to_nix(struct nix *nix, struct nix_txsch_alloc_rsp *rsp)
1173 {
1174         uint8_t lvl;
1175         uint16_t i;
1176
1177         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1178                 for (i = 0; i < rsp->schq[lvl]; i++)
1179                         plt_bitmap_set(nix->schq_bmp[lvl],
1180                                        rsp->schq_list[lvl][i]);
1181
1182                 for (i = 0; i < rsp->schq_contig[lvl]; i++)
1183                         plt_bitmap_set(nix->schq_contig_bmp[lvl],
1184                                        rsp->schq_contig_list[lvl][i]);
1185         }
1186 }
1187
1188 int
1189 nix_tm_alloc_txschq(struct nix *nix, enum roc_nix_tm_tree tree)
1190 {
1191         uint16_t schq_contig[NIX_TXSCH_LVL_CNT];
1192         struct mbox *mbox = (&nix->dev)->mbox;
1193         uint16_t schq[NIX_TXSCH_LVL_CNT];
1194         struct nix_txsch_alloc_req *req;
1195         struct nix_txsch_alloc_rsp *rsp;
1196         uint8_t hw_lvl, i;
1197         bool pend;
1198         int rc;
1199
1200         memset(schq, 0, sizeof(schq));
1201         memset(schq_contig, 0, sizeof(schq_contig));
1202
1203         /* Estimate requirement */
1204         rc = nix_tm_resource_estimate(nix, schq_contig, schq, tree);
1205         if (!rc)
1206                 return 0;
1207
1208         /* Release existing contiguous resources when realloc requested
1209          * as there is no way to guarantee continuity of old with new.
1210          */
1211         for (hw_lvl = 0; hw_lvl < NIX_TXSCH_LVL_CNT; hw_lvl++) {
1212                 if (schq_contig[hw_lvl])
1213                         nix_tm_release_resources(nix, hw_lvl, true, false);
1214         }
1215
1216         /* Alloc as needed */
1217         do {
1218                 pend = false;
1219                 req = mbox_alloc_msg_nix_txsch_alloc(mbox);
1220                 if (!req) {
1221                         rc = -ENOMEM;
1222                         goto alloc_err;
1223                 }
1224                 mbox_memcpy(req->schq, schq, sizeof(req->schq));
1225                 mbox_memcpy(req->schq_contig, schq_contig,
1226                             sizeof(req->schq_contig));
1227
1228                 /* Each alloc can be at max of MAX_TXSCHQ_PER_FUNC per level.
1229                  * So split alloc to multiple requests.
1230                  */
1231                 for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
1232                         if (req->schq[i] > MAX_TXSCHQ_PER_FUNC)
1233                                 req->schq[i] = MAX_TXSCHQ_PER_FUNC;
1234                         schq[i] -= req->schq[i];
1235
1236                         if (req->schq_contig[i] > MAX_TXSCHQ_PER_FUNC)
1237                                 req->schq_contig[i] = MAX_TXSCHQ_PER_FUNC;
1238                         schq_contig[i] -= req->schq_contig[i];
1239
1240                         if (schq[i] || schq_contig[i])
1241                                 pend = true;
1242                 }
1243
1244                 rc = mbox_process_msg(mbox, (void *)&rsp);
1245                 if (rc)
1246                         goto alloc_err;
1247
1248                 nix_tm_copy_rsp_to_nix(nix, rsp);
1249         } while (pend);
1250
1251         nix->tm_link_cfg_lvl = rsp->link_cfg_lvl;
1252         return 0;
1253 alloc_err:
1254         for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
1255                 if (nix_tm_release_resources(nix, i, true, false))
1256                         plt_err("Failed to release contig resources of "
1257                                 "lvl %d on error",
1258                                 i);
1259                 if (nix_tm_release_resources(nix, i, false, false))
1260                         plt_err("Failed to release discontig resources of "
1261                                 "lvl %d on error",
1262                                 i);
1263         }
1264         return rc;
1265 }
1266
1267 int
1268 nix_tm_prepare_default_tree(struct roc_nix *roc_nix)
1269 {
1270         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1271         uint32_t nonleaf_id = nix->nb_tx_queues;
1272         struct nix_tm_node *node = NULL;
1273         uint8_t leaf_lvl, lvl, lvl_end;
1274         uint32_t parent, i;
1275         int rc = 0;
1276
1277         /* Add ROOT, SCH1, SCH2, SCH3, [SCH4]  nodes */
1278         parent = ROC_NIX_TM_NODE_ID_INVALID;
1279         /* With TL1 access we have an extra level */
1280         lvl_end = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_SCH4 :
1281                                                        ROC_TM_LVL_SCH3);
1282
1283         for (lvl = ROC_TM_LVL_ROOT; lvl <= lvl_end; lvl++) {
1284                 rc = -ENOMEM;
1285                 node = nix_tm_node_alloc();
1286                 if (!node)
1287                         goto error;
1288
1289                 node->id = nonleaf_id;
1290                 node->parent_id = parent;
1291                 node->priority = 0;
1292                 node->weight = NIX_TM_DFLT_RR_WT;
1293                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1294                 node->lvl = lvl;
1295                 node->tree = ROC_NIX_TM_DEFAULT;
1296
1297                 rc = nix_tm_node_add(roc_nix, node);
1298                 if (rc)
1299                         goto error;
1300                 parent = nonleaf_id;
1301                 nonleaf_id++;
1302         }
1303
1304         parent = nonleaf_id - 1;
1305         leaf_lvl = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_QUEUE :
1306                                                         ROC_TM_LVL_SCH4);
1307
1308         /* Add leaf nodes */
1309         for (i = 0; i < nix->nb_tx_queues; i++) {
1310                 rc = -ENOMEM;
1311                 node = nix_tm_node_alloc();
1312                 if (!node)
1313                         goto error;
1314
1315                 node->id = i;
1316                 node->parent_id = parent;
1317                 node->priority = 0;
1318                 node->weight = NIX_TM_DFLT_RR_WT;
1319                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1320                 node->lvl = leaf_lvl;
1321                 node->tree = ROC_NIX_TM_DEFAULT;
1322
1323                 rc = nix_tm_node_add(roc_nix, node);
1324                 if (rc)
1325                         goto error;
1326         }
1327
1328         return 0;
1329 error:
1330         nix_tm_node_free(node);
1331         return rc;
1332 }
1333
1334 int
1335 roc_nix_tm_prepare_rate_limited_tree(struct roc_nix *roc_nix)
1336 {
1337         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1338         uint32_t nonleaf_id = nix->nb_tx_queues;
1339         struct nix_tm_node *node = NULL;
1340         uint8_t leaf_lvl, lvl, lvl_end;
1341         uint32_t parent, i;
1342         int rc = 0;
1343
1344         /* Add ROOT, SCH1, SCH2 nodes */
1345         parent = ROC_NIX_TM_NODE_ID_INVALID;
1346         lvl_end = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_SCH3 :
1347                                                        ROC_TM_LVL_SCH2);
1348
1349         for (lvl = ROC_TM_LVL_ROOT; lvl <= lvl_end; lvl++) {
1350                 rc = -ENOMEM;
1351                 node = nix_tm_node_alloc();
1352                 if (!node)
1353                         goto error;
1354
1355                 node->id = nonleaf_id;
1356                 node->parent_id = parent;
1357                 node->priority = 0;
1358                 node->weight = NIX_TM_DFLT_RR_WT;
1359                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1360                 node->lvl = lvl;
1361                 node->tree = ROC_NIX_TM_RLIMIT;
1362
1363                 rc = nix_tm_node_add(roc_nix, node);
1364                 if (rc)
1365                         goto error;
1366                 parent = nonleaf_id;
1367                 nonleaf_id++;
1368         }
1369
1370         /* SMQ is mapped to SCH4 when we have TL1 access and SCH3 otherwise */
1371         lvl = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_SCH4 : ROC_TM_LVL_SCH3);
1372
1373         /* Add per queue SMQ nodes i.e SCH4 / SCH3 */
1374         for (i = 0; i < nix->nb_tx_queues; i++) {
1375                 rc = -ENOMEM;
1376                 node = nix_tm_node_alloc();
1377                 if (!node)
1378                         goto error;
1379
1380                 node->id = nonleaf_id + i;
1381                 node->parent_id = parent;
1382                 node->priority = 0;
1383                 node->weight = NIX_TM_DFLT_RR_WT;
1384                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1385                 node->lvl = lvl;
1386                 node->tree = ROC_NIX_TM_RLIMIT;
1387
1388                 rc = nix_tm_node_add(roc_nix, node);
1389                 if (rc)
1390                         goto error;
1391         }
1392
1393         parent = nonleaf_id;
1394         leaf_lvl = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_QUEUE :
1395                                                         ROC_TM_LVL_SCH4);
1396
1397         /* Add leaf nodes */
1398         for (i = 0; i < nix->nb_tx_queues; i++) {
1399                 rc = -ENOMEM;
1400                 node = nix_tm_node_alloc();
1401                 if (!node)
1402                         goto error;
1403
1404                 node->id = i;
1405                 node->parent_id = parent + i;
1406                 node->priority = 0;
1407                 node->weight = NIX_TM_DFLT_RR_WT;
1408                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1409                 node->lvl = leaf_lvl;
1410                 node->tree = ROC_NIX_TM_RLIMIT;
1411
1412                 rc = nix_tm_node_add(roc_nix, node);
1413                 if (rc)
1414                         goto error;
1415         }
1416
1417         return 0;
1418 error:
1419         nix_tm_node_free(node);
1420         return rc;
1421 }
1422
1423 int
1424 nix_tm_free_resources(struct roc_nix *roc_nix, uint32_t tree_mask, bool hw_only)
1425 {
1426         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1427         struct nix_tm_shaper_profile *profile;
1428         struct nix_tm_node *node, *next_node;
1429         struct nix_tm_node_list *list;
1430         enum roc_nix_tm_tree tree;
1431         uint32_t profile_id;
1432         int rc = 0;
1433
1434         for (tree = 0; tree < ROC_NIX_TM_TREE_MAX; tree++) {
1435                 if (!(tree_mask & BIT(tree)))
1436                         continue;
1437
1438                 plt_tm_dbg("Freeing resources of tree %u", tree);
1439
1440                 list = nix_tm_node_list(nix, tree);
1441                 next_node = TAILQ_FIRST(list);
1442                 while (next_node) {
1443                         node = next_node;
1444                         next_node = TAILQ_NEXT(node, node);
1445
1446                         if (!nix_tm_is_leaf(nix, node->lvl) &&
1447                             node->flags & NIX_TM_NODE_HWRES) {
1448                                 /* Clear xoff in path for flush to succeed */
1449                                 rc = nix_tm_clear_path_xoff(nix, node);
1450                                 if (rc)
1451                                         return rc;
1452                                 rc = nix_tm_free_node_resource(nix, node);
1453                                 if (rc)
1454                                         return rc;
1455                         }
1456                 }
1457
1458                 /* Leave software elements if needed */
1459                 if (hw_only)
1460                         continue;
1461
1462                 next_node = TAILQ_FIRST(list);
1463                 while (next_node) {
1464                         node = next_node;
1465                         next_node = TAILQ_NEXT(node, node);
1466
1467                         plt_tm_dbg("Free node lvl %u id %u (%p)", node->lvl,
1468                                    node->id, node);
1469
1470                         profile_id = node->shaper_profile_id;
1471                         profile = nix_tm_shaper_profile_search(nix, profile_id);
1472                         if (profile)
1473                                 profile->ref_cnt--;
1474
1475                         TAILQ_REMOVE(list, node, node);
1476                         nix_tm_node_free(node);
1477                 }
1478         }
1479         return rc;
1480 }
1481
1482 int
1483 nix_tm_conf_init(struct roc_nix *roc_nix)
1484 {
1485         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1486         uint32_t bmp_sz, hw_lvl;
1487         void *bmp_mem;
1488         int rc, i;
1489
1490         PLT_STATIC_ASSERT(sizeof(struct nix_tm_node) <= ROC_NIX_TM_NODE_SZ);
1491         PLT_STATIC_ASSERT(sizeof(struct nix_tm_shaper_profile) <=
1492                           ROC_NIX_TM_SHAPER_PROFILE_SZ);
1493
1494         nix->tm_flags = 0;
1495         for (i = 0; i < ROC_NIX_TM_TREE_MAX; i++)
1496                 TAILQ_INIT(&nix->trees[i]);
1497
1498         TAILQ_INIT(&nix->shaper_profile_list);
1499         nix->tm_rate_min = 1E9; /* 1Gbps */
1500
1501         rc = -ENOMEM;
1502         bmp_sz = plt_bitmap_get_memory_footprint(NIX_TM_MAX_HW_TXSCHQ);
1503         bmp_mem = plt_zmalloc(bmp_sz * NIX_TXSCH_LVL_CNT * 2, 0);
1504         if (!bmp_mem)
1505                 return rc;
1506         nix->schq_bmp_mem = bmp_mem;
1507
1508         /* Init contiguous and discontiguous bitmap per lvl */
1509         rc = -EIO;
1510         for (hw_lvl = 0; hw_lvl < NIX_TXSCH_LVL_CNT; hw_lvl++) {
1511                 /* Bitmap for discontiguous resource */
1512                 nix->schq_bmp[hw_lvl] =
1513                         plt_bitmap_init(NIX_TM_MAX_HW_TXSCHQ, bmp_mem, bmp_sz);
1514                 if (!nix->schq_bmp[hw_lvl])
1515                         goto exit;
1516
1517                 bmp_mem = PLT_PTR_ADD(bmp_mem, bmp_sz);
1518
1519                 /* Bitmap for contiguous resource */
1520                 nix->schq_contig_bmp[hw_lvl] =
1521                         plt_bitmap_init(NIX_TM_MAX_HW_TXSCHQ, bmp_mem, bmp_sz);
1522                 if (!nix->schq_contig_bmp[hw_lvl])
1523                         goto exit;
1524
1525                 bmp_mem = PLT_PTR_ADD(bmp_mem, bmp_sz);
1526         }
1527
1528         /* Disable TL1 Static Priority when VF's are enabled
1529          * as otherwise VF's TL2 reallocation will be needed
1530          * runtime to support a specific topology of PF.
1531          */
1532         if (nix->pci_dev->max_vfs)
1533                 nix->tm_flags |= NIX_TM_TL1_NO_SP;
1534
1535         /* TL1 access is only for PF's */
1536         if (roc_nix_is_pf(roc_nix)) {
1537                 nix->tm_flags |= NIX_TM_TL1_ACCESS;
1538                 nix->tm_root_lvl = NIX_TXSCH_LVL_TL1;
1539         } else {
1540                 nix->tm_root_lvl = NIX_TXSCH_LVL_TL2;
1541         }
1542
1543         return 0;
1544 exit:
1545         nix_tm_conf_fini(roc_nix);
1546         return rc;
1547 }
1548
1549 void
1550 nix_tm_conf_fini(struct roc_nix *roc_nix)
1551 {
1552         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1553         uint16_t hw_lvl;
1554
1555         for (hw_lvl = 0; hw_lvl < NIX_TXSCH_LVL_CNT; hw_lvl++) {
1556                 plt_bitmap_free(nix->schq_bmp[hw_lvl]);
1557                 plt_bitmap_free(nix->schq_contig_bmp[hw_lvl]);
1558         }
1559         plt_free(nix->schq_bmp_mem);
1560 }