drivers/common/cnxk/roc_nix_tm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include "roc_api.h"
   6 #include "roc_priv.h"
   7
   8 static inline int
   9 bitmap_ctzll(uint64_t slab)
  10 {
  11         if (slab == 0)
  12                 return 0;
  13
  14         return __builtin_ctzll(slab);
  15 }
  16
  17 void
  18 nix_tm_clear_shaper_profiles(struct nix *nix)
  19 {
  20         struct nix_tm_shaper_profile *shaper_profile, *tmp;
  21         struct nix_tm_shaper_profile_list *list;
  22
  23         list = &nix->shaper_profile_list;
  24         PLT_TAILQ_FOREACH_SAFE(shaper_profile, list, shaper, tmp) {
  25                 if (shaper_profile->ref_cnt)
  26                         plt_warn("Shaper profile %u has non zero references",
  27                                  shaper_profile->id);
  28                 TAILQ_REMOVE(&nix->shaper_profile_list, shaper_profile, shaper);
  29                 nix_tm_shaper_profile_free(shaper_profile);
  30         }
  31 }
  32
  33 static int
  34 nix_tm_node_reg_conf(struct nix *nix, struct nix_tm_node *node)
  35 {
  36         uint64_t regval_mask[MAX_REGS_PER_MBOX_MSG];
  37         uint64_t regval[MAX_REGS_PER_MBOX_MSG];
  38         struct nix_tm_shaper_profile *profile;
  39         uint64_t reg[MAX_REGS_PER_MBOX_MSG];
  40         struct mbox *mbox = (&nix->dev)->mbox;
  41         struct nix_txschq_config *req;
  42         int rc = -EFAULT;
  43         uint32_t hw_lvl;
  44         uint8_t k = 0;
  45
  46         memset(regval, 0, sizeof(regval));
  47         memset(regval_mask, 0, sizeof(regval_mask));
  48
  49         profile = nix_tm_shaper_profile_search(nix, node->shaper_profile_id);
  50         hw_lvl = node->hw_lvl;
  51
  52         /* Need this trigger to configure TL1 */
  53         if (!nix_tm_have_tl1_access(nix) && hw_lvl == NIX_TXSCH_LVL_TL2) {
  54                 /* Prepare default conf for TL1 */
  55                 req = mbox_alloc_msg_nix_txschq_cfg(mbox);
  56                 req->lvl = NIX_TXSCH_LVL_TL1;
  57
  58                 k = nix_tm_tl1_default_prep(node->parent_hw_id, req->reg,
  59                                             req->regval);
  60                 req->num_regs = k;
  61                 rc = mbox_process(mbox);
  62                 if (rc)
  63                         goto error;
  64         }
  65
  66         /* Prepare topology config */
  67         k = nix_tm_topology_reg_prep(nix, node, reg, regval, regval_mask);
  68
  69         /* Prepare schedule config */
  70         k += nix_tm_sched_reg_prep(nix, node, &reg[k], &regval[k]);
  71
  72         /* Prepare shaping config */
  73         k += nix_tm_shaper_reg_prep(node, profile, &reg[k], &regval[k]);
  74
  75         if (!k)
  76                 return 0;
  77
  78         /* Copy and send config mbox */
  79         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
  80         req->lvl = hw_lvl;
  81         req->num_regs = k;
  82
  83         mbox_memcpy(req->reg, reg, sizeof(uint64_t) * k);
  84         mbox_memcpy(req->regval, regval, sizeof(uint64_t) * k);
  85         mbox_memcpy(req->regval_mask, regval_mask, sizeof(uint64_t) * k);
  86
  87         rc = mbox_process(mbox);
  88         if (rc)
  89                 goto error;
  90
  91         return 0;
  92 error:
  93         plt_err("Txschq conf failed for node %p, rc=%d", node, rc);
  94         return rc;
  95 }
  96
  97 int
  98 nix_tm_txsch_reg_config(struct nix *nix, enum roc_nix_tm_tree tree)
  99 {
 100         struct nix_tm_node_list *list;
 101         bool is_pf_or_lbk = false;
 102         struct nix_tm_node *node;
 103         bool skip_bp = false;
 104         uint32_t hw_lvl;
 105         int rc = 0;
 106
 107         list = nix_tm_node_list(nix, tree);
 108
 109         if ((!dev_is_vf(&nix->dev) || nix->lbk_link) && !nix->sdp_link)
 110                 is_pf_or_lbk = true;
 111
 112         for (hw_lvl = 0; hw_lvl <= nix->tm_root_lvl; hw_lvl++) {
 113                 TAILQ_FOREACH(node, list, node) {
 114                         if (node->hw_lvl != hw_lvl)
 115                                 continue;
 116
 117                         /* Only one TL3/TL2 Link config should have BP enable
 118                          * set per channel only for PF or lbk vf.
 119                          */
 120                         node->bp_capa = 0;
 121                         if (is_pf_or_lbk && !skip_bp &&
 122                             node->hw_lvl == nix->tm_link_cfg_lvl) {
 123                                 node->bp_capa = 1;
 124                                 skip_bp = false;
 125                         }
 126
 127                         rc = nix_tm_node_reg_conf(nix, node);
 128                         if (rc)
 129                                 goto exit;
 130                 }
 131         }
 132 exit:
 133         return rc;
 134 }
 135
 136 int
 137 nix_tm_update_parent_info(struct nix *nix, enum roc_nix_tm_tree tree)
 138 {
 139         struct nix_tm_node *child, *parent;
 140         struct nix_tm_node_list *list;
 141         uint32_t rr_prio, max_prio;
 142         uint32_t rr_num = 0;
 143
 144         list = nix_tm_node_list(nix, tree);
 145
 146         /* Release all the node hw resources locally
 147          * if parent marked as dirty and resource exists.
 148          */
 149         TAILQ_FOREACH(child, list, node) {
 150                 /* Release resource only if parent direct hierarchy changed */
 151                 if (child->flags & NIX_TM_NODE_HWRES && child->parent &&
 152                     child->parent->child_realloc) {
 153                         nix_tm_free_node_resource(nix, child);
 154                 }
 155                 child->max_prio = UINT32_MAX;
 156         }
 157
 158         TAILQ_FOREACH(parent, list, node) {
 159                 /* Count group of children of same priority i.e are RR */
 160                 rr_num = nix_tm_check_rr(nix, parent->id, tree, &rr_prio,
 161                                          &max_prio);
 162
 163                 /* Assuming that multiple RR groups are
 164                  * not configured based on capability.
 165                  */
 166                 parent->rr_prio = rr_prio;
 167                 parent->rr_num = rr_num;
 168                 parent->max_prio = max_prio;
 169         }
 170
 171         return 0;
 172 }
 173
 174 static int
 175 nix_tm_root_node_get(struct nix *nix, int tree)
 176 {
 177         struct nix_tm_node_list *list = nix_tm_node_list(nix, tree);
 178         struct nix_tm_node *tm_node;
 179
 180         TAILQ_FOREACH(tm_node, list, node) {
 181                 if (tm_node->hw_lvl == nix->tm_root_lvl)
 182                         return 1;
 183         }
 184
 185         return 0;
 186 }
 187
 188 int
 189 nix_tm_node_add(struct roc_nix *roc_nix, struct nix_tm_node *node)
 190 {
 191         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
 192         struct nix_tm_shaper_profile *profile;
 193         uint32_t node_id, parent_id, lvl;
 194         struct nix_tm_node *parent_node;
 195         uint32_t priority, profile_id;
 196         uint8_t hw_lvl, exp_next_lvl;
 197         enum roc_nix_tm_tree tree;
 198         int rc;
 199
 200         node_id = node->id;
 201         priority = node->priority;
 202         parent_id = node->parent_id;
 203         profile_id = node->shaper_profile_id;
 204         lvl = node->lvl;
 205         tree = node->tree;
 206
 207         plt_tm_dbg("Add node %s lvl %u id %u, prio 0x%x weight 0x%x "
 208                    "parent %u profile 0x%x tree %u",
 209                    nix_tm_hwlvl2str(nix_tm_lvl2nix(nix, lvl)), lvl, node_id,
 210                    priority, node->weight, parent_id, profile_id, tree);
 211
 212         if (tree >= ROC_NIX_TM_TREE_MAX)
 213                 return NIX_ERR_PARAM;
 214
 215         /* Translate sw level id's to nix hw level id's */
 216         hw_lvl = nix_tm_lvl2nix(nix, lvl);
 217         if (hw_lvl == NIX_TXSCH_LVL_CNT && !nix_tm_is_leaf(nix, lvl))
 218                 return NIX_ERR_TM_INVALID_LVL;
 219
 220         /* Leaf nodes have to be same priority */
 221         if (nix_tm_is_leaf(nix, lvl) && priority != 0)
 222                 return NIX_ERR_TM_INVALID_PRIO;
 223
 224         parent_node = nix_tm_node_search(nix, parent_id, tree);
 225
 226         if (node_id < nix->nb_tx_queues)
 227                 exp_next_lvl = NIX_TXSCH_LVL_SMQ;
 228         else
 229                 exp_next_lvl = hw_lvl + 1;
 230
 231         /* Check if there is no parent node yet */
 232         if (hw_lvl != nix->tm_root_lvl &&
 233             (!parent_node || parent_node->hw_lvl != exp_next_lvl))
 234                 return NIX_ERR_TM_INVALID_PARENT;
 235
 236         /* Check if a node already exists */
 237         if (nix_tm_node_search(nix, node_id, tree))
 238                 return NIX_ERR_TM_NODE_EXISTS;
 239
 240         /* Check if root node exists */
 241         if (hw_lvl == nix->tm_root_lvl && nix_tm_root_node_get(nix, tree))
 242                 return NIX_ERR_TM_NODE_EXISTS;
 243
 244         profile = nix_tm_shaper_profile_search(nix, profile_id);
 245         if (!nix_tm_is_leaf(nix, lvl)) {
 246                 /* Check if shaper profile exists for non leaf node */
 247                 if (!profile && profile_id != ROC_NIX_TM_SHAPER_PROFILE_NONE)
 248                         return NIX_ERR_TM_INVALID_SHAPER_PROFILE;
 249
 250                 /* Packet mode in profile should match with that of tm node */
 251                 if (profile && profile->pkt_mode != node->pkt_mode)
 252                         return NIX_ERR_TM_PKT_MODE_MISMATCH;
 253         }
 254
 255         /* Check if there is second DWRR already in siblings or holes in prio */
 256         rc = nix_tm_validate_prio(nix, lvl, parent_id, priority, tree);
 257         if (rc)
 258                 return rc;
 259
 260         if (node->weight > roc_nix_tm_max_sched_wt_get())
 261                 return NIX_ERR_TM_WEIGHT_EXCEED;
 262
 263         /* Maintain minimum weight */
 264         if (!node->weight)
 265                 node->weight = 1;
 266
 267         node->hw_lvl = nix_tm_lvl2nix(nix, lvl);
 268         node->rr_prio = 0xF;
 269         node->max_prio = UINT32_MAX;
 270         node->hw_id = NIX_TM_HW_ID_INVALID;
 271         node->flags = 0;
 272
 273         if (profile)
 274                 profile->ref_cnt++;
 275
 276         node->parent = parent_node;
 277         if (parent_node)
 278                 parent_node->child_realloc = true;
 279         node->parent_hw_id = NIX_TM_HW_ID_INVALID;
 280
 281         TAILQ_INSERT_TAIL(&nix->trees[tree], node, node);
 282         plt_tm_dbg("Added node %s lvl %u id %u (%p)",
 283                    nix_tm_hwlvl2str(node->hw_lvl), lvl, node_id, node);
 284         return 0;
 285 }
 286
 287 int
 288 nix_tm_clear_path_xoff(struct nix *nix, struct nix_tm_node *node)
 289 {
 290         struct mbox *mbox = (&nix->dev)->mbox;
 291         struct nix_txschq_config *req;
 292         struct nix_tm_node *p;
 293         int rc;
 294
 295         /* Enable nodes in path for flush to succeed */
 296         if (!nix_tm_is_leaf(nix, node->lvl))
 297                 p = node;
 298         else
 299                 p = node->parent;
 300         while (p) {
 301                 if (!(p->flags & NIX_TM_NODE_ENABLED) &&
 302                     (p->flags & NIX_TM_NODE_HWRES)) {
 303                         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
 304                         req->lvl = p->hw_lvl;
 305                         req->num_regs = nix_tm_sw_xoff_prep(p, false, req->reg,
 306                                                             req->regval);
 307                         rc = mbox_process(mbox);
 308                         if (rc)
 309                                 return rc;
 310
 311                         p->flags |= NIX_TM_NODE_ENABLED;
 312                 }
 313                 p = p->parent;
 314         }
 315
 316         return 0;
 317 }
 318
 319 int
 320 nix_tm_bp_config_set(struct roc_nix *roc_nix, uint16_t sq, uint16_t tc,
 321                      bool enable)
 322 {
 323         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
 324         enum roc_nix_tm_tree tree = nix->tm_tree;
 325         struct mbox *mbox = (&nix->dev)->mbox;
 326         struct nix_txschq_config *req = NULL;
 327         struct nix_tm_node_list *list;
 328         uint16_t link = nix->tx_link;
 329         struct nix_tm_node *sq_node;
 330         struct nix_tm_node *parent;
 331         struct nix_tm_node *node;
 332         uint8_t k = 0;
 333         int rc = 0;
 334
 335         sq_node = nix_tm_node_search(nix, sq, nix->tm_tree);
 336         if (!sq_node)
 337                 return -ENOENT;
 338
 339         parent = sq_node->parent;
 340         while (parent) {
 341                 if (parent->lvl == ROC_TM_LVL_SCH2)
 342                         break;
 343
 344                 parent = parent->parent;
 345         }
 346         if (!parent)
 347                 return -ENOENT;
 348
 349         list = nix_tm_node_list(nix, tree);
 350
 351         if (parent->rel_chan != NIX_TM_CHAN_INVALID && parent->rel_chan != tc) {
 352                 rc = -EINVAL;
 353                 goto err;
 354         }
 355
 356         TAILQ_FOREACH(node, list, node) {
 357                 if (node->hw_lvl != nix->tm_link_cfg_lvl)
 358                         continue;
 359
 360                 if (!(node->flags & NIX_TM_NODE_HWRES) || !node->bp_capa)
 361                         continue;
 362
 363                 if (node->hw_id != parent->hw_id)
 364                         continue;
 365
 366                 if (!req) {
 367                         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
 368                         req->lvl = nix->tm_link_cfg_lvl;
 369                         k = 0;
 370                 }
 371
 372                 req->reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(node->hw_id, link);
 373                 req->regval[k] = enable ? tc : 0;
 374                 req->regval[k] |= enable ? BIT_ULL(13) : 0;
 375                 req->regval_mask[k] = ~(BIT_ULL(13) | GENMASK_ULL(7, 0));
 376                 k++;
 377
 378                 if (k >= MAX_REGS_PER_MBOX_MSG) {
 379                         req->num_regs = k;
 380                         rc = mbox_process(mbox);
 381                         if (rc)
 382                                 goto err;
 383                         req = NULL;
 384                 }
 385         }
 386
 387         if (req) {
 388                 req->num_regs = k;
 389                 rc = mbox_process(mbox);
 390                 if (rc)
 391                         goto err;
 392         }
 393
 394         parent->rel_chan = enable ? tc : NIX_TM_CHAN_INVALID;
 395         return 0;
 396 err:
 397         plt_err("Failed to %s bp on link %u, rc=%d(%s)",
 398                 enable ? "enable" : "disable", link, rc, roc_error_msg_get(rc));
 399         return rc;
 400 }
 401
 402 int
 403 nix_tm_bp_config_get(struct roc_nix *roc_nix, bool *is_enabled)
 404 {
 405         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
 406         struct nix_txschq_config *req = NULL, *rsp;
 407         enum roc_nix_tm_tree tree = nix->tm_tree;
 408         struct mbox *mbox = (&nix->dev)->mbox;
 409         struct nix_tm_node_list *list;
 410         struct nix_tm_node *node;
 411         bool found = false;
 412         uint8_t enable = 1;
 413         uint8_t k = 0, i;
 414         uint16_t link;
 415         int rc = 0;
 416
 417         list = nix_tm_node_list(nix, tree);
 418         link = nix->tx_link;
 419
 420         TAILQ_FOREACH(node, list, node) {
 421                 if (node->hw_lvl != nix->tm_link_cfg_lvl)
 422                         continue;
 423
 424                 if (!(node->flags & NIX_TM_NODE_HWRES) || !node->bp_capa)
 425                         continue;
 426
 427                 found = true;
 428                 if (!req) {
 429                         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
 430                         req->read = 1;
 431                         req->lvl = nix->tm_link_cfg_lvl;
 432                         k = 0;
 433                 }
 434
 435                 req->reg[k] = NIX_AF_TL3_TL2X_LINKX_CFG(node->hw_id, link);
 436                 k++;
 437
 438                 if (k >= MAX_REGS_PER_MBOX_MSG) {
 439                         req->num_regs = k;
 440                         rc = mbox_process_msg(mbox, (void **)&rsp);
 441                         if (rc || rsp->num_regs != k)
 442                                 goto err;
 443                         req = NULL;
 444
 445                         /* Report it as enabled only if enabled or all */
 446                         for (i = 0; i < k; i++)
 447                                 enable &= !!(rsp->regval[i] & BIT_ULL(13));
 448                 }
 449         }
 450
 451         if (req) {
 452                 req->num_regs = k;
 453                 rc = mbox_process_msg(mbox, (void **)&rsp);
 454                 if (rc)
 455                         goto err;
 456                 /* Report it as enabled only if enabled or all */
 457                 for (i = 0; i < k; i++)
 458                         enable &= !!(rsp->regval[i] & BIT_ULL(13));
 459         }
 460
 461         *is_enabled = found ? !!enable : false;
 462         return 0;
 463 err:
 464         plt_err("Failed to get bp status on link %u, rc=%d(%s)", link, rc,
 465                 roc_error_msg_get(rc));
 466         return rc;
 467 }
 468
 469 int
 470 nix_tm_smq_xoff(struct nix *nix, struct nix_tm_node *node, bool enable)
 471 {
 472         struct mbox *mbox = (&nix->dev)->mbox;
 473         struct nix_txschq_config *req;
 474         uint16_t smq;
 475         int rc;
 476
 477         smq = node->hw_id;
 478         plt_tm_dbg("Setting SMQ %u XOFF/FLUSH to %s", smq,
 479                    enable ? "enable" : "disable");
 480
 481         rc = nix_tm_clear_path_xoff(nix, node);
 482         if (rc)
 483                 return rc;
 484
 485         req = mbox_alloc_msg_nix_txschq_cfg(mbox);
 486         req->lvl = NIX_TXSCH_LVL_SMQ;
 487         req->num_regs = 1;
 488
 489         req->reg[0] = NIX_AF_SMQX_CFG(smq);
 490         req->regval[0] = enable ? (BIT_ULL(50) | BIT_ULL(49)) : 0;
 491         req->regval_mask[0] =
 492                 enable ? ~(BIT_ULL(50) | BIT_ULL(49)) : ~BIT_ULL(50);
 493
 494         return mbox_process(mbox);
 495 }
 496
 497 int
 498 nix_tm_leaf_data_get(struct nix *nix, uint16_t sq, uint32_t *rr_quantum,
 499                      uint16_t *smq)
 500 {
 501         struct nix_tm_node *node;
 502         int rc;
 503
 504         node = nix_tm_node_search(nix, sq, nix->tm_tree);
 505
 506         /* Check if we found a valid leaf node */
 507         if (!node || !nix_tm_is_leaf(nix, node->lvl) || !node->parent ||
 508             node->parent->hw_id == NIX_TM_HW_ID_INVALID) {
 509                 return -EIO;
 510         }
 511
 512         /* Get SMQ Id of leaf node's parent */
 513         *smq = node->parent->hw_id;
 514         *rr_quantum = nix_tm_weight_to_rr_quantum(node->weight);
 515
 516         rc = nix_tm_smq_xoff(nix, node->parent, false);
 517         if (rc)
 518                 return rc;
 519         node->flags |= NIX_TM_NODE_ENABLED;
 520         return 0;
 521 }
 522
 523 int
 524 roc_nix_tm_sq_flush_spin(struct roc_nix_sq *sq)
 525 {
 526         struct nix *nix = roc_nix_to_nix_priv(sq->roc_nix);
 527         uint16_t sqb_cnt, head_off, tail_off;
 528         uint64_t wdata, val, prev;
 529         uint16_t qid = sq->qid;
 530         int64_t *regaddr;
 531         uint64_t timeout; /* 10's of usec */
 532
 533         /* Wait for enough time based on shaper min rate */
 534         timeout = (sq->nb_desc * roc_nix_max_pkt_len(sq->roc_nix) * 8 * 1E5);
 535         /* Wait for worst case scenario of this SQ being last priority
 536          * and so have to wait for all other SQ's drain out by their own.
 537          */
 538         timeout = timeout * nix->nb_tx_queues;
 539         timeout = timeout / nix->tm_rate_min;
 540         if (!timeout)
 541                 timeout = 10000;
 542
 543         wdata = ((uint64_t)qid << 32);
 544         regaddr = (int64_t *)(nix->base + NIX_LF_SQ_OP_STATUS);
 545         val = roc_atomic64_add_nosync(wdata, regaddr);
 546
 547         /* Spin multiple iterations as "sq->fc_cache_pkts" can still
 548          * have space to send pkts even though fc_mem is disabled
 549          */
 550
 551         while (true) {
 552                 prev = val;
 553                 plt_delay_us(10);
 554                 val = roc_atomic64_add_nosync(wdata, regaddr);
 555                 /* Continue on error */
 556                 if (val & BIT_ULL(63))
 557                         continue;
 558
 559                 if (prev != val)
 560                         continue;
 561
 562                 sqb_cnt = val & 0xFFFF;
 563                 head_off = (val >> 20) & 0x3F;
 564                 tail_off = (val >> 28) & 0x3F;
 565
 566                 /* SQ reached quiescent state */
 567                 if (sqb_cnt <= 1 && head_off == tail_off &&
 568                     (*(volatile uint64_t *)sq->fc == sq->nb_sqb_bufs)) {
 569                         break;
 570                 }
 571
 572                 /* Timeout */
 573                 if (!timeout)
 574                         goto exit;
 575                 timeout--;
 576         }
 577
 578         return 0;
 579 exit:
 580         roc_nix_tm_dump(sq->roc_nix);
 581         roc_nix_queues_ctx_dump(sq->roc_nix);
 582         return -EFAULT;
 583 }
 584
 585 /* Flush and disable tx queue and its parent SMQ */
 586 int
 587 nix_tm_sq_flush_pre(struct roc_nix_sq *sq)
 588 {
 589         struct roc_nix *roc_nix = sq->roc_nix;
 590         struct nix_tm_node *node, *sibling;
 591         struct nix_tm_node_list *list;
 592         enum roc_nix_tm_tree tree;
 593         struct mbox *mbox;
 594         struct nix *nix;
 595         uint16_t qid;
 596         int rc;
 597
 598         nix = roc_nix_to_nix_priv(roc_nix);
 599
 600         /* Need not do anything if tree is in disabled state */
 601         if (!(nix->tm_flags & NIX_TM_HIERARCHY_ENA))
 602                 return 0;
 603
 604         mbox = (&nix->dev)->mbox;
 605         qid = sq->qid;
 606
 607         tree = nix->tm_tree;
 608         list = nix_tm_node_list(nix, tree);
 609
 610         /* Find the node for this SQ */
 611         node = nix_tm_node_search(nix, qid, tree);
 612         if (!node || !(node->flags & NIX_TM_NODE_ENABLED)) {
 613                 plt_err("Invalid node/state for sq %u", qid);
 614                 return -EFAULT;
 615         }
 616
 617         /* Enable CGX RXTX to drain pkts */
 618         if (!roc_nix->io_enabled) {
 619                 /* Though it enables both RX MCAM Entries and CGX Link
 620                  * we assume all the rx queues are stopped way back.
 621                  */
 622                 mbox_alloc_msg_nix_lf_start_rx(mbox);
 623                 rc = mbox_process(mbox);
 624                 if (rc) {
 625                         plt_err("cgx start failed, rc=%d", rc);
 626                         return rc;
 627                 }
 628         }
 629
 630         /* Disable backpressure */
 631         rc = nix_tm_bp_config_set(roc_nix, sq->qid, 0, false);
 632         if (rc) {
 633                 plt_err("Failed to disable backpressure for flush, rc=%d", rc);
 634                 return rc;
 635         }
 636
 637         /* Disable smq xoff for case it was enabled earlier */
 638         rc = nix_tm_smq_xoff(nix, node->parent, false);
 639         if (rc) {
 640                 plt_err("Failed to enable smq %u, rc=%d", node->parent->hw_id,
 641                         rc);
 642                 return rc;
 643         }
 644
 645         /* As per HRM, to disable an SQ, all other SQ's
 646          * that feed to same SMQ must be paused before SMQ flush.
 647          */
 648         TAILQ_FOREACH(sibling, list, node) {
 649                 if (sibling->parent != node->parent)
 650                         continue;
 651                 if (!(sibling->flags & NIX_TM_NODE_ENABLED))
 652                         continue;
 653
 654                 qid = sibling->id;
 655                 sq = nix->sqs[qid];
 656                 if (!sq)
 657                         continue;
 658
 659                 rc = roc_nix_tm_sq_aura_fc(sq, false);
 660                 if (rc) {
 661                         plt_err("Failed to disable sqb aura fc, rc=%d", rc);
 662                         goto cleanup;
 663                 }
 664
 665                 /* Wait for sq entries to be flushed */
 666                 rc = roc_nix_tm_sq_flush_spin(sq);
 667                 if (rc) {
 668                         plt_err("Failed to drain sq %u, rc=%d\n", sq->qid, rc);
 669                         return rc;
 670                 }
 671         }
 672
 673         node->flags &= ~NIX_TM_NODE_ENABLED;
 674
 675         /* Disable and flush */
 676         rc = nix_tm_smq_xoff(nix, node->parent, true);
 677         if (rc) {
 678                 plt_err("Failed to disable smq %u, rc=%d", node->parent->hw_id,
 679                         rc);
 680                 goto cleanup;
 681         }
 682 cleanup:
 683         /* Restore cgx state */
 684         if (!roc_nix->io_enabled) {
 685                 mbox_alloc_msg_nix_lf_stop_rx(mbox);
 686                 rc |= mbox_process(mbox);
 687         }
 688
 689         return rc;
 690 }
 691
 692 int
 693 nix_tm_sq_flush_post(struct roc_nix_sq *sq)
 694 {
 695         struct roc_nix *roc_nix = sq->roc_nix;
 696         struct nix_tm_node *node, *sibling;
 697         struct nix_tm_node_list *list;
 698         enum roc_nix_tm_tree tree;
 699         struct roc_nix_sq *s_sq;
 700         bool once = false;
 701         uint16_t qid, s_qid;
 702         struct nix *nix;
 703         int rc;
 704
 705         nix = roc_nix_to_nix_priv(roc_nix);
 706
 707         /* Need not do anything if tree is in disabled state */
 708         if (!(nix->tm_flags & NIX_TM_HIERARCHY_ENA))
 709                 return 0;
 710
 711         qid = sq->qid;
 712         tree = nix->tm_tree;
 713         list = nix_tm_node_list(nix, tree);
 714
 715         /* Find the node for this SQ */
 716         node = nix_tm_node_search(nix, qid, tree);
 717         if (!node) {
 718                 plt_err("Invalid node for sq %u", qid);
 719                 return -EFAULT;
 720         }
 721
 722         /* Enable all the siblings back */
 723         TAILQ_FOREACH(sibling, list, node) {
 724                 if (sibling->parent != node->parent)
 725                         continue;
 726
 727                 if (sibling->id == qid)
 728                         continue;
 729
 730                 if (!(sibling->flags & NIX_TM_NODE_ENABLED))
 731                         continue;
 732
 733                 s_qid = sibling->id;
 734                 s_sq = nix->sqs[s_qid];
 735                 if (!s_sq)
 736                         continue;
 737
 738                 if (!once) {
 739                         /* Enable back if any SQ is still present */
 740                         rc = nix_tm_smq_xoff(nix, node->parent, false);
 741                         if (rc) {
 742                                 plt_err("Failed to enable smq %u, rc=%d",
 743                                         node->parent->hw_id, rc);
 744                                 return rc;
 745                         }
 746                         once = true;
 747                 }
 748
 749                 rc = roc_nix_tm_sq_aura_fc(s_sq, true);
 750                 if (rc) {
 751                         plt_err("Failed to enable sqb aura fc, rc=%d", rc);
 752                         return rc;
 753                 }
 754         }
 755
 756         if (!nix->rx_pause)
 757                 return 0;
 758
 759         /* Restore backpressure */
 760         rc = nix_tm_bp_config_set(roc_nix, sq->qid, 0, true);
 761         if (rc) {
 762                 plt_err("Failed to restore backpressure, rc=%d", rc);
 763                 return rc;
 764         }
 765
 766         return 0;
 767 }
 768
 769 int
 770 nix_tm_sq_sched_conf(struct nix *nix, struct nix_tm_node *node,
 771                      bool rr_quantum_only)
 772 {
 773         struct mbox *mbox = (&nix->dev)->mbox;
 774         uint16_t qid = node->id, smq;
 775         uint64_t rr_quantum;
 776         int rc;
 777
 778         smq = node->parent->hw_id;
 779         rr_quantum = nix_tm_weight_to_rr_quantum(node->weight);
 780
 781         if (rr_quantum_only)
 782                 plt_tm_dbg("Update sq(%u) rr_quantum 0x%" PRIx64, qid,
 783                            rr_quantum);
 784         else
 785                 plt_tm_dbg("Enabling sq(%u)->smq(%u), rr_quantum 0x%" PRIx64,
 786                            qid, smq, rr_quantum);
 787
 788         if (qid > nix->nb_tx_queues)
 789                 return -EFAULT;
 790
 791         if (roc_model_is_cn9k()) {
 792                 struct nix_aq_enq_req *aq;
 793
 794                 aq = mbox_alloc_msg_nix_aq_enq(mbox);
 795                 if (!aq)
 796                         return -ENOSPC;
 797
 798                 aq->qidx = qid;
 799                 aq->ctype = NIX_AQ_CTYPE_SQ;
 800                 aq->op = NIX_AQ_INSTOP_WRITE;
 801
 802                 /* smq update only when needed */
 803                 if (!rr_quantum_only) {
 804                         aq->sq.smq = smq;
 805                         aq->sq_mask.smq = ~aq->sq_mask.smq;
 806                 }
 807                 aq->sq.smq_rr_quantum = rr_quantum;
 808                 aq->sq_mask.smq_rr_quantum = ~aq->sq_mask.smq_rr_quantum;
 809         } else {
 810                 struct nix_cn10k_aq_enq_req *aq;
 811
 812                 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox);
 813                 if (!aq)
 814                         return -ENOSPC;
 815
 816                 aq->qidx = qid;
 817                 aq->ctype = NIX_AQ_CTYPE_SQ;
 818                 aq->op = NIX_AQ_INSTOP_WRITE;
 819
 820                 /* smq update only when needed */
 821                 if (!rr_quantum_only) {
 822                         aq->sq.smq = smq;
 823                         aq->sq_mask.smq = ~aq->sq_mask.smq;
 824                 }
 825                 aq->sq.smq_rr_weight = rr_quantum;
 826                 aq->sq_mask.smq_rr_weight = ~aq->sq_mask.smq_rr_weight;
 827         }
 828
 829         rc = mbox_process(mbox);
 830         if (rc)
 831                 plt_err("Failed to set smq, rc=%d", rc);
 832         return rc;
 833 }
 834
 835 int
 836 nix_tm_release_resources(struct nix *nix, uint8_t hw_lvl, bool contig,
 837                          bool above_thresh)
 838 {
 839         uint16_t avail, thresh, to_free = 0, schq;
 840         struct mbox *mbox = (&nix->dev)->mbox;
 841         struct nix_txsch_free_req *req;
 842         struct plt_bitmap *bmp;
 843         uint64_t slab = 0;
 844         uint32_t pos = 0;
 845         int rc = -ENOSPC;
 846
 847         bmp = contig ? nix->schq_contig_bmp[hw_lvl] : nix->schq_bmp[hw_lvl];
 848         thresh =
 849                 contig ? nix->contig_rsvd[hw_lvl] : nix->discontig_rsvd[hw_lvl];
 850         plt_bitmap_scan_init(bmp);
 851
 852         avail = nix_tm_resource_avail(nix, hw_lvl, contig);
 853
 854         if (above_thresh) {
 855                 /* Release only above threshold */
 856                 if (avail > thresh)
 857                         to_free = avail - thresh;
 858         } else {
 859                 /* Release everything */
 860                 to_free = avail;
 861         }
 862
 863         /* Now release resources to AF */
 864         while (to_free) {
 865                 if (!slab && !plt_bitmap_scan(bmp, &pos, &slab))
 866                         break;
 867
 868                 schq = bitmap_ctzll(slab);
 869                 slab &= ~(1ULL << schq);
 870                 schq += pos;
 871
 872                 /* Free to AF */
 873                 req = mbox_alloc_msg_nix_txsch_free(mbox);
 874                 if (req == NULL)
 875                         return rc;
 876                 req->flags = 0;
 877                 req->schq_lvl = hw_lvl;
 878                 req->schq = schq;
 879                 rc = mbox_process(mbox);
 880                 if (rc) {
 881                         plt_err("failed to release hwres %s(%u) rc %d",
 882                                 nix_tm_hwlvl2str(hw_lvl), schq, rc);
 883                         return rc;
 884                 }
 885
 886                 plt_tm_dbg("Released hwres %s(%u)", nix_tm_hwlvl2str(hw_lvl),
 887                            schq);
 888                 plt_bitmap_clear(bmp, schq);
 889                 to_free--;
 890         }
 891
 892         if (to_free) {
 893                 plt_err("resource inconsistency for %s(%u)",
 894                         nix_tm_hwlvl2str(hw_lvl), contig);
 895                 return -EFAULT;
 896         }
 897         return 0;
 898 }
 899
 900 int
 901 nix_tm_free_node_resource(struct nix *nix, struct nix_tm_node *node)
 902 {
 903         struct mbox *mbox = (&nix->dev)->mbox;
 904         struct nix_txsch_free_req *req;
 905         struct plt_bitmap *bmp;
 906         uint16_t avail, hw_id;
 907         uint8_t hw_lvl;
 908         int rc = -ENOSPC;
 909
 910         hw_lvl = node->hw_lvl;
 911         hw_id = node->hw_id;
 912         bmp = nix->schq_bmp[hw_lvl];
 913         /* Free specific HW resource */
 914         plt_tm_dbg("Free hwres %s(%u) lvl %u id %u (%p)",
 915                    nix_tm_hwlvl2str(node->hw_lvl), hw_id, node->lvl, node->id,
 916                    node);
 917
 918         avail = nix_tm_resource_avail(nix, hw_lvl, false);
 919         /* Always for now free to discontiguous queue when avail
 920          * is not sufficient.
 921          */
 922         if (nix->discontig_rsvd[hw_lvl] &&
 923             avail < nix->discontig_rsvd[hw_lvl]) {
 924                 PLT_ASSERT(hw_id < NIX_TM_MAX_HW_TXSCHQ);
 925                 PLT_ASSERT(plt_bitmap_get(bmp, hw_id) == 0);
 926                 plt_bitmap_set(bmp, hw_id);
 927                 node->hw_id = NIX_TM_HW_ID_INVALID;
 928                 node->flags &= ~NIX_TM_NODE_HWRES;
 929                 return 0;
 930         }
 931
 932         /* Free to AF */
 933         req = mbox_alloc_msg_nix_txsch_free(mbox);
 934         if (req == NULL)
 935                 return rc;
 936         req->flags = 0;
 937         req->schq_lvl = node->hw_lvl;
 938         req->schq = hw_id;
 939         rc = mbox_process(mbox);
 940         if (rc) {
 941                 plt_err("failed to release hwres %s(%u) rc %d",
 942                         nix_tm_hwlvl2str(node->hw_lvl), hw_id, rc);
 943                 return rc;
 944         }
 945
 946         /* Mark parent as dirty for reallocing it's children */
 947         if (node->parent)
 948                 node->parent->child_realloc = true;
 949
 950         node->hw_id = NIX_TM_HW_ID_INVALID;
 951         node->flags &= ~NIX_TM_NODE_HWRES;
 952         plt_tm_dbg("Released hwres %s(%u) to af",
 953                    nix_tm_hwlvl2str(node->hw_lvl), hw_id);
 954         return 0;
 955 }
 956
 957 int
 958 nix_tm_node_delete(struct roc_nix *roc_nix, uint32_t node_id,
 959                    enum roc_nix_tm_tree tree, bool free)
 960 {
 961         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
 962         struct nix_tm_shaper_profile *profile;
 963         struct nix_tm_node *node, *child;
 964         struct nix_tm_node_list *list;
 965         uint32_t profile_id;
 966         int rc;
 967
 968         plt_tm_dbg("Delete node id %u tree %u", node_id, tree);
 969
 970         node = nix_tm_node_search(nix, node_id, tree);
 971         if (!node)
 972                 return NIX_ERR_TM_INVALID_NODE;
 973
 974         list = nix_tm_node_list(nix, tree);
 975         /* Check for any existing children */
 976         TAILQ_FOREACH(child, list, node) {
 977                 if (child->parent == node)
 978                         return NIX_ERR_TM_CHILD_EXISTS;
 979         }
 980
 981         /* Remove shaper profile reference */
 982         profile_id = node->shaper_profile_id;
 983         profile = nix_tm_shaper_profile_search(nix, profile_id);
 984
 985         /* Free hw resource locally */
 986         if (node->flags & NIX_TM_NODE_HWRES) {
 987                 rc = nix_tm_free_node_resource(nix, node);
 988                 if (rc)
 989                         return rc;
 990         }
 991
 992         if (profile)
 993                 profile->ref_cnt--;
 994
 995         TAILQ_REMOVE(list, node, node);
 996
 997         plt_tm_dbg("Deleted node %s lvl %u id %u, prio 0x%x weight 0x%x "
 998                    "parent %u profile 0x%x tree %u (%p)",
 999                    nix_tm_hwlvl2str(node->hw_lvl), node->lvl, node->id,
1000                    node->priority, node->weight,
1001                    node->parent ? node->parent->id : UINT32_MAX,
1002                    node->shaper_profile_id, tree, node);
1003         /* Free only if requested */
1004         if (free)
1005                 nix_tm_node_free(node);
1006         return 0;
1007 }
1008
1009 static int
1010 nix_tm_assign_hw_id(struct nix *nix, struct nix_tm_node *parent,
1011                     uint16_t *contig_id, int *contig_cnt,
1012                     struct nix_tm_node_list *list)
1013 {
1014         struct nix_tm_node *child;
1015         struct plt_bitmap *bmp;
1016         uint8_t child_hw_lvl;
1017         int spare_schq = -1;
1018         uint32_t pos = 0;
1019         uint64_t slab;
1020         uint16_t schq;
1021
1022         child_hw_lvl = parent->hw_lvl - 1;
1023         bmp = nix->schq_bmp[child_hw_lvl];
1024         plt_bitmap_scan_init(bmp);
1025         slab = 0;
1026
1027         /* Save spare schq if it is case of RR + SP */
1028         if (parent->rr_prio != 0xf && *contig_cnt > 1)
1029                 spare_schq = *contig_id + parent->rr_prio;
1030
1031         TAILQ_FOREACH(child, list, node) {
1032                 if (!child->parent)
1033                         continue;
1034                 if (child->parent->id != parent->id)
1035                         continue;
1036
1037                 /* Resource never expected to be present */
1038                 if (child->flags & NIX_TM_NODE_HWRES) {
1039                         plt_err("Resource exists for child (%s)%u, id %u (%p)",
1040                                 nix_tm_hwlvl2str(child->hw_lvl), child->hw_id,
1041                                 child->id, child);
1042                         return -EFAULT;
1043                 }
1044
1045                 if (!slab)
1046                         plt_bitmap_scan(bmp, &pos, &slab);
1047
1048                 if (child->priority == parent->rr_prio && spare_schq != -1) {
1049                         /* Use spare schq first if present */
1050                         schq = spare_schq;
1051                         spare_schq = -1;
1052                         *contig_cnt = *contig_cnt - 1;
1053
1054                 } else if (child->priority == parent->rr_prio) {
1055                         /* Assign a discontiguous queue */
1056                         if (!slab) {
1057                                 plt_err("Schq not found for Child %u "
1058                                         "lvl %u (%p)",
1059                                         child->id, child->lvl, child);
1060                                 return -ENOENT;
1061                         }
1062
1063                         schq = bitmap_ctzll(slab);
1064                         slab &= ~(1ULL << schq);
1065                         schq += pos;
1066                         plt_bitmap_clear(bmp, schq);
1067                 } else {
1068                         /* Assign a contiguous queue */
1069                         schq = *contig_id + child->priority;
1070                         *contig_cnt = *contig_cnt - 1;
1071                 }
1072
1073                 plt_tm_dbg("Resource %s(%u), for lvl %u id %u(%p)",
1074                            nix_tm_hwlvl2str(child->hw_lvl), schq, child->lvl,
1075                            child->id, child);
1076
1077                 child->hw_id = schq;
1078                 child->parent_hw_id = parent->hw_id;
1079                 child->flags |= NIX_TM_NODE_HWRES;
1080         }
1081
1082         return 0;
1083 }
1084
1085 int
1086 nix_tm_assign_resources(struct nix *nix, enum roc_nix_tm_tree tree)
1087 {
1088         struct nix_tm_node *parent, *root = NULL;
1089         struct plt_bitmap *bmp, *bmp_contig;
1090         struct nix_tm_node_list *list;
1091         uint8_t child_hw_lvl, hw_lvl;
1092         uint16_t contig_id, j;
1093         uint64_t slab = 0;
1094         uint32_t pos = 0;
1095         int cnt, rc;
1096
1097         list = nix_tm_node_list(nix, tree);
1098         /* Walk from TL1 to TL4 parents */
1099         for (hw_lvl = NIX_TXSCH_LVL_TL1; hw_lvl > 0; hw_lvl--) {
1100                 TAILQ_FOREACH(parent, list, node) {
1101                         child_hw_lvl = parent->hw_lvl - 1;
1102                         if (parent->hw_lvl != hw_lvl)
1103                                 continue;
1104
1105                         /* Remember root for future */
1106                         if (parent->hw_lvl == nix->tm_root_lvl)
1107                                 root = parent;
1108
1109                         if (!parent->child_realloc) {
1110                                 /* Skip when parent is not dirty */
1111                                 if (nix_tm_child_res_valid(list, parent))
1112                                         continue;
1113                                 plt_err("Parent not dirty but invalid "
1114                                         "child res parent id %u(lvl %u)",
1115                                         parent->id, parent->lvl);
1116                                 return -EFAULT;
1117                         }
1118
1119                         bmp_contig = nix->schq_contig_bmp[child_hw_lvl];
1120
1121                         /* Prealloc contiguous indices for a parent */
1122                         contig_id = NIX_TM_MAX_HW_TXSCHQ;
1123                         cnt = (int)parent->max_prio + 1;
1124                         if (cnt > 0) {
1125                                 plt_bitmap_scan_init(bmp_contig);
1126                                 if (!plt_bitmap_scan(bmp_contig, &pos, &slab)) {
1127                                         plt_err("Contig schq not found");
1128                                         return -ENOENT;
1129                                 }
1130                                 contig_id = pos + bitmap_ctzll(slab);
1131
1132                                 /* Check if we have enough */
1133                                 for (j = contig_id; j < contig_id + cnt; j++) {
1134                                         if (!plt_bitmap_get(bmp_contig, j))
1135                                                 break;
1136                                 }
1137
1138                                 if (j != contig_id + cnt) {
1139                                         plt_err("Contig schq not sufficient");
1140                                         return -ENOENT;
1141                                 }
1142
1143                                 for (j = contig_id; j < contig_id + cnt; j++)
1144                                         plt_bitmap_clear(bmp_contig, j);
1145                         }
1146
1147                         /* Assign hw id to all children */
1148                         rc = nix_tm_assign_hw_id(nix, parent, &contig_id, &cnt,
1149                                                  list);
1150                         if (cnt || rc) {
1151                                 plt_err("Unexpected err, contig res alloc, "
1152                                         "parent %u, of %s, rc=%d, cnt=%d",
1153                                         parent->id, nix_tm_hwlvl2str(hw_lvl),
1154                                         rc, cnt);
1155                                 return -EFAULT;
1156                         }
1157
1158                         /* Clear the dirty bit as children's
1159                          * resources are reallocated.
1160                          */
1161                         parent->child_realloc = false;
1162                 }
1163         }
1164
1165         /* Root is always expected to be there */
1166         if (!root)
1167                 return -EFAULT;
1168
1169         if (root->flags & NIX_TM_NODE_HWRES)
1170                 return 0;
1171
1172         /* Process root node */
1173         bmp = nix->schq_bmp[nix->tm_root_lvl];
1174         plt_bitmap_scan_init(bmp);
1175         if (!plt_bitmap_scan(bmp, &pos, &slab)) {
1176                 plt_err("Resource not allocated for root");
1177                 return -EIO;
1178         }
1179
1180         root->hw_id = pos + bitmap_ctzll(slab);
1181         root->flags |= NIX_TM_NODE_HWRES;
1182         plt_bitmap_clear(bmp, root->hw_id);
1183
1184         /* Get TL1 id as well when root is not TL1 */
1185         if (!nix_tm_have_tl1_access(nix)) {
1186                 bmp = nix->schq_bmp[NIX_TXSCH_LVL_TL1];
1187
1188                 plt_bitmap_scan_init(bmp);
1189                 if (!plt_bitmap_scan(bmp, &pos, &slab)) {
1190                         plt_err("Resource not found for TL1");
1191                         return -EIO;
1192                 }
1193                 root->parent_hw_id = pos + bitmap_ctzll(slab);
1194                 plt_bitmap_clear(bmp, root->parent_hw_id);
1195         }
1196
1197         plt_tm_dbg("Resource %s(%u) for root(id %u) (%p)",
1198                    nix_tm_hwlvl2str(root->hw_lvl), root->hw_id, root->id, root);
1199
1200         return 0;
1201 }
1202
1203 void
1204 nix_tm_copy_rsp_to_nix(struct nix *nix, struct nix_txsch_alloc_rsp *rsp)
1205 {
1206         uint8_t lvl;
1207         uint16_t i;
1208
1209         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1210                 for (i = 0; i < rsp->schq[lvl]; i++)
1211                         plt_bitmap_set(nix->schq_bmp[lvl],
1212                                        rsp->schq_list[lvl][i]);
1213
1214                 for (i = 0; i < rsp->schq_contig[lvl]; i++)
1215                         plt_bitmap_set(nix->schq_contig_bmp[lvl],
1216                                        rsp->schq_contig_list[lvl][i]);
1217         }
1218 }
1219
1220 int
1221 nix_tm_alloc_txschq(struct nix *nix, enum roc_nix_tm_tree tree)
1222 {
1223         uint16_t schq_contig[NIX_TXSCH_LVL_CNT];
1224         struct mbox *mbox = (&nix->dev)->mbox;
1225         uint16_t schq[NIX_TXSCH_LVL_CNT];
1226         struct nix_txsch_alloc_req *req;
1227         struct nix_txsch_alloc_rsp *rsp;
1228         uint8_t hw_lvl, i;
1229         bool pend;
1230         int rc;
1231
1232         memset(schq, 0, sizeof(schq));
1233         memset(schq_contig, 0, sizeof(schq_contig));
1234
1235         /* Estimate requirement */
1236         rc = nix_tm_resource_estimate(nix, schq_contig, schq, tree);
1237         if (!rc)
1238                 return 0;
1239
1240         /* Release existing contiguous resources when realloc requested
1241          * as there is no way to guarantee continuity of old with new.
1242          */
1243         for (hw_lvl = 0; hw_lvl < NIX_TXSCH_LVL_CNT; hw_lvl++) {
1244                 if (schq_contig[hw_lvl])
1245                         nix_tm_release_resources(nix, hw_lvl, true, false);
1246         }
1247
1248         /* Alloc as needed */
1249         do {
1250                 pend = false;
1251                 req = mbox_alloc_msg_nix_txsch_alloc(mbox);
1252                 if (!req) {
1253                         rc = -ENOMEM;
1254                         goto alloc_err;
1255                 }
1256                 mbox_memcpy(req->schq, schq, sizeof(req->schq));
1257                 mbox_memcpy(req->schq_contig, schq_contig,
1258                             sizeof(req->schq_contig));
1259
1260                 /* Each alloc can be at max of MAX_TXSCHQ_PER_FUNC per level.
1261                  * So split alloc to multiple requests.
1262                  */
1263                 for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
1264                         if (req->schq[i] > MAX_TXSCHQ_PER_FUNC)
1265                                 req->schq[i] = MAX_TXSCHQ_PER_FUNC;
1266                         schq[i] -= req->schq[i];
1267
1268                         if (req->schq_contig[i] > MAX_TXSCHQ_PER_FUNC)
1269                                 req->schq_contig[i] = MAX_TXSCHQ_PER_FUNC;
1270                         schq_contig[i] -= req->schq_contig[i];
1271
1272                         if (schq[i] || schq_contig[i])
1273                                 pend = true;
1274                 }
1275
1276                 rc = mbox_process_msg(mbox, (void *)&rsp);
1277                 if (rc)
1278                         goto alloc_err;
1279
1280                 nix_tm_copy_rsp_to_nix(nix, rsp);
1281         } while (pend);
1282
1283         nix->tm_link_cfg_lvl = rsp->link_cfg_lvl;
1284         return 0;
1285 alloc_err:
1286         for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
1287                 if (nix_tm_release_resources(nix, i, true, false))
1288                         plt_err("Failed to release contig resources of "
1289                                 "lvl %d on error",
1290                                 i);
1291                 if (nix_tm_release_resources(nix, i, false, false))
1292                         plt_err("Failed to release discontig resources of "
1293                                 "lvl %d on error",
1294                                 i);
1295         }
1296         return rc;
1297 }
1298
1299 int
1300 nix_tm_prepare_default_tree(struct roc_nix *roc_nix)
1301 {
1302         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1303         uint32_t nonleaf_id = nix->nb_tx_queues;
1304         struct nix_tm_node *node = NULL;
1305         uint8_t leaf_lvl, lvl, lvl_end;
1306         uint32_t parent, i;
1307         int rc = 0;
1308
1309         /* Add ROOT, SCH1, SCH2, SCH3, [SCH4]  nodes */
1310         parent = ROC_NIX_TM_NODE_ID_INVALID;
1311         /* With TL1 access we have an extra level */
1312         lvl_end = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_SCH4 :
1313                                                        ROC_TM_LVL_SCH3);
1314
1315         for (lvl = ROC_TM_LVL_ROOT; lvl <= lvl_end; lvl++) {
1316                 rc = -ENOMEM;
1317                 node = nix_tm_node_alloc();
1318                 if (!node)
1319                         goto error;
1320
1321                 node->id = nonleaf_id;
1322                 node->parent_id = parent;
1323                 node->priority = 0;
1324                 node->weight = NIX_TM_DFLT_RR_WT;
1325                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1326                 node->lvl = lvl;
1327                 node->tree = ROC_NIX_TM_DEFAULT;
1328                 node->rel_chan = NIX_TM_CHAN_INVALID;
1329
1330                 rc = nix_tm_node_add(roc_nix, node);
1331                 if (rc)
1332                         goto error;
1333                 parent = nonleaf_id;
1334                 nonleaf_id++;
1335         }
1336
1337         parent = nonleaf_id - 1;
1338         leaf_lvl = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_QUEUE :
1339                                                         ROC_TM_LVL_SCH4);
1340
1341         /* Add leaf nodes */
1342         for (i = 0; i < nix->nb_tx_queues; i++) {
1343                 rc = -ENOMEM;
1344                 node = nix_tm_node_alloc();
1345                 if (!node)
1346                         goto error;
1347
1348                 node->id = i;
1349                 node->parent_id = parent;
1350                 node->priority = 0;
1351                 node->weight = NIX_TM_DFLT_RR_WT;
1352                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1353                 node->lvl = leaf_lvl;
1354                 node->tree = ROC_NIX_TM_DEFAULT;
1355                 node->rel_chan = NIX_TM_CHAN_INVALID;
1356
1357                 rc = nix_tm_node_add(roc_nix, node);
1358                 if (rc)
1359                         goto error;
1360         }
1361
1362         return 0;
1363 error:
1364         nix_tm_node_free(node);
1365         return rc;
1366 }
1367
1368 int
1369 roc_nix_tm_prepare_rate_limited_tree(struct roc_nix *roc_nix)
1370 {
1371         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1372         uint32_t nonleaf_id = nix->nb_tx_queues;
1373         struct nix_tm_node *node = NULL;
1374         uint8_t leaf_lvl, lvl, lvl_end;
1375         uint32_t parent, i;
1376         int rc = 0;
1377
1378         /* Add ROOT, SCH1, SCH2 nodes */
1379         parent = ROC_NIX_TM_NODE_ID_INVALID;
1380         lvl_end = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_SCH3 :
1381                                                        ROC_TM_LVL_SCH2);
1382
1383         for (lvl = ROC_TM_LVL_ROOT; lvl <= lvl_end; lvl++) {
1384                 rc = -ENOMEM;
1385                 node = nix_tm_node_alloc();
1386                 if (!node)
1387                         goto error;
1388
1389                 node->id = nonleaf_id;
1390                 node->parent_id = parent;
1391                 node->priority = 0;
1392                 node->weight = NIX_TM_DFLT_RR_WT;
1393                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1394                 node->lvl = lvl;
1395                 node->tree = ROC_NIX_TM_RLIMIT;
1396                 node->rel_chan = NIX_TM_CHAN_INVALID;
1397
1398                 rc = nix_tm_node_add(roc_nix, node);
1399                 if (rc)
1400                         goto error;
1401                 parent = nonleaf_id;
1402                 nonleaf_id++;
1403         }
1404
1405         /* SMQ is mapped to SCH4 when we have TL1 access and SCH3 otherwise */
1406         lvl = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_SCH4 : ROC_TM_LVL_SCH3);
1407
1408         /* Add per queue SMQ nodes i.e SCH4 / SCH3 */
1409         for (i = 0; i < nix->nb_tx_queues; i++) {
1410                 rc = -ENOMEM;
1411                 node = nix_tm_node_alloc();
1412                 if (!node)
1413                         goto error;
1414
1415                 node->id = nonleaf_id + i;
1416                 node->parent_id = parent;
1417                 node->priority = 0;
1418                 node->weight = NIX_TM_DFLT_RR_WT;
1419                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1420                 node->lvl = lvl;
1421                 node->tree = ROC_NIX_TM_RLIMIT;
1422                 node->rel_chan = NIX_TM_CHAN_INVALID;
1423
1424                 rc = nix_tm_node_add(roc_nix, node);
1425                 if (rc)
1426                         goto error;
1427         }
1428
1429         parent = nonleaf_id;
1430         leaf_lvl = (nix_tm_have_tl1_access(nix) ? ROC_TM_LVL_QUEUE :
1431                                                         ROC_TM_LVL_SCH4);
1432
1433         /* Add leaf nodes */
1434         for (i = 0; i < nix->nb_tx_queues; i++) {
1435                 rc = -ENOMEM;
1436                 node = nix_tm_node_alloc();
1437                 if (!node)
1438                         goto error;
1439
1440                 node->id = i;
1441                 node->parent_id = parent + i;
1442                 node->priority = 0;
1443                 node->weight = NIX_TM_DFLT_RR_WT;
1444                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1445                 node->lvl = leaf_lvl;
1446                 node->tree = ROC_NIX_TM_RLIMIT;
1447                 node->rel_chan = NIX_TM_CHAN_INVALID;
1448
1449                 rc = nix_tm_node_add(roc_nix, node);
1450                 if (rc)
1451                         goto error;
1452         }
1453
1454         return 0;
1455 error:
1456         nix_tm_node_free(node);
1457         return rc;
1458 }
1459
1460 int
1461 roc_nix_tm_pfc_prepare_tree(struct roc_nix *roc_nix)
1462 {
1463         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1464         uint32_t nonleaf_id = nix->nb_tx_queues;
1465         struct nix_tm_node *node = NULL;
1466         uint8_t leaf_lvl, lvl, lvl_end;
1467         uint32_t tl2_node_id;
1468         uint32_t parent, i;
1469         int rc = -ENOMEM;
1470
1471         parent = ROC_NIX_TM_NODE_ID_INVALID;
1472         lvl_end = ROC_TM_LVL_SCH3;
1473         leaf_lvl = ROC_TM_LVL_QUEUE;
1474
1475         /* TL1 node */
1476         node = nix_tm_node_alloc();
1477         if (!node)
1478                 goto error;
1479
1480         node->id = nonleaf_id;
1481         node->parent_id = parent;
1482         node->priority = 0;
1483         node->weight = NIX_TM_DFLT_RR_WT;
1484         node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1485         node->lvl = ROC_TM_LVL_ROOT;
1486         node->tree = ROC_NIX_TM_PFC;
1487         node->rel_chan = NIX_TM_CHAN_INVALID;
1488
1489         rc = nix_tm_node_add(roc_nix, node);
1490         if (rc)
1491                 goto error;
1492
1493         parent = nonleaf_id;
1494         nonleaf_id++;
1495
1496         /* TL2 node */
1497         rc = -ENOMEM;
1498         node = nix_tm_node_alloc();
1499         if (!node)
1500                 goto error;
1501
1502         node->id = nonleaf_id;
1503         node->parent_id = parent;
1504         node->priority = 0;
1505         node->weight = NIX_TM_DFLT_RR_WT;
1506         node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1507         node->lvl = ROC_TM_LVL_SCH1;
1508         node->tree = ROC_NIX_TM_PFC;
1509         node->rel_chan = NIX_TM_CHAN_INVALID;
1510
1511         rc = nix_tm_node_add(roc_nix, node);
1512         if (rc)
1513                 goto error;
1514
1515         tl2_node_id = nonleaf_id;
1516         nonleaf_id++;
1517
1518         for (i = 0; i < nix->nb_tx_queues; i++) {
1519                 parent = tl2_node_id;
1520                 for (lvl = ROC_TM_LVL_SCH2; lvl <= lvl_end; lvl++) {
1521                         rc = -ENOMEM;
1522                         node = nix_tm_node_alloc();
1523                         if (!node)
1524                                 goto error;
1525
1526                         node->id = nonleaf_id;
1527                         node->parent_id = parent;
1528                         node->priority = 0;
1529                         node->weight = NIX_TM_DFLT_RR_WT;
1530                         node->shaper_profile_id =
1531                                 ROC_NIX_TM_SHAPER_PROFILE_NONE;
1532                         node->lvl = lvl;
1533                         node->tree = ROC_NIX_TM_PFC;
1534                         node->rel_chan = NIX_TM_CHAN_INVALID;
1535
1536                         rc = nix_tm_node_add(roc_nix, node);
1537                         if (rc)
1538                                 goto error;
1539
1540                         parent = nonleaf_id;
1541                         nonleaf_id++;
1542                 }
1543
1544                 lvl = ROC_TM_LVL_SCH4;
1545
1546                 rc = -ENOMEM;
1547                 node = nix_tm_node_alloc();
1548                 if (!node)
1549                         goto error;
1550
1551                 node->id = nonleaf_id;
1552                 node->parent_id = parent;
1553                 node->priority = 0;
1554                 node->weight = NIX_TM_DFLT_RR_WT;
1555                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1556                 node->lvl = lvl;
1557                 node->tree = ROC_NIX_TM_PFC;
1558                 node->rel_chan = NIX_TM_CHAN_INVALID;
1559
1560                 rc = nix_tm_node_add(roc_nix, node);
1561                 if (rc)
1562                         goto error;
1563
1564                 parent = nonleaf_id;
1565                 nonleaf_id++;
1566
1567                 rc = -ENOMEM;
1568                 node = nix_tm_node_alloc();
1569                 if (!node)
1570                         goto error;
1571
1572                 node->id = i;
1573                 node->parent_id = parent;
1574                 node->priority = 0;
1575                 node->weight = NIX_TM_DFLT_RR_WT;
1576                 node->shaper_profile_id = ROC_NIX_TM_SHAPER_PROFILE_NONE;
1577                 node->lvl = leaf_lvl;
1578                 node->tree = ROC_NIX_TM_PFC;
1579                 node->rel_chan = NIX_TM_CHAN_INVALID;
1580
1581                 rc = nix_tm_node_add(roc_nix, node);
1582                 if (rc)
1583                         goto error;
1584         }
1585
1586         return 0;
1587 error:
1588         nix_tm_node_free(node);
1589         return rc;
1590 }
1591
1592 int
1593 nix_tm_free_resources(struct roc_nix *roc_nix, uint32_t tree_mask, bool hw_only)
1594 {
1595         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1596         struct nix_tm_shaper_profile *profile;
1597         struct nix_tm_node *node, *next_node;
1598         struct nix_tm_node_list *list;
1599         enum roc_nix_tm_tree tree;
1600         uint32_t profile_id;
1601         int rc = 0;
1602
1603         for (tree = 0; tree < ROC_NIX_TM_TREE_MAX; tree++) {
1604                 if (!(tree_mask & BIT(tree)))
1605                         continue;
1606
1607                 plt_tm_dbg("Freeing resources of tree %u", tree);
1608
1609                 list = nix_tm_node_list(nix, tree);
1610                 next_node = TAILQ_FIRST(list);
1611                 while (next_node) {
1612                         node = next_node;
1613                         next_node = TAILQ_NEXT(node, node);
1614
1615                         if (!nix_tm_is_leaf(nix, node->lvl) &&
1616                             node->flags & NIX_TM_NODE_HWRES) {
1617                                 /* Clear xoff in path for flush to succeed */
1618                                 rc = nix_tm_clear_path_xoff(nix, node);
1619                                 if (rc)
1620                                         return rc;
1621                                 rc = nix_tm_free_node_resource(nix, node);
1622                                 if (rc)
1623                                         return rc;
1624                         }
1625                 }
1626
1627                 /* Leave software elements if needed */
1628                 if (hw_only)
1629                         continue;
1630
1631                 next_node = TAILQ_FIRST(list);
1632                 while (next_node) {
1633                         node = next_node;
1634                         next_node = TAILQ_NEXT(node, node);
1635
1636                         plt_tm_dbg("Free node lvl %u id %u (%p)", node->lvl,
1637                                    node->id, node);
1638
1639                         profile_id = node->shaper_profile_id;
1640                         profile = nix_tm_shaper_profile_search(nix, profile_id);
1641                         if (profile)
1642                                 profile->ref_cnt--;
1643
1644                         TAILQ_REMOVE(list, node, node);
1645                         nix_tm_node_free(node);
1646                 }
1647         }
1648         return rc;
1649 }
1650
1651 int
1652 nix_tm_conf_init(struct roc_nix *roc_nix)
1653 {
1654         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1655         uint32_t bmp_sz, hw_lvl;
1656         void *bmp_mem;
1657         int rc, i;
1658
1659         PLT_STATIC_ASSERT(sizeof(struct nix_tm_node) <= ROC_NIX_TM_NODE_SZ);
1660         PLT_STATIC_ASSERT(sizeof(struct nix_tm_shaper_profile) <=
1661                           ROC_NIX_TM_SHAPER_PROFILE_SZ);
1662
1663         nix->tm_flags = 0;
1664         for (i = 0; i < ROC_NIX_TM_TREE_MAX; i++)
1665                 TAILQ_INIT(&nix->trees[i]);
1666
1667         TAILQ_INIT(&nix->shaper_profile_list);
1668         nix->tm_rate_min = 1E9; /* 1Gbps */
1669
1670         rc = -ENOMEM;
1671         bmp_sz = plt_bitmap_get_memory_footprint(NIX_TM_MAX_HW_TXSCHQ);
1672         bmp_mem = plt_zmalloc(bmp_sz * NIX_TXSCH_LVL_CNT * 2, 0);
1673         if (!bmp_mem)
1674                 return rc;
1675         nix->schq_bmp_mem = bmp_mem;
1676
1677         /* Init contiguous and discontiguous bitmap per lvl */
1678         rc = -EIO;
1679         for (hw_lvl = 0; hw_lvl < NIX_TXSCH_LVL_CNT; hw_lvl++) {
1680                 /* Bitmap for discontiguous resource */
1681                 nix->schq_bmp[hw_lvl] =
1682                         plt_bitmap_init(NIX_TM_MAX_HW_TXSCHQ, bmp_mem, bmp_sz);
1683                 if (!nix->schq_bmp[hw_lvl])
1684                         goto exit;
1685
1686                 bmp_mem = PLT_PTR_ADD(bmp_mem, bmp_sz);
1687
1688                 /* Bitmap for contiguous resource */
1689                 nix->schq_contig_bmp[hw_lvl] =
1690                         plt_bitmap_init(NIX_TM_MAX_HW_TXSCHQ, bmp_mem, bmp_sz);
1691                 if (!nix->schq_contig_bmp[hw_lvl])
1692                         goto exit;
1693
1694                 bmp_mem = PLT_PTR_ADD(bmp_mem, bmp_sz);
1695         }
1696
1697         rc = nix_tm_mark_init(nix);
1698         if (rc)
1699                 goto exit;
1700
1701         /* Disable TL1 Static Priority when VF's are enabled
1702          * as otherwise VF's TL2 reallocation will be needed
1703          * runtime to support a specific topology of PF.
1704          */
1705         if (nix->pci_dev->max_vfs)
1706                 nix->tm_flags |= NIX_TM_TL1_NO_SP;
1707
1708         /* TL1 access is only for PF's */
1709         if (roc_nix_is_pf(roc_nix)) {
1710                 nix->tm_flags |= NIX_TM_TL1_ACCESS;
1711                 nix->tm_root_lvl = NIX_TXSCH_LVL_TL1;
1712         } else {
1713                 nix->tm_root_lvl = NIX_TXSCH_LVL_TL2;
1714         }
1715
1716         return 0;
1717 exit:
1718         nix_tm_conf_fini(roc_nix);
1719         return rc;
1720 }
1721
1722 void
1723 nix_tm_conf_fini(struct roc_nix *roc_nix)
1724 {
1725         struct nix *nix = roc_nix_to_nix_priv(roc_nix);
1726         uint16_t hw_lvl;
1727
1728         for (hw_lvl = 0; hw_lvl < NIX_TXSCH_LVL_CNT; hw_lvl++) {
1729                 plt_bitmap_free(nix->schq_bmp[hw_lvl]);
1730                 plt_bitmap_free(nix->schq_contig_bmp[hw_lvl]);
1731         }
1732         plt_free(nix->schq_bmp_mem);
1733 }