drivers/net/octeontx2/otx2_tm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <rte_malloc.h>
   6
   7 #include "otx2_ethdev.h"
   8 #include "otx2_tm.h"
   9
  10 /* Use last LVL_CNT nodes as default nodes */
  11 #define NIX_DEFAULT_NODE_ID_START (RTE_TM_NODE_ID_NULL - NIX_TXSCH_LVL_CNT)
  12
  13 enum otx2_tm_node_level {
  14         OTX2_TM_LVL_ROOT = 0,
  15         OTX2_TM_LVL_SCH1,
  16         OTX2_TM_LVL_SCH2,
  17         OTX2_TM_LVL_SCH3,
  18         OTX2_TM_LVL_SCH4,
  19         OTX2_TM_LVL_QUEUE,
  20         OTX2_TM_LVL_MAX,
  21 };
  22
  23 static bool
  24 nix_tm_have_tl1_access(struct otx2_eth_dev *dev)
  25 {
  26         bool is_lbk = otx2_dev_is_lbk(dev);
  27         return otx2_dev_is_pf(dev) && !otx2_dev_is_A0(dev) &&
  28                 !is_lbk && !dev->maxvf;
  29 }
  30
  31 static struct otx2_nix_tm_shaper_profile *
  32 nix_tm_shaper_profile_search(struct otx2_eth_dev *dev, uint32_t shaper_id)
  33 {
  34         struct otx2_nix_tm_shaper_profile *tm_shaper_profile;
  35
  36         TAILQ_FOREACH(tm_shaper_profile, &dev->shaper_profile_list, shaper) {
  37                 if (tm_shaper_profile->shaper_profile_id == shaper_id)
  38                         return tm_shaper_profile;
  39         }
  40         return NULL;
  41 }
  42
  43 static struct otx2_nix_tm_node *
  44 nix_tm_node_search(struct otx2_eth_dev *dev,
  45                    uint32_t node_id, bool user)
  46 {
  47         struct otx2_nix_tm_node *tm_node;
  48
  49         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
  50                 if (tm_node->id == node_id &&
  51                     (user == !!(tm_node->flags & NIX_TM_NODE_USER)))
  52                         return tm_node;
  53         }
  54         return NULL;
  55 }
  56
  57 static uint32_t
  58 check_rr(struct otx2_eth_dev *dev, uint32_t priority, uint32_t parent_id)
  59 {
  60         struct otx2_nix_tm_node *tm_node;
  61         uint32_t rr_num = 0;
  62
  63         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
  64                 if (!tm_node->parent)
  65                         continue;
  66
  67                 if (!(tm_node->parent->id == parent_id))
  68                         continue;
  69
  70                 if (tm_node->priority == priority)
  71                         rr_num++;
  72         }
  73         return rr_num;
  74 }
  75
  76 static int
  77 nix_tm_update_parent_info(struct otx2_eth_dev *dev)
  78 {
  79         struct otx2_nix_tm_node *tm_node_child;
  80         struct otx2_nix_tm_node *tm_node;
  81         struct otx2_nix_tm_node *parent;
  82         uint32_t rr_num = 0;
  83         uint32_t priority;
  84
  85         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
  86                 if (!tm_node->parent)
  87                         continue;
  88                 /* Count group of children of same priority i.e are RR */
  89                 parent = tm_node->parent;
  90                 priority = tm_node->priority;
  91                 rr_num = check_rr(dev, priority, parent->id);
  92
  93                 /* Assuming that multiple RR groups are
  94                  * not configured based on capability.
  95                  */
  96                 if (rr_num > 1) {
  97                         parent->rr_prio = priority;
  98                         parent->rr_num = rr_num;
  99                 }
 100
 101                 /* Find out static priority children that are not in RR */
 102                 TAILQ_FOREACH(tm_node_child, &dev->node_list, node) {
 103                         if (!tm_node_child->parent)
 104                                 continue;
 105                         if (parent->id != tm_node_child->parent->id)
 106                                 continue;
 107                         if (parent->max_prio == UINT32_MAX &&
 108                             tm_node_child->priority != parent->rr_prio)
 109                                 parent->max_prio = 0;
 110
 111                         if (parent->max_prio < tm_node_child->priority &&
 112                             parent->rr_prio != tm_node_child->priority)
 113                                 parent->max_prio = tm_node_child->priority;
 114                 }
 115         }
 116
 117         return 0;
 118 }
 119
 120 static int
 121 nix_tm_node_add_to_list(struct otx2_eth_dev *dev, uint32_t node_id,
 122                         uint32_t parent_node_id, uint32_t priority,
 123                         uint32_t weight, uint16_t hw_lvl_id,
 124                         uint16_t level_id, bool user,
 125                         struct rte_tm_node_params *params)
 126 {
 127         struct otx2_nix_tm_shaper_profile *shaper_profile;
 128         struct otx2_nix_tm_node *tm_node, *parent_node;
 129         uint32_t shaper_profile_id;
 130
 131         shaper_profile_id = params->shaper_profile_id;
 132         shaper_profile = nix_tm_shaper_profile_search(dev, shaper_profile_id);
 133
 134         parent_node = nix_tm_node_search(dev, parent_node_id, user);
 135
 136         tm_node = rte_zmalloc("otx2_nix_tm_node",
 137                               sizeof(struct otx2_nix_tm_node), 0);
 138         if (!tm_node)
 139                 return -ENOMEM;
 140
 141         tm_node->level_id = level_id;
 142         tm_node->hw_lvl_id = hw_lvl_id;
 143
 144         tm_node->id = node_id;
 145         tm_node->priority = priority;
 146         tm_node->weight = weight;
 147         tm_node->rr_prio = 0xf;
 148         tm_node->max_prio = UINT32_MAX;
 149         tm_node->hw_id = UINT32_MAX;
 150         tm_node->flags = 0;
 151         if (user)
 152                 tm_node->flags = NIX_TM_NODE_USER;
 153         rte_memcpy(&tm_node->params, params, sizeof(struct rte_tm_node_params));
 154
 155         if (shaper_profile)
 156                 shaper_profile->reference_count++;
 157         tm_node->parent = parent_node;
 158         tm_node->parent_hw_id = UINT32_MAX;
 159
 160         TAILQ_INSERT_TAIL(&dev->node_list, tm_node, node);
 161
 162         return 0;
 163 }
 164
 165 static int
 166 nix_tm_clear_shaper_profiles(struct otx2_eth_dev *dev)
 167 {
 168         struct otx2_nix_tm_shaper_profile *shaper_profile;
 169
 170         while ((shaper_profile = TAILQ_FIRST(&dev->shaper_profile_list))) {
 171                 if (shaper_profile->reference_count)
 172                         otx2_tm_dbg("Shaper profile %u has non zero references",
 173                                     shaper_profile->shaper_profile_id);
 174                 TAILQ_REMOVE(&dev->shaper_profile_list, shaper_profile, shaper);
 175                 rte_free(shaper_profile);
 176         }
 177
 178         return 0;
 179 }
 180
 181 static int
 182 nix_tm_free_resources(struct otx2_eth_dev *dev, uint32_t flags_mask,
 183                       uint32_t flags, bool hw_only)
 184 {
 185         struct otx2_nix_tm_shaper_profile *shaper_profile;
 186         struct otx2_nix_tm_node *tm_node, *next_node;
 187         struct otx2_mbox *mbox = dev->mbox;
 188         struct nix_txsch_free_req *req;
 189         uint32_t shaper_profile_id;
 190         bool skip_node = false;
 191         int rc = 0;
 192
 193         next_node = TAILQ_FIRST(&dev->node_list);
 194         while (next_node) {
 195                 tm_node = next_node;
 196                 next_node = TAILQ_NEXT(tm_node, node);
 197
 198                 /* Check for only requested nodes */
 199                 if ((tm_node->flags & flags_mask) != flags)
 200                         continue;
 201
 202                 if (nix_tm_have_tl1_access(dev) &&
 203                     tm_node->hw_lvl_id ==  NIX_TXSCH_LVL_TL1)
 204                         skip_node = true;
 205
 206                 otx2_tm_dbg("Free hwres for node %u, hwlvl %u, hw_id %u (%p)",
 207                             tm_node->id,  tm_node->hw_lvl_id,
 208                             tm_node->hw_id, tm_node);
 209                 /* Free specific HW resource if requested */
 210                 if (!skip_node && flags_mask &&
 211                     tm_node->flags & NIX_TM_NODE_HWRES) {
 212                         req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
 213                         req->flags = 0;
 214                         req->schq_lvl = tm_node->hw_lvl_id;
 215                         req->schq = tm_node->hw_id;
 216                         rc = otx2_mbox_process(mbox);
 217                         if (rc)
 218                                 break;
 219                 } else {
 220                         skip_node = false;
 221                 }
 222                 tm_node->flags &= ~NIX_TM_NODE_HWRES;
 223
 224                 /* Leave software elements if needed */
 225                 if (hw_only)
 226                         continue;
 227
 228                 shaper_profile_id = tm_node->params.shaper_profile_id;
 229                 shaper_profile =
 230                         nix_tm_shaper_profile_search(dev, shaper_profile_id);
 231                 if (shaper_profile)
 232                         shaper_profile->reference_count--;
 233
 234                 TAILQ_REMOVE(&dev->node_list, tm_node, node);
 235                 rte_free(tm_node);
 236         }
 237
 238         if (!flags_mask) {
 239                 /* Free all hw resources */
 240                 req = otx2_mbox_alloc_msg_nix_txsch_free(mbox);
 241                 req->flags = TXSCHQ_FREE_ALL;
 242
 243                 return otx2_mbox_process(mbox);
 244         }
 245
 246         return rc;
 247 }
 248
 249 static uint8_t
 250 nix_tm_copy_rsp_to_dev(struct otx2_eth_dev *dev,
 251                        struct nix_txsch_alloc_rsp *rsp)
 252 {
 253         uint16_t schq;
 254         uint8_t lvl;
 255
 256         for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
 257                 for (schq = 0; schq < MAX_TXSCHQ_PER_FUNC; schq++) {
 258                         dev->txschq_list[lvl][schq] = rsp->schq_list[lvl][schq];
 259                         dev->txschq_contig_list[lvl][schq] =
 260                                 rsp->schq_contig_list[lvl][schq];
 261                 }
 262
 263                 dev->txschq[lvl] = rsp->schq[lvl];
 264                 dev->txschq_contig[lvl] = rsp->schq_contig[lvl];
 265         }
 266         return 0;
 267 }
 268
 269 static int
 270 nix_tm_assign_id_to_node(struct otx2_eth_dev *dev,
 271                          struct otx2_nix_tm_node *child,
 272                          struct otx2_nix_tm_node *parent)
 273 {
 274         uint32_t hw_id, schq_con_index, prio_offset;
 275         uint32_t l_id, schq_index;
 276
 277         otx2_tm_dbg("Assign hw id for child node %u, lvl %u, hw_lvl %u (%p)",
 278                     child->id, child->level_id, child->hw_lvl_id, child);
 279
 280         child->flags |= NIX_TM_NODE_HWRES;
 281
 282         /* Process root nodes */
 283         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL2 &&
 284             child->hw_lvl_id == dev->otx2_tm_root_lvl && !parent) {
 285                 int idx = 0;
 286                 uint32_t tschq_con_index;
 287
 288                 l_id = child->hw_lvl_id;
 289                 tschq_con_index = dev->txschq_contig_index[l_id];
 290                 hw_id = dev->txschq_contig_list[l_id][tschq_con_index];
 291                 child->hw_id = hw_id;
 292                 dev->txschq_contig_index[l_id]++;
 293                 /* Update TL1 hw_id for its parent for config purpose */
 294                 idx = dev->txschq_index[NIX_TXSCH_LVL_TL1]++;
 295                 hw_id = dev->txschq_list[NIX_TXSCH_LVL_TL1][idx];
 296                 child->parent_hw_id = hw_id;
 297                 return 0;
 298         }
 299         if (dev->otx2_tm_root_lvl == NIX_TXSCH_LVL_TL1 &&
 300             child->hw_lvl_id == dev->otx2_tm_root_lvl && !parent) {
 301                 uint32_t tschq_con_index;
 302
 303                 l_id = child->hw_lvl_id;
 304                 tschq_con_index = dev->txschq_index[l_id];
 305                 hw_id = dev->txschq_list[l_id][tschq_con_index];
 306                 child->hw_id = hw_id;
 307                 dev->txschq_index[l_id]++;
 308                 return 0;
 309         }
 310
 311         /* Process children with parents */
 312         l_id = child->hw_lvl_id;
 313         schq_index = dev->txschq_index[l_id];
 314         schq_con_index = dev->txschq_contig_index[l_id];
 315
 316         if (child->priority == parent->rr_prio) {
 317                 hw_id = dev->txschq_list[l_id][schq_index];
 318                 child->hw_id = hw_id;
 319                 child->parent_hw_id = parent->hw_id;
 320                 dev->txschq_index[l_id]++;
 321         } else {
 322                 prio_offset = schq_con_index + child->priority;
 323                 hw_id = dev->txschq_contig_list[l_id][prio_offset];
 324                 child->hw_id = hw_id;
 325         }
 326         return 0;
 327 }
 328
 329 static int
 330 nix_tm_assign_hw_id(struct otx2_eth_dev *dev)
 331 {
 332         struct otx2_nix_tm_node *parent, *child;
 333         uint32_t child_hw_lvl, con_index_inc, i;
 334
 335         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--) {
 336                 TAILQ_FOREACH(parent, &dev->node_list, node) {
 337                         child_hw_lvl = parent->hw_lvl_id - 1;
 338                         if (parent->hw_lvl_id != i)
 339                                 continue;
 340                         TAILQ_FOREACH(child, &dev->node_list, node) {
 341                                 if (!child->parent)
 342                                         continue;
 343                                 if (child->parent->id != parent->id)
 344                                         continue;
 345                                 nix_tm_assign_id_to_node(dev, child, parent);
 346                         }
 347
 348                         con_index_inc = parent->max_prio + 1;
 349                         dev->txschq_contig_index[child_hw_lvl] += con_index_inc;
 350
 351                         /*
 352                          * Explicitly assign id to parent node if it
 353                          * doesn't have a parent
 354                          */
 355                         if (parent->hw_lvl_id == dev->otx2_tm_root_lvl)
 356                                 nix_tm_assign_id_to_node(dev, parent, NULL);
 357                 }
 358         }
 359         return 0;
 360 }
 361
 362 static uint8_t
 363 nix_tm_count_req_schq(struct otx2_eth_dev *dev,
 364                       struct nix_txsch_alloc_req *req, uint8_t lvl)
 365 {
 366         struct otx2_nix_tm_node *tm_node;
 367         uint8_t contig_count;
 368
 369         TAILQ_FOREACH(tm_node, &dev->node_list, node) {
 370                 if (lvl == tm_node->hw_lvl_id) {
 371                         req->schq[lvl - 1] += tm_node->rr_num;
 372                         if (tm_node->max_prio != UINT32_MAX) {
 373                                 contig_count = tm_node->max_prio + 1;
 374                                 req->schq_contig[lvl - 1] += contig_count;
 375                         }
 376                 }
 377                 if (lvl == dev->otx2_tm_root_lvl &&
 378                     dev->otx2_tm_root_lvl && lvl == NIX_TXSCH_LVL_TL2 &&
 379                     tm_node->hw_lvl_id == dev->otx2_tm_root_lvl) {
 380                         req->schq_contig[dev->otx2_tm_root_lvl]++;
 381                 }
 382         }
 383
 384         req->schq[NIX_TXSCH_LVL_TL1] = 1;
 385         req->schq_contig[NIX_TXSCH_LVL_TL1] = 0;
 386
 387         return 0;
 388 }
 389
 390 static int
 391 nix_tm_prepare_txschq_req(struct otx2_eth_dev *dev,
 392                           struct nix_txsch_alloc_req *req)
 393 {
 394         uint8_t i;
 395
 396         for (i = NIX_TXSCH_LVL_TL1; i > 0; i--)
 397                 nix_tm_count_req_schq(dev, req, i);
 398
 399         for (i = 0; i < NIX_TXSCH_LVL_CNT; i++) {
 400                 dev->txschq_index[i] = 0;
 401                 dev->txschq_contig_index[i] = 0;
 402         }
 403         return 0;
 404 }
 405
 406 static int
 407 nix_tm_send_txsch_alloc_msg(struct otx2_eth_dev *dev)
 408 {
 409         struct otx2_mbox *mbox = dev->mbox;
 410         struct nix_txsch_alloc_req *req;
 411         struct nix_txsch_alloc_rsp *rsp;
 412         int rc;
 413
 414         req = otx2_mbox_alloc_msg_nix_txsch_alloc(mbox);
 415
 416         rc = nix_tm_prepare_txschq_req(dev, req);
 417         if (rc)
 418                 return rc;
 419
 420         rc = otx2_mbox_process_msg(mbox, (void *)&rsp);
 421         if (rc)
 422                 return rc;
 423
 424         nix_tm_copy_rsp_to_dev(dev, rsp);
 425
 426         nix_tm_assign_hw_id(dev);
 427         return 0;
 428 }
 429
 430 static int
 431 nix_tm_alloc_resources(struct rte_eth_dev *eth_dev, bool xmit_enable)
 432 {
 433         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 434         int rc;
 435
 436         RTE_SET_USED(xmit_enable);
 437
 438         nix_tm_update_parent_info(dev);
 439
 440         rc = nix_tm_send_txsch_alloc_msg(dev);
 441         if (rc) {
 442                 otx2_err("TM failed to alloc tm resources=%d", rc);
 443                 return rc;
 444         }
 445
 446         return 0;
 447 }
 448
 449 static int
 450 nix_tm_prepare_default_tree(struct rte_eth_dev *eth_dev)
 451 {
 452         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 453         uint32_t def = eth_dev->data->nb_tx_queues;
 454         struct rte_tm_node_params params;
 455         uint32_t leaf_parent, i;
 456         int rc = 0;
 457
 458         /* Default params */
 459         memset(&params, 0, sizeof(params));
 460         params.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
 461
 462         if (nix_tm_have_tl1_access(dev)) {
 463                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL1;
 464                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
 465                                              DEFAULT_RR_WEIGHT,
 466                                              NIX_TXSCH_LVL_TL1,
 467                                              OTX2_TM_LVL_ROOT, false, &params);
 468                 if (rc)
 469                         goto exit;
 470                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
 471                                              DEFAULT_RR_WEIGHT,
 472                                              NIX_TXSCH_LVL_TL2,
 473                                              OTX2_TM_LVL_SCH1, false, &params);
 474                 if (rc)
 475                         goto exit;
 476
 477                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
 478                                              DEFAULT_RR_WEIGHT,
 479                                              NIX_TXSCH_LVL_TL3,
 480                                              OTX2_TM_LVL_SCH2, false, &params);
 481                 if (rc)
 482                         goto exit;
 483
 484                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
 485                                              DEFAULT_RR_WEIGHT,
 486                                              NIX_TXSCH_LVL_TL4,
 487                                              OTX2_TM_LVL_SCH3, false, &params);
 488                 if (rc)
 489                         goto exit;
 490
 491                 rc = nix_tm_node_add_to_list(dev, def + 4, def + 3, 0,
 492                                              DEFAULT_RR_WEIGHT,
 493                                              NIX_TXSCH_LVL_SMQ,
 494                                              OTX2_TM_LVL_SCH4, false, &params);
 495                 if (rc)
 496                         goto exit;
 497
 498                 leaf_parent = def + 4;
 499         } else {
 500                 dev->otx2_tm_root_lvl = NIX_TXSCH_LVL_TL2;
 501                 rc = nix_tm_node_add_to_list(dev, def, RTE_TM_NODE_ID_NULL, 0,
 502                                              DEFAULT_RR_WEIGHT,
 503                                              NIX_TXSCH_LVL_TL2,
 504                                              OTX2_TM_LVL_ROOT, false, &params);
 505                 if (rc)
 506                         goto exit;
 507
 508                 rc = nix_tm_node_add_to_list(dev, def + 1, def, 0,
 509                                              DEFAULT_RR_WEIGHT,
 510                                              NIX_TXSCH_LVL_TL3,
 511                                              OTX2_TM_LVL_SCH1, false, &params);
 512                 if (rc)
 513                         goto exit;
 514
 515                 rc = nix_tm_node_add_to_list(dev, def + 2, def + 1, 0,
 516                                              DEFAULT_RR_WEIGHT,
 517                                              NIX_TXSCH_LVL_TL4,
 518                                              OTX2_TM_LVL_SCH2, false, &params);
 519                 if (rc)
 520                         goto exit;
 521
 522                 rc = nix_tm_node_add_to_list(dev, def + 3, def + 2, 0,
 523                                              DEFAULT_RR_WEIGHT,
 524                                              NIX_TXSCH_LVL_SMQ,
 525                                              OTX2_TM_LVL_SCH3, false, &params);
 526                 if (rc)
 527                         goto exit;
 528
 529                 leaf_parent = def + 3;
 530         }
 531
 532         /* Add leaf nodes */
 533         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
 534                 rc = nix_tm_node_add_to_list(dev, i, leaf_parent, 0,
 535                                              DEFAULT_RR_WEIGHT,
 536                                              NIX_TXSCH_LVL_CNT,
 537                                              OTX2_TM_LVL_QUEUE, false, &params);
 538                 if (rc)
 539                         break;
 540         }
 541
 542 exit:
 543         return rc;
 544 }
 545
 546 void otx2_nix_tm_conf_init(struct rte_eth_dev *eth_dev)
 547 {
 548         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 549
 550         TAILQ_INIT(&dev->node_list);
 551         TAILQ_INIT(&dev->shaper_profile_list);
 552 }
 553
 554 int otx2_nix_tm_init_default(struct rte_eth_dev *eth_dev)
 555 {
 556         struct otx2_eth_dev  *dev = otx2_eth_pmd_priv(eth_dev);
 557         uint16_t sq_cnt = eth_dev->data->nb_tx_queues;
 558         int rc;
 559
 560         /* Free up all resources already held */
 561         rc = nix_tm_free_resources(dev, 0, 0, false);
 562         if (rc) {
 563                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
 564                 return rc;
 565         }
 566
 567         /* Clear shaper profiles */
 568         nix_tm_clear_shaper_profiles(dev);
 569         dev->tm_flags = NIX_TM_DEFAULT_TREE;
 570
 571         rc = nix_tm_prepare_default_tree(eth_dev);
 572         if (rc != 0)
 573                 return rc;
 574
 575         rc = nix_tm_alloc_resources(eth_dev, false);
 576         if (rc != 0)
 577                 return rc;
 578         dev->tm_leaf_cnt = sq_cnt;
 579
 580         return 0;
 581 }
 582
 583 int
 584 otx2_nix_tm_fini(struct rte_eth_dev *eth_dev)
 585 {
 586         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 587         int rc;
 588
 589         /* Xmit is assumed to be disabled */
 590         /* Free up resources already held */
 591         rc = nix_tm_free_resources(dev, 0, 0, false);
 592         if (rc) {
 593                 otx2_err("Failed to freeup existing resources,rc=%d", rc);
 594                 return rc;
 595         }
 596
 597         /* Clear shaper profiles */
 598         nix_tm_clear_shaper_profiles(dev);
 599
 600         dev->tm_flags = 0;
 601         return 0;
 602 }