drivers/net/softnic/rte_eth_softnic_tm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8
   9 #include <rte_malloc.h>
  10 #include <rte_string_fns.h>
  11
  12 #include "rte_eth_softnic_internals.h"
  13 #include "rte_eth_softnic.h"
  14
  15 #define SUBPORT_TC_PERIOD       10
  16 #define PIPE_TC_PERIOD          40
  17
  18 int
  19 softnic_tmgr_init(struct pmd_internals *p)
  20 {
  21         TAILQ_INIT(&p->tmgr_port_list);
  22
  23         return 0;
  24 }
  25
  26 void
  27 softnic_tmgr_free(struct pmd_internals *p)
  28 {
  29         for ( ; ; ) {
  30                 struct softnic_tmgr_port *tmgr_port;
  31
  32                 tmgr_port = TAILQ_FIRST(&p->tmgr_port_list);
  33                 if (tmgr_port == NULL)
  34                         break;
  35
  36                 TAILQ_REMOVE(&p->tmgr_port_list, tmgr_port, node);
  37                 rte_sched_port_free(tmgr_port->s);
  38                 free(tmgr_port);
  39         }
  40 }
  41
  42 struct softnic_tmgr_port *
  43 softnic_tmgr_port_find(struct pmd_internals *p,
  44         const char *name)
  45 {
  46         struct softnic_tmgr_port *tmgr_port;
  47
  48         if (name == NULL)
  49                 return NULL;
  50
  51         TAILQ_FOREACH(tmgr_port, &p->tmgr_port_list, node)
  52                 if (strcmp(tmgr_port->name, name) == 0)
  53                         return tmgr_port;
  54
  55         return NULL;
  56 }
  57
  58 struct softnic_tmgr_port *
  59 softnic_tmgr_port_create(struct pmd_internals *p,
  60         const char *name)
  61 {
  62         struct softnic_tmgr_port *tmgr_port;
  63         struct tm_params *t = &p->soft.tm.params;
  64         struct rte_sched_port *sched;
  65         uint32_t n_subports, subport_id;
  66
  67         /* Check input params */
  68         if (name == NULL ||
  69                 softnic_tmgr_port_find(p, name))
  70                 return NULL;
  71
  72         /*
  73          * Resource
  74          */
  75
  76         /* Is hierarchy frozen? */
  77         if (p->soft.tm.hierarchy_frozen == 0)
  78                 return NULL;
  79
  80         /* Port */
  81         sched = rte_sched_port_config(&t->port_params);
  82         if (sched == NULL)
  83                 return NULL;
  84
  85         /* Subport */
  86         n_subports = t->port_params.n_subports_per_port;
  87         for (subport_id = 0; subport_id < n_subports; subport_id++) {
  88                 uint32_t n_pipes_per_subport =
  89                 t->subport_params[subport_id].n_pipes_per_subport_enabled;
  90                 uint32_t pipe_id;
  91                 int status;
  92
  93                 status = rte_sched_subport_config(sched,
  94                         subport_id,
  95                         &t->subport_params[subport_id],
  96                         t->subport_to_profile[subport_id]);
  97                 if (status) {
  98                         rte_sched_port_free(sched);
  99                         return NULL;
 100                 }
 101
 102                 /* Pipe */
 103                 for (pipe_id = 0; pipe_id < n_pipes_per_subport; pipe_id++) {
 104                         int pos = subport_id * TM_MAX_PIPES_PER_SUBPORT + pipe_id;
 105                         int profile_id = t->pipe_to_profile[pos];
 106
 107                         if (profile_id < 0)
 108                                 continue;
 109
 110                         status = rte_sched_pipe_config(sched,
 111                                 subport_id,
 112                                 pipe_id,
 113                                 profile_id);
 114                         if (status) {
 115                                 rte_sched_port_free(sched);
 116                                 return NULL;
 117                         }
 118                 }
 119         }
 120
 121         /* Node allocation */
 122         tmgr_port = calloc(1, sizeof(struct softnic_tmgr_port));
 123         if (tmgr_port == NULL) {
 124                 rte_sched_port_free(sched);
 125                 return NULL;
 126         }
 127
 128         /* Node fill in */
 129         strlcpy(tmgr_port->name, name, sizeof(tmgr_port->name));
 130         tmgr_port->s = sched;
 131
 132         /* Node add to list */
 133         TAILQ_INSERT_TAIL(&p->tmgr_port_list, tmgr_port, node);
 134
 135         return tmgr_port;
 136 }
 137
 138 static struct rte_sched_port *
 139 SCHED(struct pmd_internals *p)
 140 {
 141         struct softnic_tmgr_port *tmgr_port;
 142
 143         tmgr_port = softnic_tmgr_port_find(p, "TMGR");
 144         if (tmgr_port == NULL)
 145                 return NULL;
 146
 147         return tmgr_port->s;
 148 }
 149
 150 void
 151 tm_hierarchy_init(struct pmd_internals *p)
 152 {
 153         memset(&p->soft.tm, 0, sizeof(p->soft.tm));
 154
 155         /* Initialize shaper profile list */
 156         TAILQ_INIT(&p->soft.tm.h.shaper_profiles);
 157
 158         /* Initialize shared shaper list */
 159         TAILQ_INIT(&p->soft.tm.h.shared_shapers);
 160
 161         /* Initialize wred profile list */
 162         TAILQ_INIT(&p->soft.tm.h.wred_profiles);
 163
 164         /* Initialize TM node list */
 165         TAILQ_INIT(&p->soft.tm.h.nodes);
 166 }
 167
 168 void
 169 tm_hierarchy_free(struct pmd_internals *p)
 170 {
 171         /* Remove all nodes*/
 172         for ( ; ; ) {
 173                 struct tm_node *tm_node;
 174
 175                 tm_node = TAILQ_FIRST(&p->soft.tm.h.nodes);
 176                 if (tm_node == NULL)
 177                         break;
 178
 179                 TAILQ_REMOVE(&p->soft.tm.h.nodes, tm_node, node);
 180                 free(tm_node);
 181         }
 182
 183         /* Remove all WRED profiles */
 184         for ( ; ; ) {
 185                 struct tm_wred_profile *wred_profile;
 186
 187                 wred_profile = TAILQ_FIRST(&p->soft.tm.h.wred_profiles);
 188                 if (wred_profile == NULL)
 189                         break;
 190
 191                 TAILQ_REMOVE(&p->soft.tm.h.wred_profiles, wred_profile, node);
 192                 free(wred_profile);
 193         }
 194
 195         /* Remove all shared shapers */
 196         for ( ; ; ) {
 197                 struct tm_shared_shaper *shared_shaper;
 198
 199                 shared_shaper = TAILQ_FIRST(&p->soft.tm.h.shared_shapers);
 200                 if (shared_shaper == NULL)
 201                         break;
 202
 203                 TAILQ_REMOVE(&p->soft.tm.h.shared_shapers, shared_shaper, node);
 204                 free(shared_shaper);
 205         }
 206
 207         /* Remove all shaper profiles */
 208         for ( ; ; ) {
 209                 struct tm_shaper_profile *shaper_profile;
 210
 211                 shaper_profile = TAILQ_FIRST(&p->soft.tm.h.shaper_profiles);
 212                 if (shaper_profile == NULL)
 213                         break;
 214
 215                 TAILQ_REMOVE(&p->soft.tm.h.shaper_profiles,
 216                         shaper_profile, node);
 217                 free(shaper_profile);
 218         }
 219
 220         tm_hierarchy_init(p);
 221 }
 222
 223 static struct tm_shaper_profile *
 224 tm_shaper_profile_search(struct rte_eth_dev *dev, uint32_t shaper_profile_id)
 225 {
 226         struct pmd_internals *p = dev->data->dev_private;
 227         struct tm_shaper_profile_list *spl = &p->soft.tm.h.shaper_profiles;
 228         struct tm_shaper_profile *sp;
 229
 230         TAILQ_FOREACH(sp, spl, node)
 231                 if (shaper_profile_id == sp->shaper_profile_id)
 232                         return sp;
 233
 234         return NULL;
 235 }
 236
 237 static struct tm_shared_shaper *
 238 tm_shared_shaper_search(struct rte_eth_dev *dev, uint32_t shared_shaper_id)
 239 {
 240         struct pmd_internals *p = dev->data->dev_private;
 241         struct tm_shared_shaper_list *ssl = &p->soft.tm.h.shared_shapers;
 242         struct tm_shared_shaper *ss;
 243
 244         TAILQ_FOREACH(ss, ssl, node)
 245                 if (shared_shaper_id == ss->shared_shaper_id)
 246                         return ss;
 247
 248         return NULL;
 249 }
 250
 251 static struct tm_wred_profile *
 252 tm_wred_profile_search(struct rte_eth_dev *dev, uint32_t wred_profile_id)
 253 {
 254         struct pmd_internals *p = dev->data->dev_private;
 255         struct tm_wred_profile_list *wpl = &p->soft.tm.h.wred_profiles;
 256         struct tm_wred_profile *wp;
 257
 258         TAILQ_FOREACH(wp, wpl, node)
 259                 if (wred_profile_id == wp->wred_profile_id)
 260                         return wp;
 261
 262         return NULL;
 263 }
 264
 265 static struct tm_node *
 266 tm_node_search(struct rte_eth_dev *dev, uint32_t node_id)
 267 {
 268         struct pmd_internals *p = dev->data->dev_private;
 269         struct tm_node_list *nl = &p->soft.tm.h.nodes;
 270         struct tm_node *n;
 271
 272         TAILQ_FOREACH(n, nl, node)
 273                 if (n->node_id == node_id)
 274                         return n;
 275
 276         return NULL;
 277 }
 278
 279 static struct tm_node *
 280 tm_root_node_present(struct rte_eth_dev *dev)
 281 {
 282         struct pmd_internals *p = dev->data->dev_private;
 283         struct tm_node_list *nl = &p->soft.tm.h.nodes;
 284         struct tm_node *n;
 285
 286         TAILQ_FOREACH(n, nl, node)
 287                 if (n->parent_node_id == RTE_TM_NODE_ID_NULL)
 288                         return n;
 289
 290         return NULL;
 291 }
 292
 293 static uint32_t
 294 tm_node_subport_id(struct rte_eth_dev *dev, struct tm_node *subport_node)
 295 {
 296         struct pmd_internals *p = dev->data->dev_private;
 297         struct tm_node_list *nl = &p->soft.tm.h.nodes;
 298         struct tm_node *ns;
 299         uint32_t subport_id;
 300
 301         subport_id = 0;
 302         TAILQ_FOREACH(ns, nl, node) {
 303                 if (ns->level != TM_NODE_LEVEL_SUBPORT)
 304                         continue;
 305
 306                 if (ns->node_id == subport_node->node_id)
 307                         return subport_id;
 308
 309                 subport_id++;
 310         }
 311
 312         return UINT32_MAX;
 313 }
 314
 315 static uint32_t
 316 tm_node_pipe_id(struct rte_eth_dev *dev, struct tm_node *pipe_node)
 317 {
 318         struct pmd_internals *p = dev->data->dev_private;
 319         struct tm_node_list *nl = &p->soft.tm.h.nodes;
 320         struct tm_node *np;
 321         uint32_t pipe_id;
 322
 323         pipe_id = 0;
 324         TAILQ_FOREACH(np, nl, node) {
 325                 if (np->level != TM_NODE_LEVEL_PIPE ||
 326                         np->parent_node_id != pipe_node->parent_node_id)
 327                         continue;
 328
 329                 if (np->node_id == pipe_node->node_id)
 330                         return pipe_id;
 331
 332                 pipe_id++;
 333         }
 334
 335         return UINT32_MAX;
 336 }
 337
 338 static uint32_t
 339 tm_node_tc_id(struct rte_eth_dev *dev __rte_unused, struct tm_node *tc_node)
 340 {
 341         return tc_node->priority;
 342 }
 343
 344 static uint32_t
 345 tm_node_queue_id(struct rte_eth_dev *dev, struct tm_node *queue_node)
 346 {
 347         struct pmd_internals *p = dev->data->dev_private;
 348         struct tm_node_list *nl = &p->soft.tm.h.nodes;
 349         struct tm_node *nq;
 350         uint32_t queue_id;
 351
 352         queue_id = 0;
 353         TAILQ_FOREACH(nq, nl, node) {
 354                 if (nq->level != TM_NODE_LEVEL_QUEUE ||
 355                         nq->parent_node_id != queue_node->parent_node_id)
 356                         continue;
 357
 358                 if (nq->node_id == queue_node->node_id)
 359                         return queue_id;
 360
 361                 queue_id++;
 362         }
 363
 364         return UINT32_MAX;
 365 }
 366
 367 static uint32_t
 368 tm_level_get_max_nodes(struct rte_eth_dev *dev, enum tm_node_level level)
 369 {
 370         struct pmd_internals *p = dev->data->dev_private;
 371         uint32_t n_queues_max = p->params.tm.n_queues;
 372         uint32_t n_tc_max =
 373                 (n_queues_max * RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
 374                 / RTE_SCHED_QUEUES_PER_PIPE;
 375         uint32_t n_pipes_max = n_tc_max / RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE;
 376         uint32_t n_subports_max = n_pipes_max;
 377         uint32_t n_root_max = 1;
 378
 379         switch (level) {
 380         case TM_NODE_LEVEL_PORT:
 381                 return n_root_max;
 382         case TM_NODE_LEVEL_SUBPORT:
 383                 return n_subports_max;
 384         case TM_NODE_LEVEL_PIPE:
 385                 return n_pipes_max;
 386         case TM_NODE_LEVEL_TC:
 387                 return n_tc_max;
 388         case TM_NODE_LEVEL_QUEUE:
 389         default:
 390                 return n_queues_max;
 391         }
 392 }
 393
 394 /* Traffic manager node type get */
 395 static int
 396 pmd_tm_node_type_get(struct rte_eth_dev *dev,
 397         uint32_t node_id,
 398         int *is_leaf,
 399         struct rte_tm_error *error)
 400 {
 401         struct pmd_internals *p = dev->data->dev_private;
 402
 403         if (is_leaf == NULL)
 404                 return -rte_tm_error_set(error,
 405                    EINVAL,
 406                    RTE_TM_ERROR_TYPE_UNSPECIFIED,
 407                    NULL,
 408                    rte_strerror(EINVAL));
 409
 410         if (node_id == RTE_TM_NODE_ID_NULL ||
 411                 (tm_node_search(dev, node_id) == NULL))
 412                 return -rte_tm_error_set(error,
 413                    EINVAL,
 414                    RTE_TM_ERROR_TYPE_NODE_ID,
 415                    NULL,
 416                    rte_strerror(EINVAL));
 417
 418         *is_leaf = node_id < p->params.tm.n_queues;
 419
 420         return 0;
 421 }
 422
 423 #ifdef RTE_SCHED_RED
 424 #define WRED_SUPPORTED                                          1
 425 #else
 426 #define WRED_SUPPORTED                                          0
 427 #endif
 428
 429 #define STATS_MASK_DEFAULT                                      \
 430         (RTE_TM_STATS_N_PKTS |                                  \
 431         RTE_TM_STATS_N_BYTES |                                  \
 432         RTE_TM_STATS_N_PKTS_GREEN_DROPPED |                     \
 433         RTE_TM_STATS_N_BYTES_GREEN_DROPPED)
 434
 435 #define STATS_MASK_QUEUE                                                \
 436         (STATS_MASK_DEFAULT |                                   \
 437         RTE_TM_STATS_N_PKTS_QUEUED)
 438
 439 static const struct rte_tm_capabilities tm_cap = {
 440         .n_nodes_max = UINT32_MAX,
 441         .n_levels_max = TM_NODE_LEVEL_MAX,
 442
 443         .non_leaf_nodes_identical = 0,
 444         .leaf_nodes_identical = 1,
 445
 446         .shaper_n_max = UINT32_MAX,
 447         .shaper_private_n_max = UINT32_MAX,
 448         .shaper_private_dual_rate_n_max = 0,
 449         .shaper_private_rate_min = 1,
 450         .shaper_private_rate_max = UINT32_MAX,
 451         .shaper_private_packet_mode_supported = 0,
 452         .shaper_private_byte_mode_supported = 1,
 453
 454         .shaper_shared_n_max = UINT32_MAX,
 455         .shaper_shared_n_nodes_per_shaper_max = UINT32_MAX,
 456         .shaper_shared_n_shapers_per_node_max = 1,
 457         .shaper_shared_dual_rate_n_max = 0,
 458         .shaper_shared_rate_min = 1,
 459         .shaper_shared_rate_max = UINT32_MAX,
 460         .shaper_shared_packet_mode_supported = 0,
 461         .shaper_shared_byte_mode_supported = 1,
 462
 463         .shaper_pkt_length_adjust_min = RTE_TM_ETH_FRAMING_OVERHEAD_FCS,
 464         .shaper_pkt_length_adjust_max = RTE_TM_ETH_FRAMING_OVERHEAD_FCS,
 465
 466         .sched_n_children_max = UINT32_MAX,
 467         .sched_sp_n_priorities_max = RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE,
 468         .sched_wfq_n_children_per_group_max = UINT32_MAX,
 469         .sched_wfq_n_groups_max = 1,
 470         .sched_wfq_weight_max = UINT32_MAX,
 471         .sched_wfq_packet_mode_supported = 0,
 472         .sched_wfq_byte_mode_supported = 1,
 473
 474         .cman_wred_packet_mode_supported = WRED_SUPPORTED,
 475         .cman_wred_byte_mode_supported = 0,
 476         .cman_head_drop_supported = 0,
 477         .cman_wred_context_n_max = 0,
 478         .cman_wred_context_private_n_max = 0,
 479         .cman_wred_context_shared_n_max = 0,
 480         .cman_wred_context_shared_n_nodes_per_context_max = 0,
 481         .cman_wred_context_shared_n_contexts_per_node_max = 0,
 482
 483         .mark_vlan_dei_supported = {0, 0, 0},
 484         .mark_ip_ecn_tcp_supported = {0, 0, 0},
 485         .mark_ip_ecn_sctp_supported = {0, 0, 0},
 486         .mark_ip_dscp_supported = {0, 0, 0},
 487
 488         .dynamic_update_mask = 0,
 489
 490         .stats_mask = STATS_MASK_QUEUE,
 491 };
 492
 493 /* Traffic manager capabilities get */
 494 static int
 495 pmd_tm_capabilities_get(struct rte_eth_dev *dev __rte_unused,
 496         struct rte_tm_capabilities *cap,
 497         struct rte_tm_error *error)
 498 {
 499         if (cap == NULL)
 500                 return -rte_tm_error_set(error,
 501                    EINVAL,
 502                    RTE_TM_ERROR_TYPE_CAPABILITIES,
 503                    NULL,
 504                    rte_strerror(EINVAL));
 505
 506         memcpy(cap, &tm_cap, sizeof(*cap));
 507
 508         cap->n_nodes_max = tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PORT) +
 509                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_SUBPORT) +
 510                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PIPE) +
 511                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_TC) +
 512                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_QUEUE);
 513
 514         cap->shaper_private_n_max =
 515                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PORT) +
 516                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_SUBPORT) +
 517                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PIPE) +
 518                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_TC);
 519
 520         cap->shaper_shared_n_max = RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE *
 521                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_SUBPORT);
 522
 523         cap->shaper_n_max = cap->shaper_private_n_max +
 524                 cap->shaper_shared_n_max;
 525
 526         cap->shaper_shared_n_nodes_per_shaper_max =
 527                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PIPE);
 528
 529         cap->sched_n_children_max = RTE_MAX(
 530                 tm_level_get_max_nodes(dev, TM_NODE_LEVEL_PIPE),
 531                 (uint32_t)RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE);
 532
 533         cap->sched_wfq_n_children_per_group_max = cap->sched_n_children_max;
 534
 535         if (WRED_SUPPORTED)
 536                 cap->cman_wred_context_private_n_max =
 537                         tm_level_get_max_nodes(dev, TM_NODE_LEVEL_QUEUE);
 538
 539         cap->cman_wred_context_n_max = cap->cman_wred_context_private_n_max +
 540                 cap->cman_wred_context_shared_n_max;
 541
 542         return 0;
 543 }
 544
 545 static const struct rte_tm_level_capabilities tm_level_cap[] = {
 546         [TM_NODE_LEVEL_PORT] = {
 547                 .n_nodes_max = 1,
 548                 .n_nodes_nonleaf_max = 1,
 549                 .n_nodes_leaf_max = 0,
 550                 .non_leaf_nodes_identical = 1,
 551                 .leaf_nodes_identical = 0,
 552
 553                 {.nonleaf = {
 554                         .shaper_private_supported = 1,
 555                         .shaper_private_dual_rate_supported = 0,
 556                         .shaper_private_rate_min = 1,
 557                         .shaper_private_rate_max = UINT32_MAX,
 558                         .shaper_private_packet_mode_supported = 0,
 559                         .shaper_private_byte_mode_supported = 1,
 560                         .shaper_shared_n_max = 0,
 561                         .shaper_shared_packet_mode_supported = 0,
 562                         .shaper_shared_byte_mode_supported = 0,
 563
 564                         .sched_n_children_max = UINT32_MAX,
 565                         .sched_sp_n_priorities_max = 1,
 566                         .sched_wfq_n_children_per_group_max = UINT32_MAX,
 567                         .sched_wfq_n_groups_max = 1,
 568                         .sched_wfq_weight_max = 1,
 569                         .sched_wfq_packet_mode_supported = 0,
 570                         .sched_wfq_byte_mode_supported = 1,
 571
 572                         .stats_mask = STATS_MASK_DEFAULT,
 573                 } },
 574         },
 575
 576         [TM_NODE_LEVEL_SUBPORT] = {
 577                 .n_nodes_max = UINT32_MAX,
 578                 .n_nodes_nonleaf_max = UINT32_MAX,
 579                 .n_nodes_leaf_max = 0,
 580                 .non_leaf_nodes_identical = 1,
 581                 .leaf_nodes_identical = 0,
 582
 583                 {.nonleaf = {
 584                         .shaper_private_supported = 1,
 585                         .shaper_private_dual_rate_supported = 0,
 586                         .shaper_private_rate_min = 1,
 587                         .shaper_private_rate_max = UINT32_MAX,
 588                         .shaper_private_packet_mode_supported = 0,
 589                         .shaper_private_byte_mode_supported = 1,
 590                         .shaper_shared_n_max = 0,
 591                         .shaper_shared_packet_mode_supported = 0,
 592                         .shaper_shared_byte_mode_supported = 0,
 593
 594                         .sched_n_children_max = UINT32_MAX,
 595                         .sched_sp_n_priorities_max = 1,
 596                         .sched_wfq_n_children_per_group_max = UINT32_MAX,
 597                         .sched_wfq_n_groups_max = 1,
 598 #ifdef RTE_SCHED_SUBPORT_TC_OV
 599                         .sched_wfq_weight_max = UINT32_MAX,
 600                         .sched_wfq_packet_mode_supported = 0,
 601                         .sched_wfq_byte_mode_supported = 1,
 602 #else
 603                         .sched_wfq_weight_max = 1,
 604                         .sched_wfq_packet_mode_supported = 0,
 605                         .sched_wfq_byte_mode_supported = 1,
 606 #endif
 607
 608                         .stats_mask = STATS_MASK_DEFAULT,
 609                 } },
 610         },
 611
 612         [TM_NODE_LEVEL_PIPE] = {
 613                 .n_nodes_max = UINT32_MAX,
 614                 .n_nodes_nonleaf_max = UINT32_MAX,
 615                 .n_nodes_leaf_max = 0,
 616                 .non_leaf_nodes_identical = 1,
 617                 .leaf_nodes_identical = 0,
 618
 619                 {.nonleaf = {
 620                         .shaper_private_supported = 1,
 621                         .shaper_private_dual_rate_supported = 0,
 622                         .shaper_private_rate_min = 1,
 623                         .shaper_private_rate_max = UINT32_MAX,
 624                         .shaper_private_packet_mode_supported = 0,
 625                         .shaper_private_byte_mode_supported = 1,
 626                         .shaper_shared_n_max = 0,
 627                         .shaper_shared_packet_mode_supported = 0,
 628                         .shaper_shared_byte_mode_supported = 0,
 629
 630                         .sched_n_children_max =
 631                                 RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE,
 632                         .sched_sp_n_priorities_max =
 633                                 RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE,
 634                         .sched_wfq_n_children_per_group_max = 1,
 635                         .sched_wfq_n_groups_max = 0,
 636                         .sched_wfq_weight_max = 1,
 637                         .sched_wfq_packet_mode_supported = 0,
 638                         .sched_wfq_byte_mode_supported = 0,
 639
 640                         .stats_mask = STATS_MASK_DEFAULT,
 641                 } },
 642         },
 643
 644         [TM_NODE_LEVEL_TC] = {
 645                 .n_nodes_max = UINT32_MAX,
 646                 .n_nodes_nonleaf_max = UINT32_MAX,
 647                 .n_nodes_leaf_max = 0,
 648                 .non_leaf_nodes_identical = 1,
 649                 .leaf_nodes_identical = 0,
 650
 651                 {.nonleaf = {
 652                         .shaper_private_supported = 1,
 653                         .shaper_private_dual_rate_supported = 0,
 654                         .shaper_private_rate_min = 1,
 655                         .shaper_private_rate_max = UINT32_MAX,
 656                         .shaper_private_packet_mode_supported = 0,
 657                         .shaper_private_byte_mode_supported = 1,
 658                         .shaper_shared_n_max = 1,
 659                         .shaper_shared_packet_mode_supported = 0,
 660                         .shaper_shared_byte_mode_supported = 1,
 661
 662                         .sched_n_children_max =
 663                                 RTE_SCHED_BE_QUEUES_PER_PIPE,
 664                         .sched_sp_n_priorities_max = 1,
 665                         .sched_wfq_n_children_per_group_max =
 666                                 RTE_SCHED_BE_QUEUES_PER_PIPE,
 667                         .sched_wfq_n_groups_max = 1,
 668                         .sched_wfq_weight_max = UINT32_MAX,
 669                         .sched_wfq_packet_mode_supported = 0,
 670                         .sched_wfq_byte_mode_supported = 1,
 671
 672                         .stats_mask = STATS_MASK_DEFAULT,
 673                 } },
 674         },
 675
 676         [TM_NODE_LEVEL_QUEUE] = {
 677                 .n_nodes_max = UINT32_MAX,
 678                 .n_nodes_nonleaf_max = 0,
 679                 .n_nodes_leaf_max = UINT32_MAX,
 680                 .non_leaf_nodes_identical = 0,
 681                 .leaf_nodes_identical = 1,
 682
 683                 {.leaf = {
 684                         .shaper_private_supported = 0,
 685                         .shaper_private_dual_rate_supported = 0,
 686                         .shaper_private_rate_min = 0,
 687                         .shaper_private_rate_max = 0,
 688                         .shaper_private_packet_mode_supported = 0,
 689                         .shaper_private_byte_mode_supported = 0,
 690                         .shaper_shared_n_max = 0,
 691                         .shaper_shared_packet_mode_supported = 0,
 692                         .shaper_shared_byte_mode_supported = 0,
 693
 694                         .cman_head_drop_supported = 0,
 695                         .cman_wred_packet_mode_supported = WRED_SUPPORTED,
 696                         .cman_wred_byte_mode_supported = 0,
 697                         .cman_wred_context_private_supported = WRED_SUPPORTED,
 698                         .cman_wred_context_shared_n_max = 0,
 699
 700                         .stats_mask = STATS_MASK_QUEUE,
 701                 } },
 702         },
 703 };
 704
 705 /* Traffic manager level capabilities get */
 706 static int
 707 pmd_tm_level_capabilities_get(struct rte_eth_dev *dev __rte_unused,
 708         uint32_t level_id,
 709         struct rte_tm_level_capabilities *cap,
 710         struct rte_tm_error *error)
 711 {
 712         if (cap == NULL)
 713                 return -rte_tm_error_set(error,
 714                    EINVAL,
 715                    RTE_TM_ERROR_TYPE_CAPABILITIES,
 716                    NULL,
 717                    rte_strerror(EINVAL));
 718
 719         if (level_id >= TM_NODE_LEVEL_MAX)
 720                 return -rte_tm_error_set(error,
 721                    EINVAL,
 722                    RTE_TM_ERROR_TYPE_LEVEL_ID,
 723                    NULL,
 724                    rte_strerror(EINVAL));
 725
 726         memcpy(cap, &tm_level_cap[level_id], sizeof(*cap));
 727
 728         switch (level_id) {
 729         case TM_NODE_LEVEL_PORT:
 730                 cap->nonleaf.sched_n_children_max =
 731                         tm_level_get_max_nodes(dev,
 732                                 TM_NODE_LEVEL_SUBPORT);
 733                 cap->nonleaf.sched_wfq_n_children_per_group_max =
 734                         cap->nonleaf.sched_n_children_max;
 735                 break;
 736
 737         case TM_NODE_LEVEL_SUBPORT:
 738                 cap->n_nodes_max = tm_level_get_max_nodes(dev,
 739                         TM_NODE_LEVEL_SUBPORT);
 740                 cap->n_nodes_nonleaf_max = cap->n_nodes_max;
 741                 cap->nonleaf.sched_n_children_max =
 742                         tm_level_get_max_nodes(dev,
 743                                 TM_NODE_LEVEL_PIPE);
 744                 cap->nonleaf.sched_wfq_n_children_per_group_max =
 745                         cap->nonleaf.sched_n_children_max;
 746                 break;
 747
 748         case TM_NODE_LEVEL_PIPE:
 749                 cap->n_nodes_max = tm_level_get_max_nodes(dev,
 750                         TM_NODE_LEVEL_PIPE);
 751                 cap->n_nodes_nonleaf_max = cap->n_nodes_max;
 752                 break;
 753
 754         case TM_NODE_LEVEL_TC:
 755                 cap->n_nodes_max = tm_level_get_max_nodes(dev,
 756                         TM_NODE_LEVEL_TC);
 757                 cap->n_nodes_nonleaf_max = cap->n_nodes_max;
 758                 break;
 759
 760         case TM_NODE_LEVEL_QUEUE:
 761         default:
 762                 cap->n_nodes_max = tm_level_get_max_nodes(dev,
 763                         TM_NODE_LEVEL_QUEUE);
 764                 cap->n_nodes_leaf_max = cap->n_nodes_max;
 765                 break;
 766         }
 767
 768         return 0;
 769 }
 770
 771 static const struct rte_tm_node_capabilities tm_node_cap[] = {
 772         [TM_NODE_LEVEL_PORT] = {
 773                 .shaper_private_supported = 1,
 774                 .shaper_private_dual_rate_supported = 0,
 775                 .shaper_private_rate_min = 1,
 776                 .shaper_private_rate_max = UINT32_MAX,
 777                 .shaper_private_packet_mode_supported = 0,
 778                 .shaper_private_byte_mode_supported = 1,
 779                 .shaper_shared_n_max = 0,
 780                 .shaper_shared_packet_mode_supported = 0,
 781                 .shaper_shared_byte_mode_supported = 0,
 782
 783                 {.nonleaf = {
 784                         .sched_n_children_max = UINT32_MAX,
 785                         .sched_sp_n_priorities_max = 1,
 786                         .sched_wfq_n_children_per_group_max = UINT32_MAX,
 787                         .sched_wfq_n_groups_max = 1,
 788                         .sched_wfq_weight_max = 1,
 789                         .sched_wfq_packet_mode_supported = 0,
 790                         .sched_wfq_byte_mode_supported = 1,
 791                 } },
 792
 793                 .stats_mask = STATS_MASK_DEFAULT,
 794         },
 795
 796         [TM_NODE_LEVEL_SUBPORT] = {
 797                 .shaper_private_supported = 1,
 798                 .shaper_private_dual_rate_supported = 0,
 799                 .shaper_private_rate_min = 1,
 800                 .shaper_private_rate_max = UINT32_MAX,
 801                 .shaper_private_packet_mode_supported = 0,
 802                 .shaper_private_byte_mode_supported = 1,
 803                 .shaper_shared_n_max = 0,
 804                 .shaper_shared_packet_mode_supported = 0,
 805                 .shaper_shared_byte_mode_supported = 0,
 806
 807                 {.nonleaf = {
 808                         .sched_n_children_max = UINT32_MAX,
 809                         .sched_sp_n_priorities_max = 1,
 810                         .sched_wfq_n_children_per_group_max = UINT32_MAX,
 811                         .sched_wfq_n_groups_max = 1,
 812                         .sched_wfq_weight_max = UINT32_MAX,
 813                         .sched_wfq_packet_mode_supported = 0,
 814                         .sched_wfq_byte_mode_supported = 1,
 815                 } },
 816
 817                 .stats_mask = STATS_MASK_DEFAULT,
 818         },
 819
 820         [TM_NODE_LEVEL_PIPE] = {
 821                 .shaper_private_supported = 1,
 822                 .shaper_private_dual_rate_supported = 0,
 823                 .shaper_private_rate_min = 1,
 824                 .shaper_private_rate_max = UINT32_MAX,
 825                 .shaper_private_packet_mode_supported = 0,
 826                 .shaper_private_byte_mode_supported = 1,
 827                 .shaper_shared_n_max = 0,
 828                 .shaper_shared_packet_mode_supported = 0,
 829                 .shaper_shared_byte_mode_supported = 0,
 830
 831                 {.nonleaf = {
 832                         .sched_n_children_max =
 833                                 RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE,
 834                         .sched_sp_n_priorities_max =
 835                                 RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE,
 836                         .sched_wfq_n_children_per_group_max = 1,
 837                         .sched_wfq_n_groups_max = 0,
 838                         .sched_wfq_weight_max = 1,
 839                         .sched_wfq_packet_mode_supported = 0,
 840                         .sched_wfq_byte_mode_supported = 0,
 841                 } },
 842
 843                 .stats_mask = STATS_MASK_DEFAULT,
 844         },
 845
 846         [TM_NODE_LEVEL_TC] = {
 847                 .shaper_private_supported = 1,
 848                 .shaper_private_dual_rate_supported = 0,
 849                 .shaper_private_rate_min = 1,
 850                 .shaper_private_rate_max = UINT32_MAX,
 851                 .shaper_private_packet_mode_supported = 0,
 852                 .shaper_private_byte_mode_supported = 1,
 853                 .shaper_shared_n_max = 1,
 854                 .shaper_shared_packet_mode_supported = 0,
 855                 .shaper_shared_byte_mode_supported = 1,
 856
 857                 {.nonleaf = {
 858                         .sched_n_children_max =
 859                                 RTE_SCHED_BE_QUEUES_PER_PIPE,
 860                         .sched_sp_n_priorities_max = 1,
 861                         .sched_wfq_n_children_per_group_max =
 862                                 RTE_SCHED_BE_QUEUES_PER_PIPE,
 863                         .sched_wfq_n_groups_max = 1,
 864                         .sched_wfq_weight_max = UINT32_MAX,
 865                         .sched_wfq_packet_mode_supported = 0,
 866                         .sched_wfq_byte_mode_supported = 1,
 867                 } },
 868
 869                 .stats_mask = STATS_MASK_DEFAULT,
 870         },
 871
 872         [TM_NODE_LEVEL_QUEUE] = {
 873                 .shaper_private_supported = 0,
 874                 .shaper_private_dual_rate_supported = 0,
 875                 .shaper_private_rate_min = 0,
 876                 .shaper_private_rate_max = 0,
 877                 .shaper_private_packet_mode_supported = 0,
 878                 .shaper_private_byte_mode_supported = 0,
 879                 .shaper_shared_n_max = 0,
 880                 .shaper_shared_packet_mode_supported = 0,
 881                 .shaper_shared_byte_mode_supported = 0,
 882
 883
 884                 {.leaf = {
 885                         .cman_head_drop_supported = 0,
 886                         .cman_wred_packet_mode_supported = WRED_SUPPORTED,
 887                         .cman_wred_byte_mode_supported = 0,
 888                         .cman_wred_context_private_supported = WRED_SUPPORTED,
 889                         .cman_wred_context_shared_n_max = 0,
 890                 } },
 891
 892                 .stats_mask = STATS_MASK_QUEUE,
 893         },
 894 };
 895
 896 /* Traffic manager node capabilities get */
 897 static int
 898 pmd_tm_node_capabilities_get(struct rte_eth_dev *dev __rte_unused,
 899         uint32_t node_id,
 900         struct rte_tm_node_capabilities *cap,
 901         struct rte_tm_error *error)
 902 {
 903         struct tm_node *tm_node;
 904
 905         if (cap == NULL)
 906                 return -rte_tm_error_set(error,
 907                    EINVAL,
 908                    RTE_TM_ERROR_TYPE_CAPABILITIES,
 909                    NULL,
 910                    rte_strerror(EINVAL));
 911
 912         tm_node = tm_node_search(dev, node_id);
 913         if (tm_node == NULL)
 914                 return -rte_tm_error_set(error,
 915                    EINVAL,
 916                    RTE_TM_ERROR_TYPE_NODE_ID,
 917                    NULL,
 918                    rte_strerror(EINVAL));
 919
 920         memcpy(cap, &tm_node_cap[tm_node->level], sizeof(*cap));
 921
 922         switch (tm_node->level) {
 923         case TM_NODE_LEVEL_PORT:
 924                 cap->nonleaf.sched_n_children_max =
 925                         tm_level_get_max_nodes(dev,
 926                                 TM_NODE_LEVEL_SUBPORT);
 927                 cap->nonleaf.sched_wfq_n_children_per_group_max =
 928                         cap->nonleaf.sched_n_children_max;
 929                 break;
 930
 931         case TM_NODE_LEVEL_SUBPORT:
 932                 cap->nonleaf.sched_n_children_max =
 933                         tm_level_get_max_nodes(dev,
 934                                 TM_NODE_LEVEL_PIPE);
 935                 cap->nonleaf.sched_wfq_n_children_per_group_max =
 936                         cap->nonleaf.sched_n_children_max;
 937                 break;
 938
 939         case TM_NODE_LEVEL_PIPE:
 940         case TM_NODE_LEVEL_TC:
 941         case TM_NODE_LEVEL_QUEUE:
 942         default:
 943                 break;
 944         }
 945
 946         return 0;
 947 }
 948
 949 static int
 950 shaper_profile_check(struct rte_eth_dev *dev,
 951         uint32_t shaper_profile_id,
 952         struct rte_tm_shaper_params *profile,
 953         struct rte_tm_error *error)
 954 {
 955         struct tm_shaper_profile *sp;
 956
 957         /* Shaper profile ID must not be NONE. */
 958         if (shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE)
 959                 return -rte_tm_error_set(error,
 960                         EINVAL,
 961                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
 962                         NULL,
 963                         rte_strerror(EINVAL));
 964
 965         /* Shaper profile must not exist. */
 966         sp = tm_shaper_profile_search(dev, shaper_profile_id);
 967         if (sp)
 968                 return -rte_tm_error_set(error,
 969                         EEXIST,
 970                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
 971                         NULL,
 972                         rte_strerror(EEXIST));
 973
 974         /* Profile must not be NULL. */
 975         if (profile == NULL)
 976                 return -rte_tm_error_set(error,
 977                         EINVAL,
 978                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE,
 979                         NULL,
 980                         rte_strerror(EINVAL));
 981
 982         /* Peak rate: non-zero, 32-bit */
 983         if (profile->peak.rate == 0 ||
 984                 profile->peak.rate >= UINT32_MAX)
 985                 return -rte_tm_error_set(error,
 986                         EINVAL,
 987                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_RATE,
 988                         NULL,
 989                         rte_strerror(EINVAL));
 990
 991         /* Peak size: non-zero, 32-bit */
 992         if (profile->peak.size == 0 ||
 993                 profile->peak.size >= UINT32_MAX)
 994                 return -rte_tm_error_set(error,
 995                         EINVAL,
 996                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PEAK_SIZE,
 997                         NULL,
 998                         rte_strerror(EINVAL));
 999
1000         /* Dual-rate profiles are not supported. */
1001         if (profile->committed.rate != 0)
1002                 return -rte_tm_error_set(error,
1003                         EINVAL,
1004                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_COMMITTED_RATE,
1005                         NULL,
1006                         rte_strerror(EINVAL));
1007
1008         /* Packet length adjust: 24 bytes */
1009         if (profile->pkt_length_adjust != RTE_TM_ETH_FRAMING_OVERHEAD_FCS)
1010                 return -rte_tm_error_set(error,
1011                         EINVAL,
1012                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PKT_ADJUST_LEN,
1013                         NULL,
1014                         rte_strerror(EINVAL));
1015
1016         /* Packet mode is not supported. */
1017         if (profile->packet_mode != 0)
1018                 return -rte_tm_error_set(error,
1019                         EINVAL,
1020                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_PACKET_MODE,
1021                         NULL,
1022                         rte_strerror(EINVAL));
1023         return 0;
1024 }
1025
1026 /* Traffic manager shaper profile add */
1027 static int
1028 pmd_tm_shaper_profile_add(struct rte_eth_dev *dev,
1029         uint32_t shaper_profile_id,
1030         struct rte_tm_shaper_params *profile,
1031         struct rte_tm_error *error)
1032 {
1033         struct pmd_internals *p = dev->data->dev_private;
1034         struct tm_shaper_profile_list *spl = &p->soft.tm.h.shaper_profiles;
1035         struct tm_shaper_profile *sp;
1036         int status;
1037
1038         /* Check input params */
1039         status = shaper_profile_check(dev, shaper_profile_id, profile, error);
1040         if (status)
1041                 return status;
1042
1043         /* Memory allocation */
1044         sp = calloc(1, sizeof(struct tm_shaper_profile));
1045         if (sp == NULL)
1046                 return -rte_tm_error_set(error,
1047                         ENOMEM,
1048                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
1049                         NULL,
1050                         rte_strerror(ENOMEM));
1051
1052         /* Fill in */
1053         sp->shaper_profile_id = shaper_profile_id;
1054         memcpy(&sp->params, profile, sizeof(sp->params));
1055
1056         /* Add to list */
1057         TAILQ_INSERT_TAIL(spl, sp, node);
1058         p->soft.tm.h.n_shaper_profiles++;
1059
1060         return 0;
1061 }
1062
1063 /* Traffic manager shaper profile delete */
1064 static int
1065 pmd_tm_shaper_profile_delete(struct rte_eth_dev *dev,
1066         uint32_t shaper_profile_id,
1067         struct rte_tm_error *error)
1068 {
1069         struct pmd_internals *p = dev->data->dev_private;
1070         struct tm_shaper_profile *sp;
1071
1072         /* Check existing */
1073         sp = tm_shaper_profile_search(dev, shaper_profile_id);
1074         if (sp == NULL)
1075                 return -rte_tm_error_set(error,
1076                         EINVAL,
1077                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
1078                         NULL,
1079                         rte_strerror(EINVAL));
1080
1081         /* Check unused */
1082         if (sp->n_users)
1083                 return -rte_tm_error_set(error,
1084                         EBUSY,
1085                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
1086                         NULL,
1087                         rte_strerror(EBUSY));
1088
1089         /* Remove from list */
1090         TAILQ_REMOVE(&p->soft.tm.h.shaper_profiles, sp, node);
1091         p->soft.tm.h.n_shaper_profiles--;
1092         free(sp);
1093
1094         return 0;
1095 }
1096
1097 static struct tm_node *
1098 tm_shared_shaper_get_tc(struct rte_eth_dev *dev,
1099         struct tm_shared_shaper *ss)
1100 {
1101         struct pmd_internals *p = dev->data->dev_private;
1102         struct tm_node_list *nl = &p->soft.tm.h.nodes;
1103         struct tm_node *n;
1104
1105         /* Subport: each TC uses shared shaper  */
1106         TAILQ_FOREACH(n, nl, node) {
1107                 if (n->level != TM_NODE_LEVEL_TC ||
1108                         n->params.n_shared_shapers == 0 ||
1109                         n->params.shared_shaper_id[0] != ss->shared_shaper_id)
1110                         continue;
1111
1112                 return n;
1113         }
1114
1115         return NULL;
1116 }
1117
1118 static int
1119 subport_profile_exists(struct rte_eth_dev *dev,
1120         struct rte_sched_subport_profile_params *sp,
1121         uint32_t *subport_profile_id)
1122 {
1123         struct pmd_internals *p = dev->data->dev_private;
1124         struct tm_params *t = &p->soft.tm.params;
1125         uint32_t i;
1126
1127         for (i = 0; i < t->n_subport_profiles; i++)
1128                 if (memcmp(&t->subport_profile[i], sp, sizeof(*sp)) == 0) {
1129                         if (subport_profile_id)
1130                                 *subport_profile_id = i;
1131                         return 1;
1132                 }
1133
1134         return 0;
1135 }
1136
1137 static int
1138 update_subport_tc_rate(struct rte_eth_dev *dev,
1139         struct tm_node *nt,
1140         struct tm_shared_shaper *ss,
1141         struct tm_shaper_profile *sp_new)
1142 {
1143         struct rte_sched_subport_profile_params subport_profile;
1144         struct pmd_internals *p = dev->data->dev_private;
1145         uint32_t tc_id = tm_node_tc_id(dev, nt);
1146         struct tm_node *np = nt->parent_node;
1147         struct tm_node *ns = np->parent_node;
1148         uint32_t subport_id = tm_node_subport_id(dev, ns);
1149         struct tm_params *t = &p->soft.tm.params;
1150         uint32_t subport_profile_id = t->subport_to_profile[subport_id];
1151         struct tm_shaper_profile *sp_old = tm_shaper_profile_search(dev,
1152                 ss->shaper_profile_id);
1153
1154         /* Derive new subport configuration. */
1155         memcpy(&subport_profile,
1156                 &p->soft.tm.params.subport_profile[subport_profile_id],
1157                 sizeof(subport_profile));
1158         subport_profile.tc_rate[tc_id] = sp_new->params.peak.rate;
1159
1160         /* Update the subport configuration. */
1161         if (rte_sched_subport_config(SCHED(p),
1162                 subport_id, NULL, subport_profile_id))
1163                 return -1;
1164
1165         /* Commit changes. */
1166         sp_old->n_users--;
1167
1168         ss->shaper_profile_id = sp_new->shaper_profile_id;
1169         sp_new->n_users++;
1170
1171         memcpy(&p->soft.tm.params.subport_profile[subport_profile_id],
1172                 &subport_profile,
1173                 sizeof(subport_profile));
1174
1175         return 0;
1176 }
1177
1178 /* Traffic manager shared shaper add/update */
1179 static int
1180 pmd_tm_shared_shaper_add_update(struct rte_eth_dev *dev,
1181         uint32_t shared_shaper_id,
1182         uint32_t shaper_profile_id,
1183         struct rte_tm_error *error)
1184 {
1185         struct pmd_internals *p = dev->data->dev_private;
1186         struct tm_shared_shaper *ss;
1187         struct tm_shaper_profile *sp;
1188         struct tm_node *nt;
1189
1190         /* Shaper profile must be valid. */
1191         sp = tm_shaper_profile_search(dev, shaper_profile_id);
1192         if (sp == NULL)
1193                 return -rte_tm_error_set(error,
1194                         EINVAL,
1195                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE_ID,
1196                         NULL,
1197                         rte_strerror(EINVAL));
1198
1199         /**
1200          * Add new shared shaper
1201          */
1202         ss = tm_shared_shaper_search(dev, shared_shaper_id);
1203         if (ss == NULL) {
1204                 struct tm_shared_shaper_list *ssl =
1205                         &p->soft.tm.h.shared_shapers;
1206
1207                 /* Hierarchy must not be frozen */
1208                 if (p->soft.tm.hierarchy_frozen)
1209                         return -rte_tm_error_set(error,
1210                                 EBUSY,
1211                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
1212                                 NULL,
1213                                 rte_strerror(EBUSY));
1214
1215                 /* Memory allocation */
1216                 ss = calloc(1, sizeof(struct tm_shared_shaper));
1217                 if (ss == NULL)
1218                         return -rte_tm_error_set(error,
1219                                 ENOMEM,
1220                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
1221                                 NULL,
1222                                 rte_strerror(ENOMEM));
1223
1224                 /* Fill in */
1225                 ss->shared_shaper_id = shared_shaper_id;
1226                 ss->shaper_profile_id = shaper_profile_id;
1227
1228                 /* Add to list */
1229                 TAILQ_INSERT_TAIL(ssl, ss, node);
1230                 p->soft.tm.h.n_shared_shapers++;
1231
1232                 return 0;
1233         }
1234
1235         /**
1236          * Update existing shared shaper
1237          */
1238         /* Hierarchy must be frozen (run-time update) */
1239         if (p->soft.tm.hierarchy_frozen == 0)
1240                 return -rte_tm_error_set(error,
1241                         EBUSY,
1242                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
1243                         NULL,
1244                         rte_strerror(EBUSY));
1245
1246
1247         /* Propagate change. */
1248         nt = tm_shared_shaper_get_tc(dev, ss);
1249         if (update_subport_tc_rate(dev, nt, ss, sp))
1250                 return -rte_tm_error_set(error,
1251                         EINVAL,
1252                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
1253                         NULL,
1254                         rte_strerror(EINVAL));
1255
1256         return 0;
1257 }
1258
1259 /* Traffic manager shared shaper delete */
1260 static int
1261 pmd_tm_shared_shaper_delete(struct rte_eth_dev *dev,
1262         uint32_t shared_shaper_id,
1263         struct rte_tm_error *error)
1264 {
1265         struct pmd_internals *p = dev->data->dev_private;
1266         struct tm_shared_shaper *ss;
1267
1268         /* Check existing */
1269         ss = tm_shared_shaper_search(dev, shared_shaper_id);
1270         if (ss == NULL)
1271                 return -rte_tm_error_set(error,
1272                         EINVAL,
1273                         RTE_TM_ERROR_TYPE_SHARED_SHAPER_ID,
1274                         NULL,
1275                         rte_strerror(EINVAL));
1276
1277         /* Check unused */
1278         if (ss->n_users)
1279                 return -rte_tm_error_set(error,
1280                         EBUSY,
1281                         RTE_TM_ERROR_TYPE_SHARED_SHAPER_ID,
1282                         NULL,
1283                         rte_strerror(EBUSY));
1284
1285         /* Remove from list */
1286         TAILQ_REMOVE(&p->soft.tm.h.shared_shapers, ss, node);
1287         p->soft.tm.h.n_shared_shapers--;
1288         free(ss);
1289
1290         return 0;
1291 }
1292
1293 static int
1294 wred_profile_check(struct rte_eth_dev *dev,
1295         uint32_t wred_profile_id,
1296         struct rte_tm_wred_params *profile,
1297         struct rte_tm_error *error)
1298 {
1299         struct tm_wred_profile *wp;
1300         enum rte_color color;
1301
1302         /* WRED profile ID must not be NONE. */
1303         if (wred_profile_id == RTE_TM_WRED_PROFILE_ID_NONE)
1304                 return -rte_tm_error_set(error,
1305                         EINVAL,
1306                         RTE_TM_ERROR_TYPE_WRED_PROFILE_ID,
1307                         NULL,
1308                         rte_strerror(EINVAL));
1309
1310         /* WRED profile must not exist. */
1311         wp = tm_wred_profile_search(dev, wred_profile_id);
1312         if (wp)
1313                 return -rte_tm_error_set(error,
1314                         EEXIST,
1315                         RTE_TM_ERROR_TYPE_WRED_PROFILE_ID,
1316                         NULL,
1317                         rte_strerror(EEXIST));
1318
1319         /* Profile must not be NULL. */
1320         if (profile == NULL)
1321                 return -rte_tm_error_set(error,
1322                         EINVAL,
1323                         RTE_TM_ERROR_TYPE_WRED_PROFILE,
1324                         NULL,
1325                         rte_strerror(EINVAL));
1326
1327         /* WRED profile should be in packet mode */
1328         if (profile->packet_mode == 0)
1329                 return -rte_tm_error_set(error,
1330                         ENOTSUP,
1331                         RTE_TM_ERROR_TYPE_WRED_PROFILE,
1332                         NULL,
1333                         rte_strerror(ENOTSUP));
1334
1335         /* min_th <= max_th, max_th > 0  */
1336         for (color = RTE_COLOR_GREEN; color < RTE_COLORS; color++) {
1337                 uint32_t min_th = profile->red_params[color].min_th;
1338                 uint32_t max_th = profile->red_params[color].max_th;
1339
1340                 if (min_th > max_th ||
1341                         max_th == 0 ||
1342                         min_th > UINT16_MAX ||
1343                         max_th > UINT16_MAX)
1344                         return -rte_tm_error_set(error,
1345                                 EINVAL,
1346                                 RTE_TM_ERROR_TYPE_WRED_PROFILE,
1347                                 NULL,
1348                                 rte_strerror(EINVAL));
1349         }
1350
1351         return 0;
1352 }
1353
1354 /* Traffic manager WRED profile add */
1355 static int
1356 pmd_tm_wred_profile_add(struct rte_eth_dev *dev,
1357         uint32_t wred_profile_id,
1358         struct rte_tm_wred_params *profile,
1359         struct rte_tm_error *error)
1360 {
1361         struct pmd_internals *p = dev->data->dev_private;
1362         struct tm_wred_profile_list *wpl = &p->soft.tm.h.wred_profiles;
1363         struct tm_wred_profile *wp;
1364         int status;
1365
1366         /* Check input params */
1367         status = wred_profile_check(dev, wred_profile_id, profile, error);
1368         if (status)
1369                 return status;
1370
1371         /* Memory allocation */
1372         wp = calloc(1, sizeof(struct tm_wred_profile));
1373         if (wp == NULL)
1374                 return -rte_tm_error_set(error,
1375                         ENOMEM,
1376                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
1377                         NULL,
1378                         rte_strerror(ENOMEM));
1379
1380         /* Fill in */
1381         wp->wred_profile_id = wred_profile_id;
1382         memcpy(&wp->params, profile, sizeof(wp->params));
1383
1384         /* Add to list */
1385         TAILQ_INSERT_TAIL(wpl, wp, node);
1386         p->soft.tm.h.n_wred_profiles++;
1387
1388         return 0;
1389 }
1390
1391 /* Traffic manager WRED profile delete */
1392 static int
1393 pmd_tm_wred_profile_delete(struct rte_eth_dev *dev,
1394         uint32_t wred_profile_id,
1395         struct rte_tm_error *error)
1396 {
1397         struct pmd_internals *p = dev->data->dev_private;
1398         struct tm_wred_profile *wp;
1399
1400         /* Check existing */
1401         wp = tm_wred_profile_search(dev, wred_profile_id);
1402         if (wp == NULL)
1403                 return -rte_tm_error_set(error,
1404                         EINVAL,
1405                         RTE_TM_ERROR_TYPE_WRED_PROFILE_ID,
1406                         NULL,
1407                         rte_strerror(EINVAL));
1408
1409         /* Check unused */
1410         if (wp->n_users)
1411                 return -rte_tm_error_set(error,
1412                         EBUSY,
1413                         RTE_TM_ERROR_TYPE_WRED_PROFILE_ID,
1414                         NULL,
1415                         rte_strerror(EBUSY));
1416
1417         /* Remove from list */
1418         TAILQ_REMOVE(&p->soft.tm.h.wred_profiles, wp, node);
1419         p->soft.tm.h.n_wred_profiles--;
1420         free(wp);
1421
1422         return 0;
1423 }
1424
1425 static int
1426 node_add_check_port(struct rte_eth_dev *dev,
1427         uint32_t node_id,
1428         uint32_t parent_node_id __rte_unused,
1429         uint32_t priority,
1430         uint32_t weight,
1431         uint32_t level_id __rte_unused,
1432         struct rte_tm_node_params *params,
1433         struct rte_tm_error *error)
1434 {
1435         struct pmd_internals *p = dev->data->dev_private;
1436         struct tm_shaper_profile *sp = tm_shaper_profile_search(dev,
1437                 params->shaper_profile_id);
1438
1439         /* node type: non-leaf */
1440         if (node_id < p->params.tm.n_queues)
1441                 return -rte_tm_error_set(error,
1442                         EINVAL,
1443                         RTE_TM_ERROR_TYPE_NODE_ID,
1444                         NULL,
1445                         rte_strerror(EINVAL));
1446
1447         /* Priority must be 0 */
1448         if (priority != 0)
1449                 return -rte_tm_error_set(error,
1450                         EINVAL,
1451                         RTE_TM_ERROR_TYPE_NODE_PRIORITY,
1452                         NULL,
1453                         rte_strerror(EINVAL));
1454
1455         /* Weight must be 1 */
1456         if (weight != 1)
1457                 return -rte_tm_error_set(error,
1458                         EINVAL,
1459                         RTE_TM_ERROR_TYPE_NODE_WEIGHT,
1460                         NULL,
1461                         rte_strerror(EINVAL));
1462
1463         /* Shaper must be valid */
1464         if (params->shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE ||
1465                 sp == NULL)
1466                 return -rte_tm_error_set(error,
1467                         EINVAL,
1468                         RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID,
1469                         NULL,
1470                         rte_strerror(EINVAL));
1471
1472         /* No shared shapers */
1473         if (params->n_shared_shapers != 0)
1474                 return -rte_tm_error_set(error,
1475                         EINVAL,
1476                         RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS,
1477                         NULL,
1478                         rte_strerror(EINVAL));
1479
1480         /* Number of SP priorities must be 1 */
1481         if (params->nonleaf.n_sp_priorities != 1)
1482                 return -rte_tm_error_set(error,
1483                         EINVAL,
1484                         RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES,
1485                         NULL,
1486                         rte_strerror(EINVAL));
1487
1488         /* Stats */
1489         if (params->stats_mask & ~STATS_MASK_DEFAULT)
1490                 return -rte_tm_error_set(error,
1491                         EINVAL,
1492                         RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS,
1493                         NULL,
1494                         rte_strerror(EINVAL));
1495
1496         return 0;
1497 }
1498
1499 static int
1500 node_add_check_subport(struct rte_eth_dev *dev,
1501         uint32_t node_id,
1502         uint32_t parent_node_id __rte_unused,
1503         uint32_t priority,
1504         uint32_t weight,
1505         uint32_t level_id __rte_unused,
1506         struct rte_tm_node_params *params,
1507         struct rte_tm_error *error)
1508 {
1509         struct pmd_internals *p = dev->data->dev_private;
1510
1511         /* node type: non-leaf */
1512         if (node_id < p->params.tm.n_queues)
1513                 return -rte_tm_error_set(error,
1514                         EINVAL,
1515                         RTE_TM_ERROR_TYPE_NODE_ID,
1516                         NULL,
1517                         rte_strerror(EINVAL));
1518
1519         /* Priority must be 0 */
1520         if (priority != 0)
1521                 return -rte_tm_error_set(error,
1522                         EINVAL,
1523                         RTE_TM_ERROR_TYPE_NODE_PRIORITY,
1524                         NULL,
1525                         rte_strerror(EINVAL));
1526
1527         /* Weight must be 1 */
1528         if (weight != 1)
1529                 return -rte_tm_error_set(error,
1530                         EINVAL,
1531                         RTE_TM_ERROR_TYPE_NODE_WEIGHT,
1532                         NULL,
1533                         rte_strerror(EINVAL));
1534
1535         /* Shaper must be valid */
1536         if (params->shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE ||
1537                 (!tm_shaper_profile_search(dev, params->shaper_profile_id)))
1538                 return -rte_tm_error_set(error,
1539                         EINVAL,
1540                         RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID,
1541                         NULL,
1542                         rte_strerror(EINVAL));
1543
1544         /* No shared shapers */
1545         if (params->n_shared_shapers != 0)
1546                 return -rte_tm_error_set(error,
1547                         EINVAL,
1548                         RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS,
1549                         NULL,
1550                         rte_strerror(EINVAL));
1551
1552         /* Number of SP priorities must be 1 */
1553         if (params->nonleaf.n_sp_priorities != 1)
1554                 return -rte_tm_error_set(error,
1555                         EINVAL,
1556                         RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES,
1557                         NULL,
1558                         rte_strerror(EINVAL));
1559
1560         /* Stats */
1561         if (params->stats_mask & ~STATS_MASK_DEFAULT)
1562                 return -rte_tm_error_set(error,
1563                         EINVAL,
1564                         RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS,
1565                         NULL,
1566                         rte_strerror(EINVAL));
1567
1568         return 0;
1569 }
1570
1571 static int
1572 node_add_check_pipe(struct rte_eth_dev *dev,
1573         uint32_t node_id,
1574         uint32_t parent_node_id __rte_unused,
1575         uint32_t priority,
1576         uint32_t weight __rte_unused,
1577         uint32_t level_id __rte_unused,
1578         struct rte_tm_node_params *params,
1579         struct rte_tm_error *error)
1580 {
1581         struct pmd_internals *p = dev->data->dev_private;
1582
1583         /* node type: non-leaf */
1584         if (node_id < p->params.tm.n_queues)
1585                 return -rte_tm_error_set(error,
1586                         EINVAL,
1587                         RTE_TM_ERROR_TYPE_NODE_ID,
1588                         NULL,
1589                         rte_strerror(EINVAL));
1590
1591         /* Priority must be 0 */
1592         if (priority != 0)
1593                 return -rte_tm_error_set(error,
1594                         EINVAL,
1595                         RTE_TM_ERROR_TYPE_NODE_PRIORITY,
1596                         NULL,
1597                         rte_strerror(EINVAL));
1598
1599         /* Shaper must be valid */
1600         if (params->shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE ||
1601                 (!tm_shaper_profile_search(dev, params->shaper_profile_id)))
1602                 return -rte_tm_error_set(error,
1603                         EINVAL,
1604                         RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID,
1605                         NULL,
1606                         rte_strerror(EINVAL));
1607
1608         /* No shared shapers */
1609         if (params->n_shared_shapers != 0)
1610                 return -rte_tm_error_set(error,
1611                         EINVAL,
1612                         RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS,
1613                         NULL,
1614                         rte_strerror(EINVAL));
1615
1616         /* Number of SP priorities must be 4 */
1617         if (params->nonleaf.n_sp_priorities !=
1618                 RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
1619                 return -rte_tm_error_set(error,
1620                         EINVAL,
1621                         RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES,
1622                         NULL,
1623                         rte_strerror(EINVAL));
1624
1625         /* WFQ mode must be byte mode */
1626         if (params->nonleaf.wfq_weight_mode != NULL &&
1627                 params->nonleaf.wfq_weight_mode[0] != 0 &&
1628                 params->nonleaf.wfq_weight_mode[1] != 0 &&
1629                 params->nonleaf.wfq_weight_mode[2] != 0 &&
1630                 params->nonleaf.wfq_weight_mode[3] != 0)
1631                 return -rte_tm_error_set(error,
1632                         EINVAL,
1633                         RTE_TM_ERROR_TYPE_NODE_PARAMS_WFQ_WEIGHT_MODE,
1634                         NULL,
1635                         rte_strerror(EINVAL));
1636
1637         /* Stats */
1638         if (params->stats_mask & ~STATS_MASK_DEFAULT)
1639                 return -rte_tm_error_set(error,
1640                         EINVAL,
1641                         RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS,
1642                         NULL,
1643                         rte_strerror(EINVAL));
1644
1645         return 0;
1646 }
1647
1648 static int
1649 node_add_check_tc(struct rte_eth_dev *dev,
1650         uint32_t node_id,
1651         uint32_t parent_node_id __rte_unused,
1652         uint32_t priority __rte_unused,
1653         uint32_t weight,
1654         uint32_t level_id __rte_unused,
1655         struct rte_tm_node_params *params,
1656         struct rte_tm_error *error)
1657 {
1658         struct pmd_internals *p = dev->data->dev_private;
1659
1660         /* node type: non-leaf */
1661         if (node_id < p->params.tm.n_queues)
1662                 return -rte_tm_error_set(error,
1663                         EINVAL,
1664                         RTE_TM_ERROR_TYPE_NODE_ID,
1665                         NULL,
1666                         rte_strerror(EINVAL));
1667
1668         /* Weight must be 1 */
1669         if (weight != 1)
1670                 return -rte_tm_error_set(error,
1671                         EINVAL,
1672                         RTE_TM_ERROR_TYPE_NODE_WEIGHT,
1673                         NULL,
1674                         rte_strerror(EINVAL));
1675
1676         /* Shaper must be valid */
1677         if (params->shaper_profile_id == RTE_TM_SHAPER_PROFILE_ID_NONE ||
1678                 (!tm_shaper_profile_search(dev, params->shaper_profile_id)))
1679                 return -rte_tm_error_set(error,
1680                         EINVAL,
1681                         RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID,
1682                         NULL,
1683                         rte_strerror(EINVAL));
1684
1685         /* Single valid shared shaper */
1686         if (params->n_shared_shapers > 1)
1687                 return -rte_tm_error_set(error,
1688                         EINVAL,
1689                         RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS,
1690                         NULL,
1691                         rte_strerror(EINVAL));
1692
1693         if (params->n_shared_shapers == 1 &&
1694                 (params->shared_shaper_id == NULL ||
1695                 (!tm_shared_shaper_search(dev, params->shared_shaper_id[0]))))
1696                 return -rte_tm_error_set(error,
1697                         EINVAL,
1698                         RTE_TM_ERROR_TYPE_NODE_PARAMS_SHARED_SHAPER_ID,
1699                         NULL,
1700                         rte_strerror(EINVAL));
1701
1702         /* Number of priorities must be 1 */
1703         if (params->nonleaf.n_sp_priorities != 1)
1704                 return -rte_tm_error_set(error,
1705                         EINVAL,
1706                         RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SP_PRIORITIES,
1707                         NULL,
1708                         rte_strerror(EINVAL));
1709
1710         /* Stats */
1711         if (params->stats_mask & ~STATS_MASK_DEFAULT)
1712                 return -rte_tm_error_set(error,
1713                         EINVAL,
1714                         RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS,
1715                         NULL,
1716                         rte_strerror(EINVAL));
1717
1718         return 0;
1719 }
1720
1721 static int
1722 node_add_check_queue(struct rte_eth_dev *dev,
1723         uint32_t node_id,
1724         uint32_t parent_node_id __rte_unused,
1725         uint32_t priority,
1726         uint32_t weight __rte_unused,
1727         uint32_t level_id __rte_unused,
1728         struct rte_tm_node_params *params,
1729         struct rte_tm_error *error)
1730 {
1731         struct pmd_internals *p = dev->data->dev_private;
1732
1733         /* node type: leaf */
1734         if (node_id >= p->params.tm.n_queues)
1735                 return -rte_tm_error_set(error,
1736                         EINVAL,
1737                         RTE_TM_ERROR_TYPE_NODE_ID,
1738                         NULL,
1739                         rte_strerror(EINVAL));
1740
1741         /* Priority must be 0 */
1742         if (priority != 0)
1743                 return -rte_tm_error_set(error,
1744                         EINVAL,
1745                         RTE_TM_ERROR_TYPE_NODE_PRIORITY,
1746                         NULL,
1747                         rte_strerror(EINVAL));
1748
1749         /* No shaper */
1750         if (params->shaper_profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE)
1751                 return -rte_tm_error_set(error,
1752                         EINVAL,
1753                         RTE_TM_ERROR_TYPE_NODE_PARAMS_SHAPER_PROFILE_ID,
1754                         NULL,
1755                         rte_strerror(EINVAL));
1756
1757         /* No shared shapers */
1758         if (params->n_shared_shapers != 0)
1759                 return -rte_tm_error_set(error,
1760                         EINVAL,
1761                         RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_SHAPERS,
1762                         NULL,
1763                         rte_strerror(EINVAL));
1764
1765         /* Congestion management must not be head drop */
1766         if (params->leaf.cman == RTE_TM_CMAN_HEAD_DROP)
1767                 return -rte_tm_error_set(error,
1768                         EINVAL,
1769                         RTE_TM_ERROR_TYPE_NODE_PARAMS_CMAN,
1770                         NULL,
1771                         rte_strerror(EINVAL));
1772
1773         /* Congestion management set to WRED */
1774         if (params->leaf.cman == RTE_TM_CMAN_WRED) {
1775                 uint32_t wred_profile_id = params->leaf.wred.wred_profile_id;
1776                 struct tm_wred_profile *wp = tm_wred_profile_search(dev,
1777                         wred_profile_id);
1778
1779                 /* WRED profile (for private WRED context) must be valid */
1780                 if (wred_profile_id == RTE_TM_WRED_PROFILE_ID_NONE ||
1781                         wp == NULL)
1782                         return -rte_tm_error_set(error,
1783                                 EINVAL,
1784                                 RTE_TM_ERROR_TYPE_NODE_PARAMS_WRED_PROFILE_ID,
1785                                 NULL,
1786                                 rte_strerror(EINVAL));
1787
1788                 /* No shared WRED contexts */
1789                 if (params->leaf.wred.n_shared_wred_contexts != 0)
1790                         return -rte_tm_error_set(error,
1791                                 EINVAL,
1792                                 RTE_TM_ERROR_TYPE_NODE_PARAMS_N_SHARED_WRED_CONTEXTS,
1793                                 NULL,
1794                                 rte_strerror(EINVAL));
1795         }
1796
1797         /* Stats */
1798         if (params->stats_mask & ~STATS_MASK_QUEUE)
1799                 return -rte_tm_error_set(error,
1800                         EINVAL,
1801                         RTE_TM_ERROR_TYPE_NODE_PARAMS_STATS,
1802                         NULL,
1803                         rte_strerror(EINVAL));
1804
1805         return 0;
1806 }
1807
1808 static int
1809 node_add_check(struct rte_eth_dev *dev,
1810         uint32_t node_id,
1811         uint32_t parent_node_id,
1812         uint32_t priority,
1813         uint32_t weight,
1814         uint32_t level_id,
1815         struct rte_tm_node_params *params,
1816         struct rte_tm_error *error)
1817 {
1818         struct tm_node *pn;
1819         uint32_t level;
1820         int status;
1821
1822         /* node_id, parent_node_id:
1823          *    -node_id must not be RTE_TM_NODE_ID_NULL
1824          *    -node_id must not be in use
1825          *    -root node add (parent_node_id is RTE_TM_NODE_ID_NULL):
1826          *        -root node must not exist
1827          *    -non-root node add (parent_node_id is not RTE_TM_NODE_ID_NULL):
1828          *        -parent_node_id must be valid
1829          */
1830         if (node_id == RTE_TM_NODE_ID_NULL)
1831                 return -rte_tm_error_set(error,
1832                         EINVAL,
1833                         RTE_TM_ERROR_TYPE_NODE_ID,
1834                         NULL,
1835                         rte_strerror(EINVAL));
1836
1837         if (tm_node_search(dev, node_id))
1838                 return -rte_tm_error_set(error,
1839                         EEXIST,
1840                         RTE_TM_ERROR_TYPE_NODE_ID,
1841                         NULL,
1842                         rte_strerror(EEXIST));
1843
1844         if (parent_node_id == RTE_TM_NODE_ID_NULL) {
1845                 pn = NULL;
1846                 if (tm_root_node_present(dev))
1847                         return -rte_tm_error_set(error,
1848                                 EEXIST,
1849                                 RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID,
1850                                 NULL,
1851                                 rte_strerror(EEXIST));
1852         } else {
1853                 pn = tm_node_search(dev, parent_node_id);
1854                 if (pn == NULL)
1855                         return -rte_tm_error_set(error,
1856                                 EINVAL,
1857                                 RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID,
1858                                 NULL,
1859                                 rte_strerror(EINVAL));
1860         }
1861
1862         /* priority: must be 0 .. 3 */
1863         if (priority >= RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
1864                 return -rte_tm_error_set(error,
1865                         EINVAL,
1866                         RTE_TM_ERROR_TYPE_NODE_PRIORITY,
1867                         NULL,
1868                         rte_strerror(EINVAL));
1869
1870         /* weight: must be 1 .. 255 */
1871         if (weight == 0 || weight >= UINT8_MAX)
1872                 return -rte_tm_error_set(error,
1873                         EINVAL,
1874                         RTE_TM_ERROR_TYPE_NODE_WEIGHT,
1875                         NULL,
1876                         rte_strerror(EINVAL));
1877
1878         /* level_id: if valid, then
1879          *    -root node add (parent_node_id is RTE_TM_NODE_ID_NULL):
1880          *        -level_id must be zero
1881          *    -non-root node add (parent_node_id is not RTE_TM_NODE_ID_NULL):
1882          *        -level_id must be parent level ID plus one
1883          */
1884         level = (pn == NULL) ? 0 : pn->level + 1;
1885         if (level_id != RTE_TM_NODE_LEVEL_ID_ANY && level_id != level)
1886                 return -rte_tm_error_set(error,
1887                         EINVAL,
1888                         RTE_TM_ERROR_TYPE_LEVEL_ID,
1889                         NULL,
1890                         rte_strerror(EINVAL));
1891
1892         /* params: must not be NULL */
1893         if (params == NULL)
1894                 return -rte_tm_error_set(error,
1895                         EINVAL,
1896                         RTE_TM_ERROR_TYPE_NODE_PARAMS,
1897                         NULL,
1898                         rte_strerror(EINVAL));
1899
1900         /* params: per level checks */
1901         switch (level) {
1902         case TM_NODE_LEVEL_PORT:
1903                 status = node_add_check_port(dev, node_id,
1904                         parent_node_id, priority, weight, level_id,
1905                         params, error);
1906                 if (status)
1907                         return status;
1908                 break;
1909
1910         case TM_NODE_LEVEL_SUBPORT:
1911                 status = node_add_check_subport(dev, node_id,
1912                         parent_node_id, priority, weight, level_id,
1913                         params, error);
1914                 if (status)
1915                         return status;
1916                 break;
1917
1918         case TM_NODE_LEVEL_PIPE:
1919                 status = node_add_check_pipe(dev, node_id,
1920                         parent_node_id, priority, weight, level_id,
1921                         params, error);
1922                 if (status)
1923                         return status;
1924                 break;
1925
1926         case TM_NODE_LEVEL_TC:
1927                 status = node_add_check_tc(dev, node_id,
1928                         parent_node_id, priority, weight, level_id,
1929                         params, error);
1930                 if (status)
1931                         return status;
1932                 break;
1933
1934         case TM_NODE_LEVEL_QUEUE:
1935                 status = node_add_check_queue(dev, node_id,
1936                         parent_node_id, priority, weight, level_id,
1937                         params, error);
1938                 if (status)
1939                         return status;
1940                 break;
1941
1942         default:
1943                 return -rte_tm_error_set(error,
1944                         EINVAL,
1945                         RTE_TM_ERROR_TYPE_LEVEL_ID,
1946                         NULL,
1947                         rte_strerror(EINVAL));
1948         }
1949
1950         return 0;
1951 }
1952
1953 /* Traffic manager node add */
1954 static int
1955 pmd_tm_node_add(struct rte_eth_dev *dev,
1956         uint32_t node_id,
1957         uint32_t parent_node_id,
1958         uint32_t priority,
1959         uint32_t weight,
1960         uint32_t level_id,
1961         struct rte_tm_node_params *params,
1962         struct rte_tm_error *error)
1963 {
1964         struct pmd_internals *p = dev->data->dev_private;
1965         struct tm_node_list *nl = &p->soft.tm.h.nodes;
1966         struct tm_node *n;
1967         uint32_t i;
1968         int status;
1969
1970         /* Checks */
1971         if (p->soft.tm.hierarchy_frozen)
1972                 return -rte_tm_error_set(error,
1973                         EBUSY,
1974                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
1975                         NULL,
1976                         rte_strerror(EBUSY));
1977
1978         status = node_add_check(dev, node_id, parent_node_id, priority, weight,
1979                 level_id, params, error);
1980         if (status)
1981                 return status;
1982
1983         /* Memory allocation */
1984         n = calloc(1, sizeof(struct tm_node));
1985         if (n == NULL)
1986                 return -rte_tm_error_set(error,
1987                         ENOMEM,
1988                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
1989                         NULL,
1990                         rte_strerror(ENOMEM));
1991
1992         /* Fill in */
1993         n->node_id = node_id;
1994         n->parent_node_id = parent_node_id;
1995         n->priority = priority;
1996         n->weight = weight;
1997
1998         if (parent_node_id != RTE_TM_NODE_ID_NULL) {
1999                 n->parent_node = tm_node_search(dev, parent_node_id);
2000                 n->level = n->parent_node->level + 1;
2001         }
2002
2003         if (params->shaper_profile_id != RTE_TM_SHAPER_PROFILE_ID_NONE)
2004                 n->shaper_profile = tm_shaper_profile_search(dev,
2005                         params->shaper_profile_id);
2006
2007         if (n->level == TM_NODE_LEVEL_QUEUE &&
2008                 params->leaf.cman == RTE_TM_CMAN_WRED)
2009                 n->wred_profile = tm_wred_profile_search(dev,
2010                         params->leaf.wred.wred_profile_id);
2011
2012         memcpy(&n->params, params, sizeof(n->params));
2013
2014         /* Add to list */
2015         TAILQ_INSERT_TAIL(nl, n, node);
2016         p->soft.tm.h.n_nodes++;
2017
2018         /* Update dependencies */
2019         if (n->parent_node)
2020                 n->parent_node->n_children++;
2021
2022         if (n->shaper_profile)
2023                 n->shaper_profile->n_users++;
2024
2025         for (i = 0; i < params->n_shared_shapers; i++) {
2026                 struct tm_shared_shaper *ss;
2027
2028                 ss = tm_shared_shaper_search(dev, params->shared_shaper_id[i]);
2029                 ss->n_users++;
2030         }
2031
2032         if (n->wred_profile)
2033                 n->wred_profile->n_users++;
2034
2035         p->soft.tm.h.n_tm_nodes[n->level]++;
2036
2037         return 0;
2038 }
2039
2040 /* Traffic manager node delete */
2041 static int
2042 pmd_tm_node_delete(struct rte_eth_dev *dev,
2043         uint32_t node_id,
2044         struct rte_tm_error *error)
2045 {
2046         struct pmd_internals *p = dev->data->dev_private;
2047         struct tm_node *n;
2048         uint32_t i;
2049
2050         /* Check hierarchy changes are currently allowed */
2051         if (p->soft.tm.hierarchy_frozen)
2052                 return -rte_tm_error_set(error,
2053                         EBUSY,
2054                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2055                         NULL,
2056                         rte_strerror(EBUSY));
2057
2058         /* Check existing */
2059         n = tm_node_search(dev, node_id);
2060         if (n == NULL)
2061                 return -rte_tm_error_set(error,
2062                         EINVAL,
2063                         RTE_TM_ERROR_TYPE_NODE_ID,
2064                         NULL,
2065                         rte_strerror(EINVAL));
2066
2067         /* Check unused */
2068         if (n->n_children)
2069                 return -rte_tm_error_set(error,
2070                         EBUSY,
2071                         RTE_TM_ERROR_TYPE_NODE_ID,
2072                         NULL,
2073                         rte_strerror(EBUSY));
2074
2075         /* Update dependencies */
2076         p->soft.tm.h.n_tm_nodes[n->level]--;
2077
2078         if (n->wred_profile)
2079                 n->wred_profile->n_users--;
2080
2081         for (i = 0; i < n->params.n_shared_shapers; i++) {
2082                 struct tm_shared_shaper *ss;
2083
2084                 ss = tm_shared_shaper_search(dev,
2085                                 n->params.shared_shaper_id[i]);
2086                 ss->n_users--;
2087         }
2088
2089         if (n->shaper_profile)
2090                 n->shaper_profile->n_users--;
2091
2092         if (n->parent_node)
2093                 n->parent_node->n_children--;
2094
2095         /* Remove from list */
2096         TAILQ_REMOVE(&p->soft.tm.h.nodes, n, node);
2097         p->soft.tm.h.n_nodes--;
2098         free(n);
2099
2100         return 0;
2101 }
2102
2103
2104 static void
2105 pipe_profile_build(struct rte_eth_dev *dev,
2106         struct tm_node *np,
2107         struct rte_sched_pipe_params *pp)
2108 {
2109         struct pmd_internals *p = dev->data->dev_private;
2110         struct tm_hierarchy *h = &p->soft.tm.h;
2111         struct tm_node_list *nl = &h->nodes;
2112         struct tm_node *nt, *nq;
2113
2114         memset(pp, 0, sizeof(*pp));
2115
2116         /* Pipe */
2117         pp->tb_rate = np->shaper_profile->params.peak.rate;
2118         pp->tb_size = np->shaper_profile->params.peak.size;
2119
2120         /* Traffic Class (TC) */
2121         pp->tc_period = PIPE_TC_PERIOD;
2122
2123         pp->tc_ov_weight = np->weight;
2124
2125         TAILQ_FOREACH(nt, nl, node) {
2126                 uint32_t queue_id = 0;
2127
2128                 if (nt->level != TM_NODE_LEVEL_TC ||
2129                         nt->parent_node_id != np->node_id)
2130                         continue;
2131
2132                 pp->tc_rate[nt->priority] =
2133                         nt->shaper_profile->params.peak.rate;
2134
2135                 /* Queue */
2136                 TAILQ_FOREACH(nq, nl, node) {
2137
2138                         if (nq->level != TM_NODE_LEVEL_QUEUE ||
2139                                 nq->parent_node_id != nt->node_id)
2140                                 continue;
2141
2142                         if (nt->priority == RTE_SCHED_TRAFFIC_CLASS_BE)
2143                                 pp->wrr_weights[queue_id] = nq->weight;
2144
2145                         queue_id++;
2146                 }
2147         }
2148 }
2149
2150 static int
2151 pipe_profile_free_exists(struct rte_eth_dev *dev,
2152         uint32_t *pipe_profile_id)
2153 {
2154         struct pmd_internals *p = dev->data->dev_private;
2155         struct tm_params *t = &p->soft.tm.params;
2156
2157         if (t->n_pipe_profiles < TM_MAX_PIPE_PROFILE) {
2158                 *pipe_profile_id = t->n_pipe_profiles;
2159                 return 1;
2160         }
2161
2162         return 0;
2163 }
2164
2165 static int
2166 pipe_profile_exists(struct rte_eth_dev *dev,
2167         struct rte_sched_pipe_params *pp,
2168         uint32_t *pipe_profile_id)
2169 {
2170         struct pmd_internals *p = dev->data->dev_private;
2171         struct tm_params *t = &p->soft.tm.params;
2172         uint32_t i;
2173
2174         for (i = 0; i < t->n_pipe_profiles; i++)
2175                 if (memcmp(&t->pipe_profiles[i], pp, sizeof(*pp)) == 0) {
2176                         if (pipe_profile_id)
2177                                 *pipe_profile_id = i;
2178                         return 1;
2179                 }
2180
2181         return 0;
2182 }
2183
2184 static void
2185 pipe_profile_install(struct rte_eth_dev *dev,
2186         struct rte_sched_pipe_params *pp,
2187         uint32_t pipe_profile_id)
2188 {
2189         struct pmd_internals *p = dev->data->dev_private;
2190         struct tm_params *t = &p->soft.tm.params;
2191
2192         memcpy(&t->pipe_profiles[pipe_profile_id], pp, sizeof(*pp));
2193         t->n_pipe_profiles++;
2194 }
2195
2196 static void
2197 pipe_profile_mark(struct rte_eth_dev *dev,
2198         uint32_t subport_id,
2199         uint32_t pipe_id,
2200         uint32_t pipe_profile_id)
2201 {
2202         struct pmd_internals *p = dev->data->dev_private;
2203         struct tm_hierarchy *h = &p->soft.tm.h;
2204         struct tm_params *t = &p->soft.tm.params;
2205         uint32_t n_pipes_per_subport, pos;
2206
2207         n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] /
2208                 h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT];
2209         pos = subport_id * n_pipes_per_subport + pipe_id;
2210
2211         t->pipe_to_profile[pos] = pipe_profile_id;
2212 }
2213
2214 static struct rte_sched_pipe_params *
2215 pipe_profile_get(struct rte_eth_dev *dev, struct tm_node *np)
2216 {
2217         struct pmd_internals *p = dev->data->dev_private;
2218         struct tm_hierarchy *h = &p->soft.tm.h;
2219         struct tm_params *t = &p->soft.tm.params;
2220         uint32_t n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] /
2221                 h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT];
2222
2223         uint32_t subport_id = tm_node_subport_id(dev, np->parent_node);
2224         uint32_t pipe_id = tm_node_pipe_id(dev, np);
2225
2226         uint32_t pos = subport_id * n_pipes_per_subport + pipe_id;
2227         uint32_t pipe_profile_id = t->pipe_to_profile[pos];
2228
2229         return &t->pipe_profiles[pipe_profile_id];
2230 }
2231
2232 static int
2233 pipe_profiles_generate(struct rte_eth_dev *dev)
2234 {
2235         struct pmd_internals *p = dev->data->dev_private;
2236         struct tm_hierarchy *h = &p->soft.tm.h;
2237         struct tm_node_list *nl = &h->nodes;
2238         struct tm_node *ns, *np;
2239         uint32_t subport_id;
2240
2241         /* Objective: Fill in the following fields in struct tm_params:
2242          *    - pipe_profiles
2243          *    - n_pipe_profiles
2244          *    - pipe_to_profile
2245          */
2246
2247         subport_id = 0;
2248         TAILQ_FOREACH(ns, nl, node) {
2249                 uint32_t pipe_id;
2250
2251                 if (ns->level != TM_NODE_LEVEL_SUBPORT)
2252                         continue;
2253
2254                 pipe_id = 0;
2255                 TAILQ_FOREACH(np, nl, node) {
2256                         struct rte_sched_pipe_params pp;
2257                         uint32_t pos;
2258
2259                         memset(&pp, 0, sizeof(pp));
2260
2261                         if (np->level != TM_NODE_LEVEL_PIPE ||
2262                                 np->parent_node_id != ns->node_id)
2263                                 continue;
2264
2265                         pipe_profile_build(dev, np, &pp);
2266
2267                         if (!pipe_profile_exists(dev, &pp, &pos)) {
2268                                 if (!pipe_profile_free_exists(dev, &pos))
2269                                         return -1;
2270
2271                                 pipe_profile_install(dev, &pp, pos);
2272                         }
2273
2274                         pipe_profile_mark(dev, subport_id, pipe_id, pos);
2275
2276                         pipe_id++;
2277                 }
2278
2279                 subport_id++;
2280         }
2281
2282         return 0;
2283 }
2284
2285 static struct tm_wred_profile *
2286 tm_tc_wred_profile_get(struct rte_eth_dev *dev, uint32_t tc_id)
2287 {
2288         struct pmd_internals *p = dev->data->dev_private;
2289         struct tm_hierarchy *h = &p->soft.tm.h;
2290         struct tm_node_list *nl = &h->nodes;
2291         struct tm_node *nq;
2292
2293         TAILQ_FOREACH(nq, nl, node) {
2294                 if (nq->level != TM_NODE_LEVEL_QUEUE ||
2295                         nq->parent_node->priority != tc_id)
2296                         continue;
2297
2298                 return nq->wred_profile;
2299         }
2300
2301         return NULL;
2302 }
2303
2304 #ifdef RTE_SCHED_RED
2305
2306 static void
2307 wred_profiles_set(struct rte_eth_dev *dev, uint32_t subport_id)
2308 {
2309         struct pmd_internals *p = dev->data->dev_private;
2310         struct rte_sched_subport_params *pp =
2311                 &p->soft.tm.params.subport_params[subport_id];
2312
2313         uint32_t tc_id;
2314         enum rte_color color;
2315
2316         for (tc_id = 0; tc_id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc_id++)
2317                 for (color = RTE_COLOR_GREEN; color < RTE_COLORS; color++) {
2318                         struct rte_red_params *dst =
2319                                 &pp->red_params[tc_id][color];
2320                         struct tm_wred_profile *src_wp =
2321                                 tm_tc_wred_profile_get(dev, tc_id);
2322                         struct rte_tm_red_params *src =
2323                                 &src_wp->params.red_params[color];
2324
2325                         memcpy(dst, src, sizeof(*dst));
2326                 }
2327 }
2328
2329 #else
2330
2331 #define wred_profiles_set(dev, subport_id)
2332
2333 #endif
2334
2335 static struct tm_shared_shaper *
2336 tm_tc_shared_shaper_get(struct rte_eth_dev *dev, struct tm_node *tc_node)
2337 {
2338         return (tc_node->params.n_shared_shapers) ?
2339                 tm_shared_shaper_search(dev,
2340                         tc_node->params.shared_shaper_id[0]) :
2341                 NULL;
2342 }
2343
2344 static struct tm_shared_shaper *
2345 tm_subport_tc_shared_shaper_get(struct rte_eth_dev *dev,
2346         struct tm_node *subport_node,
2347         uint32_t tc_id)
2348 {
2349         struct pmd_internals *p = dev->data->dev_private;
2350         struct tm_node_list *nl = &p->soft.tm.h.nodes;
2351         struct tm_node *n;
2352
2353         TAILQ_FOREACH(n, nl, node) {
2354                 if (n->level != TM_NODE_LEVEL_TC ||
2355                         n->parent_node->parent_node_id !=
2356                                 subport_node->node_id ||
2357                         n->priority != tc_id)
2358                         continue;
2359
2360                 return tm_tc_shared_shaper_get(dev, n);
2361         }
2362
2363         return NULL;
2364 }
2365
2366 static struct rte_sched_subport_profile_params *
2367 subport_profile_get(struct rte_eth_dev *dev, struct tm_node *np)
2368 {
2369         struct pmd_internals *p = dev->data->dev_private;
2370         struct tm_params *t = &p->soft.tm.params;
2371         uint32_t subport_id = tm_node_subport_id(dev, np->parent_node);
2372
2373         return &t->subport_profile[subport_id];
2374 }
2375
2376 static void
2377 subport_profile_mark(struct rte_eth_dev *dev,
2378         uint32_t subport_id,
2379         uint32_t subport_profile_id)
2380 {
2381         struct pmd_internals *p = dev->data->dev_private;
2382         struct tm_params *t = &p->soft.tm.params;
2383
2384         t->subport_to_profile[subport_id] = subport_profile_id;
2385 }
2386
2387 static void
2388 subport_profile_install(struct rte_eth_dev *dev,
2389         struct rte_sched_subport_profile_params *sp,
2390         uint32_t subport_profile_id)
2391 {
2392         struct pmd_internals *p = dev->data->dev_private;
2393         struct tm_params *t = &p->soft.tm.params;
2394
2395         memcpy(&t->subport_profile[subport_profile_id],
2396                 sp, sizeof(*sp));
2397         t->n_subport_profiles++;
2398 }
2399
2400 static int
2401 subport_profile_free_exists(struct rte_eth_dev *dev,
2402         uint32_t *subport_profile_id)
2403 {
2404         struct pmd_internals *p = dev->data->dev_private;
2405         struct tm_params *t = &p->soft.tm.params;
2406
2407         if (t->n_subport_profiles < TM_MAX_SUBPORT_PROFILE) {
2408                 *subport_profile_id = t->n_subport_profiles;
2409                 return 1;
2410         }
2411
2412         return 0;
2413 }
2414
2415 static void
2416 subport_profile_build(struct rte_eth_dev *dev, struct tm_node *np,
2417         struct rte_sched_subport_profile_params *sp)
2418 {
2419         uint32_t i;
2420         memset(sp, 0, sizeof(*sp));
2421
2422         sp->tb_rate = np->shaper_profile->params.peak.rate;
2423         sp->tb_size = np->shaper_profile->params.peak.size;
2424
2425         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
2426                 struct tm_shared_shaper *ss;
2427                 struct tm_shaper_profile *ssp;
2428
2429                 ss = tm_subport_tc_shared_shaper_get(dev, np, i);
2430                 ssp = (ss) ? tm_shaper_profile_search(dev,
2431                         ss->shaper_profile_id) :
2432                         np->shaper_profile;
2433                 sp->tc_rate[i] = ssp->params.peak.rate;
2434         }
2435
2436         /* Traffic Class (TC) */
2437         sp->tc_period = SUBPORT_TC_PERIOD;
2438 }
2439
2440 static int
2441 subport_profiles_generate(struct rte_eth_dev *dev)
2442 {
2443         struct pmd_internals *p = dev->data->dev_private;
2444         struct tm_hierarchy *h = &p->soft.tm.h;
2445         struct tm_node_list *nl = &h->nodes;
2446         struct tm_node *ns;
2447         uint32_t subport_id;
2448
2449         /* Objective: Fill in the following fields in struct tm_params:
2450          *    - subport_profiles
2451          *    - n_subport_profiles
2452          *    - subport_to_profile
2453          */
2454
2455         subport_id = 0;
2456         TAILQ_FOREACH(ns, nl, node) {
2457                 if (ns->level != TM_NODE_LEVEL_SUBPORT)
2458                         continue;
2459
2460                 struct rte_sched_subport_profile_params sp;
2461                 uint32_t pos;
2462
2463                 memset(&sp, 0, sizeof(sp));
2464
2465                 subport_profile_build(dev, ns, &sp);
2466
2467                 if (!subport_profile_exists(dev, &sp, &pos)) {
2468                         if (!subport_profile_free_exists(dev, &pos))
2469                                 return -1;
2470
2471                         subport_profile_install(dev, &sp, pos);
2472                 }
2473
2474                 subport_profile_mark(dev, subport_id, pos);
2475
2476                 subport_id++;
2477         }
2478
2479         return 0;
2480 }
2481
2482
2483 static int
2484 hierarchy_commit_check(struct rte_eth_dev *dev, struct rte_tm_error *error)
2485 {
2486         struct pmd_internals *p = dev->data->dev_private;
2487         struct tm_hierarchy *h = &p->soft.tm.h;
2488         struct tm_node_list *nl = &h->nodes;
2489         struct tm_shared_shaper_list *ssl = &h->shared_shapers;
2490         struct tm_wred_profile_list *wpl = &h->wred_profiles;
2491         struct tm_node *nr = tm_root_node_present(dev), *ns, *np, *nt, *nq;
2492         struct tm_shared_shaper *ss;
2493
2494         uint32_t n_pipes_per_subport;
2495
2496         /* Root node exists. */
2497         if (nr == NULL)
2498                 return -rte_tm_error_set(error,
2499                         EINVAL,
2500                         RTE_TM_ERROR_TYPE_LEVEL_ID,
2501                         NULL,
2502                         rte_strerror(EINVAL));
2503
2504         /* There is at least one subport, max is not exceeded. */
2505         if (nr->n_children == 0 || nr->n_children > TM_MAX_SUBPORTS)
2506                 return -rte_tm_error_set(error,
2507                         EINVAL,
2508                         RTE_TM_ERROR_TYPE_LEVEL_ID,
2509                         NULL,
2510                         rte_strerror(EINVAL));
2511
2512         /* There is at least one pipe. */
2513         if (h->n_tm_nodes[TM_NODE_LEVEL_PIPE] == 0)
2514                 return -rte_tm_error_set(error,
2515                         EINVAL,
2516                         RTE_TM_ERROR_TYPE_LEVEL_ID,
2517                         NULL,
2518                         rte_strerror(EINVAL));
2519
2520         /* Number of pipes is the same for all subports. Maximum number of pipes
2521          * per subport is not exceeded.
2522          */
2523         n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] /
2524                 h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT];
2525
2526         if (n_pipes_per_subport > TM_MAX_PIPES_PER_SUBPORT)
2527                 return -rte_tm_error_set(error,
2528                         EINVAL,
2529                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2530                         NULL,
2531                         rte_strerror(EINVAL));
2532
2533         TAILQ_FOREACH(ns, nl, node) {
2534                 if (ns->level != TM_NODE_LEVEL_SUBPORT)
2535                         continue;
2536
2537                 if (ns->n_children != n_pipes_per_subport)
2538                         return -rte_tm_error_set(error,
2539                                 EINVAL,
2540                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
2541                                 NULL,
2542                                 rte_strerror(EINVAL));
2543         }
2544
2545         /* Each pipe has exactly 13 TCs, with exactly one TC for each priority */
2546         TAILQ_FOREACH(np, nl, node) {
2547                 uint32_t mask = 0, mask_expected =
2548                         RTE_LEN2MASK(RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE,
2549                                 uint32_t);
2550
2551                 if (np->level != TM_NODE_LEVEL_PIPE)
2552                         continue;
2553
2554                 if (np->n_children != RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE)
2555                         return -rte_tm_error_set(error,
2556                                 EINVAL,
2557                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
2558                                 NULL,
2559                                 rte_strerror(EINVAL));
2560
2561                 TAILQ_FOREACH(nt, nl, node) {
2562                         if (nt->level != TM_NODE_LEVEL_TC ||
2563                                 nt->parent_node_id != np->node_id)
2564                                 continue;
2565
2566                         mask |= 1 << nt->priority;
2567                 }
2568
2569                 if (mask != mask_expected)
2570                         return -rte_tm_error_set(error,
2571                                 EINVAL,
2572                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
2573                                 NULL,
2574                                 rte_strerror(EINVAL));
2575         }
2576
2577         /** Each Strict priority TC has exactly 1 packet queues while
2578          *      lowest priority TC (Best-effort) has 4 queues.
2579          */
2580         TAILQ_FOREACH(nt, nl, node) {
2581                 if (nt->level != TM_NODE_LEVEL_TC)
2582                         continue;
2583
2584                 if (nt->n_children != 1 && nt->n_children != RTE_SCHED_BE_QUEUES_PER_PIPE)
2585                         return -rte_tm_error_set(error,
2586                                 EINVAL,
2587                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
2588                                 NULL,
2589                                 rte_strerror(EINVAL));
2590         }
2591
2592         /**
2593          * Shared shapers:
2594          *    -For each TC #i, all pipes in the same subport use the same
2595          *     shared shaper (or no shared shaper) for their TC#i.
2596          *    -Each shared shaper needs to have at least one user. All its
2597          *     users have to be TC nodes with the same priority and the same
2598          *     subport.
2599          */
2600         TAILQ_FOREACH(ns, nl, node) {
2601                 struct tm_shared_shaper *s[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
2602                 uint32_t id;
2603
2604                 if (ns->level != TM_NODE_LEVEL_SUBPORT)
2605                         continue;
2606
2607                 for (id = 0; id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; id++)
2608                         s[id] = tm_subport_tc_shared_shaper_get(dev, ns, id);
2609
2610                 TAILQ_FOREACH(nt, nl, node) {
2611                         struct tm_shared_shaper *subport_ss, *tc_ss;
2612
2613                         if (nt->level != TM_NODE_LEVEL_TC ||
2614                                 nt->parent_node->parent_node_id !=
2615                                         ns->node_id)
2616                                 continue;
2617
2618                         subport_ss = s[nt->priority];
2619                         tc_ss = tm_tc_shared_shaper_get(dev, nt);
2620
2621                         if (subport_ss == NULL && tc_ss == NULL)
2622                                 continue;
2623
2624                         if ((subport_ss == NULL && tc_ss != NULL) ||
2625                                 (subport_ss != NULL && tc_ss == NULL) ||
2626                                 subport_ss->shared_shaper_id !=
2627                                         tc_ss->shared_shaper_id)
2628                                 return -rte_tm_error_set(error,
2629                                         EINVAL,
2630                                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2631                                         NULL,
2632                                         rte_strerror(EINVAL));
2633                 }
2634         }
2635
2636         TAILQ_FOREACH(ss, ssl, node) {
2637                 struct tm_node *nt_any = tm_shared_shaper_get_tc(dev, ss);
2638                 uint32_t n_users = 0;
2639
2640                 if (nt_any != NULL)
2641                         TAILQ_FOREACH(nt, nl, node) {
2642                                 if (nt->level != TM_NODE_LEVEL_TC ||
2643                                         nt->priority != nt_any->priority ||
2644                                         nt->parent_node->parent_node_id !=
2645                                         nt_any->parent_node->parent_node_id)
2646                                         continue;
2647
2648                                 n_users++;
2649                         }
2650
2651                 if (ss->n_users == 0 || ss->n_users != n_users)
2652                         return -rte_tm_error_set(error,
2653                                 EINVAL,
2654                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
2655                                 NULL,
2656                                 rte_strerror(EINVAL));
2657         }
2658
2659         /* Not too many subport profiles. */
2660         if (subport_profiles_generate(dev))
2661                 return -rte_tm_error_set(error,
2662                         EINVAL,
2663                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2664                         NULL,
2665                         rte_strerror(EINVAL));
2666
2667
2668         /* Not too many pipe profiles. */
2669         if (pipe_profiles_generate(dev))
2670                 return -rte_tm_error_set(error,
2671                         EINVAL,
2672                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2673                         NULL,
2674                         rte_strerror(EINVAL));
2675
2676         /**
2677          * WRED (when used, i.e. at least one WRED profile defined):
2678          *    -Each WRED profile must have at least one user.
2679          *    -All leaf nodes must have their private WRED context enabled.
2680          *    -For each TC #i, all leaf nodes must use the same WRED profile
2681          *     for their private WRED context.
2682          */
2683         if (h->n_wred_profiles) {
2684                 struct tm_wred_profile *wp;
2685                 struct tm_wred_profile *w[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
2686                 uint32_t id;
2687
2688                 TAILQ_FOREACH(wp, wpl, node)
2689                         if (wp->n_users == 0)
2690                                 return -rte_tm_error_set(error,
2691                                         EINVAL,
2692                                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2693                                         NULL,
2694                                         rte_strerror(EINVAL));
2695
2696                 for (id = 0; id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; id++) {
2697                         w[id] = tm_tc_wred_profile_get(dev, id);
2698
2699                         if (w[id] == NULL)
2700                                 return -rte_tm_error_set(error,
2701                                         EINVAL,
2702                                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2703                                         NULL,
2704                                         rte_strerror(EINVAL));
2705                 }
2706
2707                 TAILQ_FOREACH(nq, nl, node) {
2708                         uint32_t id;
2709
2710                         if (nq->level != TM_NODE_LEVEL_QUEUE)
2711                                 continue;
2712
2713                         id = nq->parent_node->priority;
2714
2715                         if (nq->wred_profile == NULL ||
2716                                 nq->wred_profile->wred_profile_id !=
2717                                         w[id]->wred_profile_id)
2718                                 return -rte_tm_error_set(error,
2719                                         EINVAL,
2720                                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2721                                         NULL,
2722                                         rte_strerror(EINVAL));
2723                 }
2724         }
2725
2726         return 0;
2727 }
2728
2729 static void
2730 hierarchy_blueprints_create(struct rte_eth_dev *dev)
2731 {
2732         struct pmd_internals *p = dev->data->dev_private;
2733         struct tm_params *t = &p->soft.tm.params;
2734         struct tm_hierarchy *h = &p->soft.tm.h;
2735
2736         struct tm_node_list *nl = &h->nodes;
2737         struct tm_node *root = tm_root_node_present(dev), *n;
2738
2739         uint32_t subport_id;
2740
2741         t->port_params = (struct rte_sched_port_params) {
2742                 .name = dev->data->name,
2743                 .socket = dev->data->numa_node,
2744                 .rate = root->shaper_profile->params.peak.rate,
2745                 .mtu = dev->data->mtu,
2746                 .frame_overhead =
2747                         root->shaper_profile->params.pkt_length_adjust,
2748                 .n_subports_per_port = root->n_children,
2749                 .n_subport_profiles = t->n_subport_profiles,
2750                 .subport_profiles = t->subport_profile,
2751                 .n_max_subport_profiles = TM_MAX_SUBPORT_PROFILE,
2752                 .n_pipes_per_subport = TM_MAX_PIPES_PER_SUBPORT,
2753         };
2754
2755         subport_id = 0;
2756         TAILQ_FOREACH(n, nl, node) {
2757
2758                 if (n->level != TM_NODE_LEVEL_SUBPORT)
2759                         continue;
2760
2761                 t->subport_params[subport_id] =
2762                         (struct rte_sched_subport_params) {
2763                                 .n_pipes_per_subport_enabled =
2764                                         h->n_tm_nodes[TM_NODE_LEVEL_PIPE] /
2765                                         h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT],
2766                                 .qsize = {p->params.tm.qsize[0],
2767                                         p->params.tm.qsize[1],
2768                                         p->params.tm.qsize[2],
2769                                         p->params.tm.qsize[3],
2770                                         p->params.tm.qsize[4],
2771                                         p->params.tm.qsize[5],
2772                                         p->params.tm.qsize[6],
2773                                         p->params.tm.qsize[7],
2774                                         p->params.tm.qsize[8],
2775                                         p->params.tm.qsize[9],
2776                                         p->params.tm.qsize[10],
2777                                         p->params.tm.qsize[11],
2778                                         p->params.tm.qsize[12],
2779                                 },
2780                                 .pipe_profiles = t->pipe_profiles,
2781                                 .n_pipe_profiles = t->n_pipe_profiles,
2782                                 .n_max_pipe_profiles = TM_MAX_PIPE_PROFILE,
2783                 };
2784                 wred_profiles_set(dev, subport_id);
2785                 subport_id++;
2786         }
2787 }
2788
2789 /* Traffic manager hierarchy commit */
2790 static int
2791 pmd_tm_hierarchy_commit(struct rte_eth_dev *dev,
2792         int clear_on_fail,
2793         struct rte_tm_error *error)
2794 {
2795         struct pmd_internals *p = dev->data->dev_private;
2796         int status;
2797
2798         /* Checks */
2799         if (p->soft.tm.hierarchy_frozen)
2800                 return -rte_tm_error_set(error,
2801                         EBUSY,
2802                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2803                         NULL,
2804                         rte_strerror(EBUSY));
2805
2806         status = hierarchy_commit_check(dev, error);
2807         if (status) {
2808                 if (clear_on_fail)
2809                         tm_hierarchy_free(p);
2810
2811                 return status;
2812         }
2813
2814         /* Create blueprints */
2815         hierarchy_blueprints_create(dev);
2816
2817         /* Freeze hierarchy */
2818         p->soft.tm.hierarchy_frozen = 1;
2819
2820         return 0;
2821 }
2822
2823 #ifdef RTE_SCHED_SUBPORT_TC_OV
2824
2825 static int
2826 update_pipe_weight(struct rte_eth_dev *dev, struct tm_node *np, uint32_t weight)
2827 {
2828         struct pmd_internals *p = dev->data->dev_private;
2829         uint32_t pipe_id = tm_node_pipe_id(dev, np);
2830
2831         struct tm_node *ns = np->parent_node;
2832         uint32_t subport_id = tm_node_subport_id(dev, ns);
2833
2834         struct rte_sched_pipe_params *profile0 = pipe_profile_get(dev, np);
2835         struct rte_sched_pipe_params profile1;
2836         uint32_t pipe_profile_id;
2837
2838         /* Derive new pipe profile. */
2839         memcpy(&profile1, profile0, sizeof(profile1));
2840         profile1.tc_ov_weight = (uint8_t)weight;
2841
2842         /* Since implementation does not allow adding more pipe profiles after
2843          * port configuration, the pipe configuration can be successfully
2844          * updated only if the new profile is also part of the existing set of
2845          * pipe profiles.
2846          */
2847         if (pipe_profile_exists(dev, &profile1, &pipe_profile_id) == 0)
2848                 return -1;
2849
2850         /* Update the pipe profile used by the current pipe. */
2851         if (rte_sched_pipe_config(SCHED(p), subport_id, pipe_id,
2852                 (int32_t)pipe_profile_id))
2853                 return -1;
2854
2855         /* Commit changes. */
2856         pipe_profile_mark(dev, subport_id, pipe_id, pipe_profile_id);
2857         np->weight = weight;
2858
2859         return 0;
2860 }
2861
2862 #endif
2863
2864 static int
2865 update_queue_weight(struct rte_eth_dev *dev,
2866         struct tm_node *nq, uint32_t weight)
2867 {
2868         struct pmd_internals *p = dev->data->dev_private;
2869         uint32_t queue_id = tm_node_queue_id(dev, nq);
2870
2871         struct tm_node *nt = nq->parent_node;
2872
2873         struct tm_node *np = nt->parent_node;
2874         uint32_t pipe_id = tm_node_pipe_id(dev, np);
2875
2876         struct tm_node *ns = np->parent_node;
2877         uint32_t subport_id = tm_node_subport_id(dev, ns);
2878
2879         uint32_t pipe_be_queue_id =
2880                 queue_id - RTE_SCHED_TRAFFIC_CLASS_BE;
2881
2882         struct rte_sched_pipe_params *profile0 = pipe_profile_get(dev, np);
2883         struct rte_sched_pipe_params profile1;
2884         uint32_t pipe_profile_id;
2885
2886         /* Derive new pipe profile. */
2887         memcpy(&profile1, profile0, sizeof(profile1));
2888         profile1.wrr_weights[pipe_be_queue_id] = (uint8_t)weight;
2889
2890         /* Since implementation does not allow adding more pipe profiles after
2891          * port configuration, the pipe configuration can be successfully
2892          * updated only if the new profile is also part of the existing set
2893          * of pipe profiles.
2894          */
2895         if (pipe_profile_exists(dev, &profile1, &pipe_profile_id) == 0)
2896                 return -1;
2897
2898         /* Update the pipe profile used by the current pipe. */
2899         if (rte_sched_pipe_config(SCHED(p), subport_id, pipe_id,
2900                 (int32_t)pipe_profile_id))
2901                 return -1;
2902
2903         /* Commit changes. */
2904         pipe_profile_mark(dev, subport_id, pipe_id, pipe_profile_id);
2905         nq->weight = weight;
2906
2907         return 0;
2908 }
2909
2910 /* Traffic manager node parent update */
2911 static int
2912 pmd_tm_node_parent_update(struct rte_eth_dev *dev,
2913         uint32_t node_id,
2914         uint32_t parent_node_id,
2915         uint32_t priority,
2916         uint32_t weight,
2917         struct rte_tm_error *error)
2918 {
2919         struct tm_node *n;
2920
2921         /* Port must be started and TM used. */
2922         if (dev->data->dev_started == 0 && (tm_used(dev) == 0))
2923                 return -rte_tm_error_set(error,
2924                         EBUSY,
2925                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
2926                         NULL,
2927                         rte_strerror(EBUSY));
2928
2929         /* Node must be valid */
2930         n = tm_node_search(dev, node_id);
2931         if (n == NULL)
2932                 return -rte_tm_error_set(error,
2933                         EINVAL,
2934                         RTE_TM_ERROR_TYPE_NODE_ID,
2935                         NULL,
2936                         rte_strerror(EINVAL));
2937
2938         /* Parent node must be the same */
2939         if (n->parent_node_id != parent_node_id)
2940                 return -rte_tm_error_set(error,
2941                         EINVAL,
2942                         RTE_TM_ERROR_TYPE_NODE_PARENT_NODE_ID,
2943                         NULL,
2944                         rte_strerror(EINVAL));
2945
2946         /* Priority must be the same */
2947         if (n->priority != priority)
2948                 return -rte_tm_error_set(error,
2949                         EINVAL,
2950                         RTE_TM_ERROR_TYPE_NODE_PRIORITY,
2951                         NULL,
2952                         rte_strerror(EINVAL));
2953
2954         /* weight: must be 1 .. 255 */
2955         if (weight == 0 || weight >= UINT8_MAX)
2956                 return -rte_tm_error_set(error,
2957                         EINVAL,
2958                         RTE_TM_ERROR_TYPE_NODE_WEIGHT,
2959                         NULL,
2960                         rte_strerror(EINVAL));
2961
2962         switch (n->level) {
2963         case TM_NODE_LEVEL_PORT:
2964                 return -rte_tm_error_set(error,
2965                         EINVAL,
2966                         RTE_TM_ERROR_TYPE_NODE_WEIGHT,
2967                         NULL,
2968                         rte_strerror(EINVAL));
2969                 /* fall-through */
2970         case TM_NODE_LEVEL_SUBPORT:
2971                 return -rte_tm_error_set(error,
2972                         EINVAL,
2973                         RTE_TM_ERROR_TYPE_NODE_WEIGHT,
2974                         NULL,
2975                         rte_strerror(EINVAL));
2976                 /* fall-through */
2977         case TM_NODE_LEVEL_PIPE:
2978 #ifdef RTE_SCHED_SUBPORT_TC_OV
2979                 if (update_pipe_weight(dev, n, weight))
2980                         return -rte_tm_error_set(error,
2981                                 EINVAL,
2982                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
2983                                 NULL,
2984                                 rte_strerror(EINVAL));
2985                 return 0;
2986 #else
2987                 return -rte_tm_error_set(error,
2988                         EINVAL,
2989                         RTE_TM_ERROR_TYPE_NODE_WEIGHT,
2990                         NULL,
2991                         rte_strerror(EINVAL));
2992 #endif
2993                 /* fall-through */
2994         case TM_NODE_LEVEL_TC:
2995                 return -rte_tm_error_set(error,
2996                         EINVAL,
2997                         RTE_TM_ERROR_TYPE_NODE_WEIGHT,
2998                         NULL,
2999                         rte_strerror(EINVAL));
3000                 /* fall-through */
3001         case TM_NODE_LEVEL_QUEUE:
3002                 /* fall-through */
3003         default:
3004                 if (update_queue_weight(dev, n, weight))
3005                         return -rte_tm_error_set(error,
3006                                 EINVAL,
3007                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
3008                                 NULL,
3009                                 rte_strerror(EINVAL));
3010                 return 0;
3011         }
3012 }
3013
3014 static int
3015 update_subport_rate(struct rte_eth_dev *dev,
3016         struct tm_node *ns,
3017         struct tm_shaper_profile *sp)
3018 {
3019         struct pmd_internals *p = dev->data->dev_private;
3020         uint32_t subport_id = tm_node_subport_id(dev, ns);
3021
3022         struct rte_sched_subport_profile_params *profile0 =
3023                                         subport_profile_get(dev, ns);
3024         struct rte_sched_subport_profile_params profile1;
3025         uint32_t subport_profile_id;
3026
3027         /* Derive new pipe profile. */
3028         memcpy(&profile1, profile0, sizeof(profile1));
3029         profile1.tb_rate = sp->params.peak.rate;
3030         profile1.tb_size = sp->params.peak.size;
3031
3032         /* Since implementation does not allow adding more subport profiles
3033          * after port configuration, the pipe configuration can be successfully
3034          * updated only if the new profile is also part of the existing set of
3035          * pipe profiles.
3036          */
3037         if (subport_profile_exists(dev, &profile1, &subport_profile_id) == 0)
3038                 return -1;
3039
3040         /* Update the subport configuration. */
3041         if (rte_sched_subport_config(SCHED(p), subport_id,
3042                 NULL, subport_profile_id))
3043                 return -1;
3044
3045         /* Commit changes. */
3046         ns->shaper_profile->n_users--;
3047
3048         ns->shaper_profile = sp;
3049         ns->params.shaper_profile_id = sp->shaper_profile_id;
3050         sp->n_users++;
3051
3052         subport_profile_mark(dev, subport_id, subport_profile_id);
3053
3054         memcpy(&p->soft.tm.params.subport_profile[subport_profile_id],
3055                 &profile1,
3056                 sizeof(profile1));
3057
3058         return 0;
3059 }
3060
3061 static int
3062 update_pipe_rate(struct rte_eth_dev *dev,
3063         struct tm_node *np,
3064         struct tm_shaper_profile *sp)
3065 {
3066         struct pmd_internals *p = dev->data->dev_private;
3067         uint32_t pipe_id = tm_node_pipe_id(dev, np);
3068
3069         struct tm_node *ns = np->parent_node;
3070         uint32_t subport_id = tm_node_subport_id(dev, ns);
3071
3072         struct rte_sched_pipe_params *profile0 = pipe_profile_get(dev, np);
3073         struct rte_sched_pipe_params profile1;
3074         uint32_t pipe_profile_id;
3075
3076         /* Derive new pipe profile. */
3077         memcpy(&profile1, profile0, sizeof(profile1));
3078         profile1.tb_rate = sp->params.peak.rate;
3079         profile1.tb_size = sp->params.peak.size;
3080
3081         /* Since implementation does not allow adding more pipe profiles after
3082          * port configuration, the pipe configuration can be successfully
3083          * updated only if the new profile is also part of the existing set of
3084          * pipe profiles.
3085          */
3086         if (pipe_profile_exists(dev, &profile1, &pipe_profile_id) == 0)
3087                 return -1;
3088
3089         /* Update the pipe profile used by the current pipe. */
3090         if (rte_sched_pipe_config(SCHED(p), subport_id, pipe_id,
3091                 (int32_t)pipe_profile_id))
3092                 return -1;
3093
3094         /* Commit changes. */
3095         pipe_profile_mark(dev, subport_id, pipe_id, pipe_profile_id);
3096         np->shaper_profile->n_users--;
3097         np->shaper_profile = sp;
3098         np->params.shaper_profile_id = sp->shaper_profile_id;
3099         sp->n_users++;
3100
3101         return 0;
3102 }
3103
3104 static int
3105 update_tc_rate(struct rte_eth_dev *dev,
3106         struct tm_node *nt,
3107         struct tm_shaper_profile *sp)
3108 {
3109         struct pmd_internals *p = dev->data->dev_private;
3110         uint32_t tc_id = tm_node_tc_id(dev, nt);
3111
3112         struct tm_node *np = nt->parent_node;
3113         uint32_t pipe_id = tm_node_pipe_id(dev, np);
3114
3115         struct tm_node *ns = np->parent_node;
3116         uint32_t subport_id = tm_node_subport_id(dev, ns);
3117
3118         struct rte_sched_pipe_params *profile0 = pipe_profile_get(dev, np);
3119         struct rte_sched_pipe_params profile1;
3120         uint32_t pipe_profile_id;
3121
3122         /* Derive new pipe profile. */
3123         memcpy(&profile1, profile0, sizeof(profile1));
3124         profile1.tc_rate[tc_id] = sp->params.peak.rate;
3125
3126         /* Since implementation does not allow adding more pipe profiles after
3127          * port configuration, the pipe configuration can be successfully
3128          * updated only if the new profile is also part of the existing set of
3129          * pipe profiles.
3130          */
3131         if (pipe_profile_exists(dev, &profile1, &pipe_profile_id) == 0)
3132                 return -1;
3133
3134         /* Update the pipe profile used by the current pipe. */
3135         if (rte_sched_pipe_config(SCHED(p), subport_id, pipe_id,
3136                 (int32_t)pipe_profile_id))
3137                 return -1;
3138
3139         /* Commit changes. */
3140         pipe_profile_mark(dev, subport_id, pipe_id, pipe_profile_id);
3141         nt->shaper_profile->n_users--;
3142         nt->shaper_profile = sp;
3143         nt->params.shaper_profile_id = sp->shaper_profile_id;
3144         sp->n_users++;
3145
3146         return 0;
3147 }
3148
3149 /* Traffic manager node shaper update */
3150 static int
3151 pmd_tm_node_shaper_update(struct rte_eth_dev *dev,
3152         uint32_t node_id,
3153         uint32_t shaper_profile_id,
3154         struct rte_tm_error *error)
3155 {
3156         struct tm_node *n;
3157         struct tm_shaper_profile *sp;
3158
3159         /* Port must be started and TM used. */
3160         if (dev->data->dev_started == 0 && (tm_used(dev) == 0))
3161                 return -rte_tm_error_set(error,
3162                         EBUSY,
3163                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
3164                         NULL,
3165                         rte_strerror(EBUSY));
3166
3167         /* Node must be valid */
3168         n = tm_node_search(dev, node_id);
3169         if (n == NULL)
3170                 return -rte_tm_error_set(error,
3171                         EINVAL,
3172                         RTE_TM_ERROR_TYPE_NODE_ID,
3173                         NULL,
3174                         rte_strerror(EINVAL));
3175
3176         /* Shaper profile must be valid. */
3177         sp = tm_shaper_profile_search(dev, shaper_profile_id);
3178         if (sp == NULL)
3179                 return -rte_tm_error_set(error,
3180                         EINVAL,
3181                         RTE_TM_ERROR_TYPE_SHAPER_PROFILE,
3182                         NULL,
3183                         rte_strerror(EINVAL));
3184
3185         switch (n->level) {
3186         case TM_NODE_LEVEL_PORT:
3187                 return -rte_tm_error_set(error,
3188                         EINVAL,
3189                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
3190                         NULL,
3191                         rte_strerror(EINVAL));
3192                 /* fall-through */
3193         case TM_NODE_LEVEL_SUBPORT:
3194                 if (update_subport_rate(dev, n, sp))
3195                         return -rte_tm_error_set(error,
3196                                 EINVAL,
3197                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
3198                                 NULL,
3199                                 rte_strerror(EINVAL));
3200                 return 0;
3201                 /* fall-through */
3202         case TM_NODE_LEVEL_PIPE:
3203                 if (update_pipe_rate(dev, n, sp))
3204                         return -rte_tm_error_set(error,
3205                                 EINVAL,
3206                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
3207                                 NULL,
3208                                 rte_strerror(EINVAL));
3209                 return 0;
3210                 /* fall-through */
3211         case TM_NODE_LEVEL_TC:
3212                 if (update_tc_rate(dev, n, sp))
3213                         return -rte_tm_error_set(error,
3214                                 EINVAL,
3215                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
3216                                 NULL,
3217                                 rte_strerror(EINVAL));
3218                 return 0;
3219                 /* fall-through */
3220         case TM_NODE_LEVEL_QUEUE:
3221                 /* fall-through */
3222         default:
3223                 return -rte_tm_error_set(error,
3224                         EINVAL,
3225                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
3226                         NULL,
3227                         rte_strerror(EINVAL));
3228         }
3229 }
3230
3231 static inline uint32_t
3232 tm_port_queue_id(struct rte_eth_dev *dev,
3233         uint32_t port_subport_id,
3234         uint32_t subport_pipe_id,
3235         uint32_t pipe_tc_id,
3236         uint32_t tc_queue_id)
3237 {
3238         struct pmd_internals *p = dev->data->dev_private;
3239         struct tm_hierarchy *h = &p->soft.tm.h;
3240         uint32_t n_pipes_per_subport = h->n_tm_nodes[TM_NODE_LEVEL_PIPE] /
3241                         h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT];
3242
3243         uint32_t port_pipe_id =
3244                 port_subport_id * n_pipes_per_subport + subport_pipe_id;
3245
3246         uint32_t port_queue_id =
3247                 port_pipe_id * RTE_SCHED_QUEUES_PER_PIPE + pipe_tc_id + tc_queue_id;
3248
3249         return port_queue_id;
3250 }
3251
3252 static int
3253 read_port_stats(struct rte_eth_dev *dev,
3254         struct tm_node *nr,
3255         struct rte_tm_node_stats *stats,
3256         uint64_t *stats_mask,
3257         int clear)
3258 {
3259         struct pmd_internals *p = dev->data->dev_private;
3260         struct tm_hierarchy *h = &p->soft.tm.h;
3261         uint32_t n_subports_per_port = h->n_tm_nodes[TM_NODE_LEVEL_SUBPORT];
3262         uint32_t subport_id;
3263
3264         for (subport_id = 0; subport_id < n_subports_per_port; subport_id++) {
3265                 struct rte_sched_subport_stats s;
3266                 uint32_t tc_ov, id;
3267
3268                 /* Stats read */
3269                 int status = rte_sched_subport_read_stats(SCHED(p),
3270                         subport_id,
3271                         &s,
3272                         &tc_ov);
3273                 if (status)
3274                         return status;
3275
3276                 /* Stats accumulate */
3277                 for (id = 0; id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; id++) {
3278                         nr->stats.n_pkts +=
3279                                 s.n_pkts_tc[id] - s.n_pkts_tc_dropped[id];
3280                         nr->stats.n_bytes +=
3281                                 s.n_bytes_tc[id] - s.n_bytes_tc_dropped[id];
3282                         nr->stats.leaf.n_pkts_dropped[RTE_COLOR_GREEN] +=
3283                                 s.n_pkts_tc_dropped[id];
3284                         nr->stats.leaf.n_bytes_dropped[RTE_COLOR_GREEN] +=
3285                                 s.n_bytes_tc_dropped[id];
3286                 }
3287         }
3288
3289         /* Stats copy */
3290         if (stats)
3291                 memcpy(stats, &nr->stats, sizeof(*stats));
3292
3293         if (stats_mask)
3294                 *stats_mask = STATS_MASK_DEFAULT;
3295
3296         /* Stats clear */
3297         if (clear)
3298                 memset(&nr->stats, 0, sizeof(nr->stats));
3299
3300         return 0;
3301 }
3302
3303 static int
3304 read_subport_stats(struct rte_eth_dev *dev,
3305         struct tm_node *ns,
3306         struct rte_tm_node_stats *stats,
3307         uint64_t *stats_mask,
3308         int clear)
3309 {
3310         struct pmd_internals *p = dev->data->dev_private;
3311         uint32_t subport_id = tm_node_subport_id(dev, ns);
3312         struct rte_sched_subport_stats s;
3313         uint32_t tc_ov, tc_id;
3314
3315         /* Stats read */
3316         int status = rte_sched_subport_read_stats(SCHED(p),
3317                 subport_id,
3318                 &s,
3319                 &tc_ov);
3320         if (status)
3321                 return status;
3322
3323         /* Stats accumulate */
3324         for (tc_id = 0; tc_id < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; tc_id++) {
3325                 ns->stats.n_pkts +=
3326                         s.n_pkts_tc[tc_id] - s.n_pkts_tc_dropped[tc_id];
3327                 ns->stats.n_bytes +=
3328                         s.n_bytes_tc[tc_id] - s.n_bytes_tc_dropped[tc_id];
3329                 ns->stats.leaf.n_pkts_dropped[RTE_COLOR_GREEN] +=
3330                         s.n_pkts_tc_dropped[tc_id];
3331                 ns->stats.leaf.n_bytes_dropped[RTE_COLOR_GREEN] +=
3332                         s.n_bytes_tc_dropped[tc_id];
3333         }
3334
3335         /* Stats copy */
3336         if (stats)
3337                 memcpy(stats, &ns->stats, sizeof(*stats));
3338
3339         if (stats_mask)
3340                 *stats_mask = STATS_MASK_DEFAULT;
3341
3342         /* Stats clear */
3343         if (clear)
3344                 memset(&ns->stats, 0, sizeof(ns->stats));
3345
3346         return 0;
3347 }
3348
3349 static int
3350 read_pipe_stats(struct rte_eth_dev *dev,
3351         struct tm_node *np,
3352         struct rte_tm_node_stats *stats,
3353         uint64_t *stats_mask,
3354         int clear)
3355 {
3356         struct pmd_internals *p = dev->data->dev_private;
3357
3358         uint32_t pipe_id = tm_node_pipe_id(dev, np);
3359
3360         struct tm_node *ns = np->parent_node;
3361         uint32_t subport_id = tm_node_subport_id(dev, ns);
3362         uint32_t tc_id, queue_id;
3363         uint32_t i;
3364
3365         /* Stats read */
3366         for (i = 0; i < RTE_SCHED_QUEUES_PER_PIPE; i++) {
3367                 struct rte_sched_queue_stats s;
3368                 uint16_t qlen;
3369
3370                 if (i < RTE_SCHED_TRAFFIC_CLASS_BE) {
3371                         tc_id = i;
3372                         queue_id = i;
3373                 } else {
3374                         tc_id = RTE_SCHED_TRAFFIC_CLASS_BE;
3375                         queue_id = i - tc_id;
3376                 }
3377
3378                 uint32_t qid = tm_port_queue_id(dev,
3379                         subport_id,
3380                         pipe_id,
3381                         tc_id,
3382                         queue_id);
3383
3384                 int status = rte_sched_queue_read_stats(SCHED(p),
3385                         qid,
3386                         &s,
3387                         &qlen);
3388                 if (status)
3389                         return status;
3390
3391                 /* Stats accumulate */
3392                 np->stats.n_pkts += s.n_pkts - s.n_pkts_dropped;
3393                 np->stats.n_bytes += s.n_bytes - s.n_bytes_dropped;
3394                 np->stats.leaf.n_pkts_dropped[RTE_COLOR_GREEN] += s.n_pkts_dropped;
3395                 np->stats.leaf.n_bytes_dropped[RTE_COLOR_GREEN] +=
3396                         s.n_bytes_dropped;
3397                 np->stats.leaf.n_pkts_queued = qlen;
3398         }
3399
3400         /* Stats copy */
3401         if (stats)
3402                 memcpy(stats, &np->stats, sizeof(*stats));
3403
3404         if (stats_mask)
3405                 *stats_mask = STATS_MASK_DEFAULT;
3406
3407         /* Stats clear */
3408         if (clear)
3409                 memset(&np->stats, 0, sizeof(np->stats));
3410
3411         return 0;
3412 }
3413
3414 static int
3415 read_tc_stats(struct rte_eth_dev *dev,
3416         struct tm_node *nt,
3417         struct rte_tm_node_stats *stats,
3418         uint64_t *stats_mask,
3419         int clear)
3420 {
3421         struct pmd_internals *p = dev->data->dev_private;
3422
3423         uint32_t tc_id = tm_node_tc_id(dev, nt);
3424
3425         struct tm_node *np = nt->parent_node;
3426         uint32_t pipe_id = tm_node_pipe_id(dev, np);
3427
3428         struct tm_node *ns = np->parent_node;
3429         uint32_t subport_id = tm_node_subport_id(dev, ns);
3430         struct rte_sched_queue_stats s;
3431         uint32_t qid, i;
3432         uint16_t qlen;
3433         int status;
3434
3435         /* Stats read */
3436         if (tc_id < RTE_SCHED_TRAFFIC_CLASS_BE) {
3437                 qid = tm_port_queue_id(dev,
3438                         subport_id,
3439                         pipe_id,
3440                         tc_id,
3441                         0);
3442
3443                 status = rte_sched_queue_read_stats(SCHED(p),
3444                         qid,
3445                         &s,
3446                         &qlen);
3447                 if (status)
3448                         return status;
3449
3450                 /* Stats accumulate */
3451                 nt->stats.n_pkts += s.n_pkts - s.n_pkts_dropped;
3452                 nt->stats.n_bytes += s.n_bytes - s.n_bytes_dropped;
3453                 nt->stats.leaf.n_pkts_dropped[RTE_COLOR_GREEN] += s.n_pkts_dropped;
3454                 nt->stats.leaf.n_bytes_dropped[RTE_COLOR_GREEN] +=
3455                         s.n_bytes_dropped;
3456                 nt->stats.leaf.n_pkts_queued = qlen;
3457         } else {
3458                 for (i = 0; i < RTE_SCHED_BE_QUEUES_PER_PIPE; i++) {
3459                         qid = tm_port_queue_id(dev,
3460                                 subport_id,
3461                                 pipe_id,
3462                                 tc_id,
3463                                 i);
3464
3465                         status = rte_sched_queue_read_stats(SCHED(p),
3466                                 qid,
3467                                 &s,
3468                                 &qlen);
3469                         if (status)
3470                                 return status;
3471
3472                         /* Stats accumulate */
3473                         nt->stats.n_pkts += s.n_pkts - s.n_pkts_dropped;
3474                         nt->stats.n_bytes += s.n_bytes - s.n_bytes_dropped;
3475                         nt->stats.leaf.n_pkts_dropped[RTE_COLOR_GREEN] +=
3476                                 s.n_pkts_dropped;
3477                         nt->stats.leaf.n_bytes_dropped[RTE_COLOR_GREEN] +=
3478                                 s.n_bytes_dropped;
3479                         nt->stats.leaf.n_pkts_queued = qlen;
3480                 }
3481         }
3482
3483         /* Stats copy */
3484         if (stats)
3485                 memcpy(stats, &nt->stats, sizeof(*stats));
3486
3487         if (stats_mask)
3488                 *stats_mask = STATS_MASK_DEFAULT;
3489
3490         /* Stats clear */
3491         if (clear)
3492                 memset(&nt->stats, 0, sizeof(nt->stats));
3493
3494         return 0;
3495 }
3496
3497 static int
3498 read_queue_stats(struct rte_eth_dev *dev,
3499         struct tm_node *nq,
3500         struct rte_tm_node_stats *stats,
3501         uint64_t *stats_mask,
3502         int clear)
3503 {
3504         struct pmd_internals *p = dev->data->dev_private;
3505         struct rte_sched_queue_stats s;
3506         uint16_t qlen;
3507
3508         uint32_t queue_id = tm_node_queue_id(dev, nq);
3509
3510         struct tm_node *nt = nq->parent_node;
3511         uint32_t tc_id = tm_node_tc_id(dev, nt);
3512
3513         struct tm_node *np = nt->parent_node;
3514         uint32_t pipe_id = tm_node_pipe_id(dev, np);
3515
3516         struct tm_node *ns = np->parent_node;
3517         uint32_t subport_id = tm_node_subport_id(dev, ns);
3518
3519         /* Stats read */
3520         uint32_t qid = tm_port_queue_id(dev,
3521                 subport_id,
3522                 pipe_id,
3523                 tc_id,
3524                 queue_id);
3525
3526         int status = rte_sched_queue_read_stats(SCHED(p),
3527                 qid,
3528                 &s,
3529                 &qlen);
3530         if (status)
3531                 return status;
3532
3533         /* Stats accumulate */
3534         nq->stats.n_pkts += s.n_pkts - s.n_pkts_dropped;
3535         nq->stats.n_bytes += s.n_bytes - s.n_bytes_dropped;
3536         nq->stats.leaf.n_pkts_dropped[RTE_COLOR_GREEN] += s.n_pkts_dropped;
3537         nq->stats.leaf.n_bytes_dropped[RTE_COLOR_GREEN] +=
3538                 s.n_bytes_dropped;
3539         nq->stats.leaf.n_pkts_queued = qlen;
3540
3541         /* Stats copy */
3542         if (stats)
3543                 memcpy(stats, &nq->stats, sizeof(*stats));
3544
3545         if (stats_mask)
3546                 *stats_mask = STATS_MASK_QUEUE;
3547
3548         /* Stats clear */
3549         if (clear)
3550                 memset(&nq->stats, 0, sizeof(nq->stats));
3551
3552         return 0;
3553 }
3554
3555 /* Traffic manager read stats counters for specific node */
3556 static int
3557 pmd_tm_node_stats_read(struct rte_eth_dev *dev,
3558         uint32_t node_id,
3559         struct rte_tm_node_stats *stats,
3560         uint64_t *stats_mask,
3561         int clear,
3562         struct rte_tm_error *error)
3563 {
3564         struct tm_node *n;
3565
3566         /* Port must be started and TM used. */
3567         if (dev->data->dev_started == 0 && (tm_used(dev) == 0))
3568                 return -rte_tm_error_set(error,
3569                         EBUSY,
3570                         RTE_TM_ERROR_TYPE_UNSPECIFIED,
3571                         NULL,
3572                         rte_strerror(EBUSY));
3573
3574         /* Node must be valid */
3575         n = tm_node_search(dev, node_id);
3576         if (n == NULL)
3577                 return -rte_tm_error_set(error,
3578                         EINVAL,
3579                         RTE_TM_ERROR_TYPE_NODE_ID,
3580                         NULL,
3581                         rte_strerror(EINVAL));
3582
3583         switch (n->level) {
3584         case TM_NODE_LEVEL_PORT:
3585                 if (read_port_stats(dev, n, stats, stats_mask, clear))
3586                         return -rte_tm_error_set(error,
3587                                 EINVAL,
3588                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
3589                                 NULL,
3590                                 rte_strerror(EINVAL));
3591                 return 0;
3592
3593         case TM_NODE_LEVEL_SUBPORT:
3594                 if (read_subport_stats(dev, n, stats, stats_mask, clear))
3595                         return -rte_tm_error_set(error,
3596                                 EINVAL,
3597                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
3598                                 NULL,
3599                                 rte_strerror(EINVAL));
3600                 return 0;
3601
3602         case TM_NODE_LEVEL_PIPE:
3603                 if (read_pipe_stats(dev, n, stats, stats_mask, clear))
3604                         return -rte_tm_error_set(error,
3605                                 EINVAL,
3606                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
3607                                 NULL,
3608                                 rte_strerror(EINVAL));
3609                 return 0;
3610
3611         case TM_NODE_LEVEL_TC:
3612                 if (read_tc_stats(dev, n, stats, stats_mask, clear))
3613                         return -rte_tm_error_set(error,
3614                                 EINVAL,
3615                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
3616                                 NULL,
3617                                 rte_strerror(EINVAL));
3618                 return 0;
3619
3620         case TM_NODE_LEVEL_QUEUE:
3621         default:
3622                 if (read_queue_stats(dev, n, stats, stats_mask, clear))
3623                         return -rte_tm_error_set(error,
3624                                 EINVAL,
3625                                 RTE_TM_ERROR_TYPE_UNSPECIFIED,
3626                                 NULL,
3627                                 rte_strerror(EINVAL));
3628                 return 0;
3629         }
3630 }
3631
3632 const struct rte_tm_ops pmd_tm_ops = {
3633         .node_type_get = pmd_tm_node_type_get,
3634         .capabilities_get = pmd_tm_capabilities_get,
3635         .level_capabilities_get = pmd_tm_level_capabilities_get,
3636         .node_capabilities_get = pmd_tm_node_capabilities_get,
3637
3638         .wred_profile_add = pmd_tm_wred_profile_add,
3639         .wred_profile_delete = pmd_tm_wred_profile_delete,
3640         .shared_wred_context_add_update = NULL,
3641         .shared_wred_context_delete = NULL,
3642
3643         .shaper_profile_add = pmd_tm_shaper_profile_add,
3644         .shaper_profile_delete = pmd_tm_shaper_profile_delete,
3645         .shared_shaper_add_update = pmd_tm_shared_shaper_add_update,
3646         .shared_shaper_delete = pmd_tm_shared_shaper_delete,
3647
3648         .node_add = pmd_tm_node_add,
3649         .node_delete = pmd_tm_node_delete,
3650         .node_suspend = NULL,
3651         .node_resume = NULL,
3652         .hierarchy_commit = pmd_tm_hierarchy_commit,
3653
3654         .node_parent_update = pmd_tm_node_parent_update,
3655         .node_shaper_update = pmd_tm_node_shaper_update,
3656         .node_shared_shaper_update = NULL,
3657         .node_stats_update = NULL,
3658         .node_wfq_weight_mode_update = NULL,
3659         .node_cman_update = NULL,
3660         .node_wred_context_update = NULL,
3661         .node_shared_wred_context_update = NULL,
3662
3663         .node_stats_read = pmd_tm_node_stats_read,
3664 };