app/test-pmd/tm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4 #include <stdio.h>
   5 #include <sys/stat.h>
   6
   7 #include <rte_cycles.h>
   8 #include <rte_mbuf.h>
   9 #include <rte_ethdev.h>
  10 #include <rte_flow.h>
  11 #include <rte_meter.h>
  12 #include <rte_eth_softnic.h>
  13 #include <rte_tm.h>
  14
  15 #include "testpmd.h"
  16
  17 #define SUBPORT_NODES_PER_PORT          1
  18 #define PIPE_NODES_PER_SUBPORT          4096
  19 #define TC_NODES_PER_PIPE                       4
  20 #define QUEUE_NODES_PER_TC                      4
  21
  22 #define NUM_PIPE_NODES                                          \
  23         (SUBPORT_NODES_PER_PORT * PIPE_NODES_PER_SUBPORT)
  24
  25 #define NUM_TC_NODES                                            \
  26         (NUM_PIPE_NODES * TC_NODES_PER_PIPE)
  27
  28 #define ROOT_NODE_ID                            1000000
  29 #define SUBPORT_NODES_START_ID          900000
  30 #define PIPE_NODES_START_ID                     800000
  31 #define TC_NODES_START_ID                       700000
  32
  33 #define STATS_MASK_DEFAULT                                      \
  34         (RTE_TM_STATS_N_PKTS |                                  \
  35         RTE_TM_STATS_N_BYTES |                                  \
  36         RTE_TM_STATS_N_PKTS_GREEN_DROPPED |                     \
  37         RTE_TM_STATS_N_BYTES_GREEN_DROPPED)
  38
  39 #define STATS_MASK_QUEUE                                        \
  40         (STATS_MASK_DEFAULT |                                   \
  41         RTE_TM_STATS_N_PKTS_QUEUED)
  42
  43 #define BYTES_IN_MBPS                           (1000 * 1000 / 8)
  44 #define TOKEN_BUCKET_SIZE                       1000000
  45
  46 /* TM Hierarchy Levels */
  47 enum tm_hierarchy_level {
  48         TM_NODE_LEVEL_PORT = 0,
  49         TM_NODE_LEVEL_SUBPORT,
  50         TM_NODE_LEVEL_PIPE,
  51         TM_NODE_LEVEL_TC,
  52         TM_NODE_LEVEL_QUEUE,
  53         TM_NODE_LEVEL_MAX,
  54 };
  55
  56 struct tm_hierarchy {
  57         /* TM Nodes */
  58         uint32_t root_node_id;
  59         uint32_t subport_node_id[SUBPORT_NODES_PER_PORT];
  60         uint32_t pipe_node_id[SUBPORT_NODES_PER_PORT][PIPE_NODES_PER_SUBPORT];
  61         uint32_t tc_node_id[NUM_PIPE_NODES][TC_NODES_PER_PIPE];
  62         uint32_t queue_node_id[NUM_TC_NODES][QUEUE_NODES_PER_TC];
  63
  64         /* TM Hierarchy Nodes Shaper Rates */
  65         uint32_t root_node_shaper_rate;
  66         uint32_t subport_node_shaper_rate;
  67         uint32_t pipe_node_shaper_rate;
  68         uint32_t tc_node_shaper_rate;
  69         uint32_t tc_node_shared_shaper_rate;
  70
  71         uint32_t n_shapers;
  72 };
  73
  74 #define BITFIELD(byte_array, slab_pos, slab_mask, slab_shr)     \
  75 ({                                                              \
  76         uint64_t slab = *((uint64_t *) &byte_array[slab_pos]);  \
  77         uint64_t val =                          \
  78                 (rte_be_to_cpu_64(slab) & slab_mask) >> slab_shr;       \
  79         val;                                            \
  80 })
  81
  82 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe,           \
  83         traffic_class, queue, color)                          \
  84         ((((uint64_t) (queue)) & 0x3) |                       \
  85         ((((uint64_t) (traffic_class)) & 0x3) << 2) |         \
  86         ((((uint64_t) (color)) & 0x3) << 4) |                 \
  87         ((((uint64_t) (subport)) & 0xFFFF) << 16) |           \
  88         ((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32))
  89
  90
  91 static void
  92 pkt_metadata_set(struct rte_port *p, struct rte_mbuf **pkts,
  93         uint32_t n_pkts)
  94 {
  95         struct softnic_port_tm *tm = &p->softport.tm;
  96         uint32_t i;
  97
  98         for (i = 0; i < (n_pkts & (~0x3)); i += 4) {
  99                 struct rte_mbuf *pkt0 = pkts[i];
 100                 struct rte_mbuf *pkt1 = pkts[i + 1];
 101                 struct rte_mbuf *pkt2 = pkts[i + 2];
 102                 struct rte_mbuf *pkt3 = pkts[i + 3];
 103
 104                 uint8_t *pkt0_data = rte_pktmbuf_mtod(pkt0, uint8_t *);
 105                 uint8_t *pkt1_data = rte_pktmbuf_mtod(pkt1, uint8_t *);
 106                 uint8_t *pkt2_data = rte_pktmbuf_mtod(pkt2, uint8_t *);
 107                 uint8_t *pkt3_data = rte_pktmbuf_mtod(pkt3, uint8_t *);
 108
 109                 uint64_t pkt0_subport = BITFIELD(pkt0_data,
 110                                         tm->tm_pktfield0_slabpos,
 111                                         tm->tm_pktfield0_slabmask,
 112                                         tm->tm_pktfield0_slabshr);
 113                 uint64_t pkt0_pipe = BITFIELD(pkt0_data,
 114                                         tm->tm_pktfield1_slabpos,
 115                                         tm->tm_pktfield1_slabmask,
 116                                         tm->tm_pktfield1_slabshr);
 117                 uint64_t pkt0_dscp = BITFIELD(pkt0_data,
 118                                         tm->tm_pktfield2_slabpos,
 119                                         tm->tm_pktfield2_slabmask,
 120                                         tm->tm_pktfield2_slabshr);
 121                 uint32_t pkt0_tc = tm->tm_tc_table[pkt0_dscp & 0x3F] >> 2;
 122                 uint32_t pkt0_tc_q = tm->tm_tc_table[pkt0_dscp & 0x3F] & 0x3;
 123                 uint64_t pkt1_subport = BITFIELD(pkt1_data,
 124                                         tm->tm_pktfield0_slabpos,
 125                                         tm->tm_pktfield0_slabmask,
 126                                         tm->tm_pktfield0_slabshr);
 127                 uint64_t pkt1_pipe = BITFIELD(pkt1_data,
 128                                         tm->tm_pktfield1_slabpos,
 129                                         tm->tm_pktfield1_slabmask,
 130                                         tm->tm_pktfield1_slabshr);
 131                 uint64_t pkt1_dscp = BITFIELD(pkt1_data,
 132                                         tm->tm_pktfield2_slabpos,
 133                                         tm->tm_pktfield2_slabmask,
 134                                         tm->tm_pktfield2_slabshr);
 135                 uint32_t pkt1_tc = tm->tm_tc_table[pkt1_dscp & 0x3F] >> 2;
 136                 uint32_t pkt1_tc_q = tm->tm_tc_table[pkt1_dscp & 0x3F] & 0x3;
 137
 138                 uint64_t pkt2_subport = BITFIELD(pkt2_data,
 139                                         tm->tm_pktfield0_slabpos,
 140                                         tm->tm_pktfield0_slabmask,
 141                                         tm->tm_pktfield0_slabshr);
 142                 uint64_t pkt2_pipe = BITFIELD(pkt2_data,
 143                                         tm->tm_pktfield1_slabpos,
 144                                         tm->tm_pktfield1_slabmask,
 145                                         tm->tm_pktfield1_slabshr);
 146                 uint64_t pkt2_dscp = BITFIELD(pkt2_data,
 147                                         tm->tm_pktfield2_slabpos,
 148                                         tm->tm_pktfield2_slabmask,
 149                                         tm->tm_pktfield2_slabshr);
 150                 uint32_t pkt2_tc = tm->tm_tc_table[pkt2_dscp & 0x3F] >> 2;
 151                 uint32_t pkt2_tc_q = tm->tm_tc_table[pkt2_dscp & 0x3F] & 0x3;
 152
 153                 uint64_t pkt3_subport = BITFIELD(pkt3_data,
 154                                         tm->tm_pktfield0_slabpos,
 155                                         tm->tm_pktfield0_slabmask,
 156                                         tm->tm_pktfield0_slabshr);
 157                 uint64_t pkt3_pipe = BITFIELD(pkt3_data,
 158                                         tm->tm_pktfield1_slabpos,
 159                                         tm->tm_pktfield1_slabmask,
 160                                         tm->tm_pktfield1_slabshr);
 161                 uint64_t pkt3_dscp = BITFIELD(pkt3_data,
 162                                         tm->tm_pktfield2_slabpos,
 163                                         tm->tm_pktfield2_slabmask,
 164                                         tm->tm_pktfield2_slabshr);
 165                 uint32_t pkt3_tc = tm->tm_tc_table[pkt3_dscp & 0x3F] >> 2;
 166                 uint32_t pkt3_tc_q = tm->tm_tc_table[pkt3_dscp & 0x3F] & 0x3;
 167
 168                 uint64_t pkt0_sched = RTE_SCHED_PORT_HIERARCHY(pkt0_subport,
 169                                                 pkt0_pipe,
 170                                                 pkt0_tc,
 171                                                 pkt0_tc_q,
 172                                                 0);
 173                 uint64_t pkt1_sched = RTE_SCHED_PORT_HIERARCHY(pkt1_subport,
 174                                                 pkt1_pipe,
 175                                                 pkt1_tc,
 176                                                 pkt1_tc_q,
 177                                                 0);
 178                 uint64_t pkt2_sched = RTE_SCHED_PORT_HIERARCHY(pkt2_subport,
 179                                                 pkt2_pipe,
 180                                                 pkt2_tc,
 181                                                 pkt2_tc_q,
 182                                                 0);
 183                 uint64_t pkt3_sched = RTE_SCHED_PORT_HIERARCHY(pkt3_subport,
 184                                                 pkt3_pipe,
 185                                                 pkt3_tc,
 186                                                 pkt3_tc_q,
 187                                                 0);
 188
 189                 pkt0->hash.sched.lo = pkt0_sched & 0xFFFFFFFF;
 190                 pkt0->hash.sched.hi = pkt0_sched >> 32;
 191                 pkt1->hash.sched.lo = pkt1_sched & 0xFFFFFFFF;
 192                 pkt1->hash.sched.hi = pkt1_sched >> 32;
 193                 pkt2->hash.sched.lo = pkt2_sched & 0xFFFFFFFF;
 194                 pkt2->hash.sched.hi = pkt2_sched >> 32;
 195                 pkt3->hash.sched.lo = pkt3_sched & 0xFFFFFFFF;
 196                 pkt3->hash.sched.hi = pkt3_sched >> 32;
 197         }
 198
 199         for (; i < n_pkts; i++) {
 200                 struct rte_mbuf *pkt = pkts[i];
 201
 202                 uint8_t *pkt_data = rte_pktmbuf_mtod(pkt, uint8_t *);
 203
 204                 uint64_t pkt_subport = BITFIELD(pkt_data,
 205                                         tm->tm_pktfield0_slabpos,
 206                                         tm->tm_pktfield0_slabmask,
 207                                         tm->tm_pktfield0_slabshr);
 208                 uint64_t pkt_pipe = BITFIELD(pkt_data,
 209                                         tm->tm_pktfield1_slabpos,
 210                                         tm->tm_pktfield1_slabmask,
 211                                         tm->tm_pktfield1_slabshr);
 212                 uint64_t pkt_dscp = BITFIELD(pkt_data,
 213                                         tm->tm_pktfield2_slabpos,
 214                                         tm->tm_pktfield2_slabmask,
 215                                         tm->tm_pktfield2_slabshr);
 216                 uint32_t pkt_tc = tm->tm_tc_table[pkt_dscp & 0x3F] >> 2;
 217                 uint32_t pkt_tc_q = tm->tm_tc_table[pkt_dscp & 0x3F] & 0x3;
 218
 219                 uint64_t pkt_sched = RTE_SCHED_PORT_HIERARCHY(pkt_subport,
 220                                                 pkt_pipe,
 221                                                 pkt_tc,
 222                                                 pkt_tc_q,
 223                                                 0);
 224
 225                 pkt->hash.sched.lo = pkt_sched & 0xFFFFFFFF;
 226                 pkt->hash.sched.hi = pkt_sched >> 32;
 227         }
 228 }
 229
 230 /*
 231  * Soft port packet forward
 232  */
 233 static void
 234 softport_packet_fwd(struct fwd_stream *fs)
 235 {
 236         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 237         struct rte_port *rte_tx_port = &ports[fs->tx_port];
 238         uint16_t nb_rx;
 239         uint16_t nb_tx;
 240         uint32_t retry;
 241
 242 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 243         uint64_t start_tsc;
 244         uint64_t end_tsc;
 245         uint64_t core_cycles;
 246 #endif
 247
 248 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 249         start_tsc = rte_rdtsc();
 250 #endif
 251
 252         /*  Packets Receive */
 253         nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
 254                         pkts_burst, nb_pkt_per_burst);
 255         fs->rx_packets += nb_rx;
 256
 257 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 258         fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 259 #endif
 260
 261         if (rte_tx_port->softnic_enable) {
 262                 /* Set packet metadata if tm flag enabled */
 263                 if (rte_tx_port->softport.tm_flag)
 264                         pkt_metadata_set(rte_tx_port, pkts_burst, nb_rx);
 265
 266                 /* Softport run */
 267                 rte_pmd_softnic_run(fs->tx_port);
 268         }
 269         nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
 270                         pkts_burst, nb_rx);
 271
 272         /* Retry if necessary */
 273         if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
 274                 retry = 0;
 275                 while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
 276                         rte_delay_us(burst_tx_delay_time);
 277                         nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
 278                                         &pkts_burst[nb_tx], nb_rx - nb_tx);
 279                 }
 280         }
 281         fs->tx_packets += nb_tx;
 282
 283 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 284         fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
 285 #endif
 286
 287         if (unlikely(nb_tx < nb_rx)) {
 288                 fs->fwd_dropped += (nb_rx - nb_tx);
 289                 do {
 290                         rte_pktmbuf_free(pkts_burst[nb_tx]);
 291                 } while (++nb_tx < nb_rx);
 292         }
 293 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 294         end_tsc = rte_rdtsc();
 295         core_cycles = (end_tsc - start_tsc);
 296         fs->core_cycles = (uint64_t) (fs->core_cycles + core_cycles);
 297 #endif
 298 }
 299
 300 static void
 301 set_tm_hiearchy_nodes_shaper_rate(portid_t port_id, struct tm_hierarchy *h)
 302 {
 303         struct rte_eth_link link_params;
 304         uint64_t tm_port_rate;
 305
 306         memset(&link_params, 0, sizeof(link_params));
 307
 308         rte_eth_link_get(port_id, &link_params);
 309         tm_port_rate = (uint64_t)link_params.link_speed * BYTES_IN_MBPS;
 310
 311         if (tm_port_rate > UINT32_MAX)
 312                 tm_port_rate = UINT32_MAX;
 313
 314         /* Set tm hierarchy shapers rate */
 315         h->root_node_shaper_rate = tm_port_rate;
 316         h->subport_node_shaper_rate =
 317                 tm_port_rate / SUBPORT_NODES_PER_PORT;
 318         h->pipe_node_shaper_rate
 319                 = h->subport_node_shaper_rate / PIPE_NODES_PER_SUBPORT;
 320         h->tc_node_shaper_rate = h->pipe_node_shaper_rate;
 321         h->tc_node_shared_shaper_rate = h->subport_node_shaper_rate;
 322 }
 323
 324 static int
 325 softport_tm_root_node_add(portid_t port_id, struct tm_hierarchy *h,
 326         struct rte_tm_error *error)
 327 {
 328         struct rte_tm_node_params rnp;
 329         struct rte_tm_shaper_params rsp;
 330         uint32_t priority, weight, level_id, shaper_profile_id;
 331
 332         memset(&rsp, 0, sizeof(struct rte_tm_shaper_params));
 333         memset(&rnp, 0, sizeof(struct rte_tm_node_params));
 334
 335         /* Shaper profile Parameters */
 336         rsp.peak.rate = h->root_node_shaper_rate;
 337         rsp.peak.size = TOKEN_BUCKET_SIZE;
 338         rsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 339         shaper_profile_id = 0;
 340
 341         if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
 342                 &rsp, error)) {
 343                 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
 344                         __func__, error->type, error->message,
 345                         shaper_profile_id);
 346                 return -1;
 347         }
 348
 349         /* Root Node Parameters */
 350         h->root_node_id = ROOT_NODE_ID;
 351         weight = 1;
 352         priority = 0;
 353         level_id = TM_NODE_LEVEL_PORT;
 354         rnp.shaper_profile_id = shaper_profile_id;
 355         rnp.nonleaf.n_sp_priorities = 1;
 356         rnp.stats_mask = STATS_MASK_DEFAULT;
 357
 358         /* Add Node to TM Hierarchy */
 359         if (rte_tm_node_add(port_id, h->root_node_id, RTE_TM_NODE_ID_NULL,
 360                 priority, weight, level_id, &rnp, error)) {
 361                 printf("%s ERROR(%d)-%s!(node_id %u, parent_id %u, level %u)\n",
 362                         __func__, error->type, error->message,
 363                         h->root_node_id, RTE_TM_NODE_ID_NULL,
 364                         level_id);
 365                 return -1;
 366         }
 367         /* Update */
 368         h->n_shapers++;
 369
 370         printf("  Root node added (Start id %u, Count %u, level %u)\n",
 371                 h->root_node_id, 1, level_id);
 372
 373         return 0;
 374 }
 375
 376 static int
 377 softport_tm_subport_node_add(portid_t port_id, struct tm_hierarchy *h,
 378         struct rte_tm_error *error)
 379 {
 380         uint32_t subport_parent_node_id, subport_node_id = 0;
 381         struct rte_tm_node_params snp;
 382         struct rte_tm_shaper_params ssp;
 383         uint32_t priority, weight, level_id, shaper_profile_id;
 384         uint32_t i;
 385
 386         memset(&ssp, 0, sizeof(struct rte_tm_shaper_params));
 387         memset(&snp, 0, sizeof(struct rte_tm_node_params));
 388
 389         shaper_profile_id = h->n_shapers;
 390
 391         /* Add Shaper Profile to TM Hierarchy */
 392         for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
 393                 ssp.peak.rate = h->subport_node_shaper_rate;
 394                 ssp.peak.size = TOKEN_BUCKET_SIZE;
 395                 ssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 396
 397                 if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
 398                         &ssp, error)) {
 399                         printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
 400                                 __func__, error->type, error->message,
 401                                 shaper_profile_id);
 402                         return -1;
 403                 }
 404
 405                 /* Node Parameters */
 406                 h->subport_node_id[i] = SUBPORT_NODES_START_ID + i;
 407                 subport_parent_node_id = h->root_node_id;
 408                 weight = 1;
 409                 priority = 0;
 410                 level_id = TM_NODE_LEVEL_SUBPORT;
 411                 snp.shaper_profile_id = shaper_profile_id;
 412                 snp.nonleaf.n_sp_priorities = 1;
 413                 snp.stats_mask = STATS_MASK_DEFAULT;
 414
 415                 /* Add Node to TM Hiearchy */
 416                 if (rte_tm_node_add(port_id,
 417                                 h->subport_node_id[i],
 418                                 subport_parent_node_id,
 419                                 priority, weight,
 420                                 level_id,
 421                                 &snp,
 422                                 error)) {
 423                         printf("%s ERROR(%d)-%s!(node %u,parent %u,level %u)\n",
 424                                         __func__,
 425                                         error->type,
 426                                         error->message,
 427                                         h->subport_node_id[i],
 428                                         subport_parent_node_id,
 429                                         level_id);
 430                         return -1;
 431                 }
 432                 shaper_profile_id++;
 433                 subport_node_id++;
 434         }
 435         /* Update */
 436         h->n_shapers = shaper_profile_id;
 437
 438         printf("  Subport nodes added (Start id %u, Count %u, level %u)\n",
 439                 h->subport_node_id[0], SUBPORT_NODES_PER_PORT, level_id);
 440
 441         return 0;
 442 }
 443
 444 static int
 445 softport_tm_pipe_node_add(portid_t port_id, struct tm_hierarchy *h,
 446         struct rte_tm_error *error)
 447 {
 448         uint32_t pipe_parent_node_id;
 449         struct rte_tm_node_params pnp;
 450         struct rte_tm_shaper_params psp;
 451         uint32_t priority, weight, level_id, shaper_profile_id;
 452         uint32_t i, j;
 453
 454         memset(&psp, 0, sizeof(struct rte_tm_shaper_params));
 455         memset(&pnp, 0, sizeof(struct rte_tm_node_params));
 456
 457         shaper_profile_id = h->n_shapers;
 458
 459         /* Shaper Profile Parameters */
 460         psp.peak.rate = h->pipe_node_shaper_rate;
 461         psp.peak.size = TOKEN_BUCKET_SIZE;
 462         psp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 463
 464         /* Pipe Node Parameters */
 465         weight = 1;
 466         priority = 0;
 467         level_id = TM_NODE_LEVEL_PIPE;
 468         pnp.nonleaf.n_sp_priorities = 4;
 469         pnp.stats_mask = STATS_MASK_DEFAULT;
 470
 471         /* Add Shaper Profiles and Nodes to TM Hierarchy */
 472         for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
 473                 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
 474                         if (rte_tm_shaper_profile_add(port_id,
 475                                 shaper_profile_id, &psp, error)) {
 476                                 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
 477                                         __func__, error->type, error->message,
 478                                         shaper_profile_id);
 479                                 return -1;
 480                         }
 481                         pnp.shaper_profile_id = shaper_profile_id;
 482                         pipe_parent_node_id = h->subport_node_id[i];
 483                         h->pipe_node_id[i][j] = PIPE_NODES_START_ID +
 484                                 (i * PIPE_NODES_PER_SUBPORT) + j;
 485
 486                         if (rte_tm_node_add(port_id,
 487                                         h->pipe_node_id[i][j],
 488                                         pipe_parent_node_id,
 489                                         priority, weight, level_id,
 490                                         &pnp,
 491                                         error)) {
 492                                 printf("%s ERROR(%d)-%s!(node %u,parent %u )\n",
 493                                         __func__,
 494                                         error->type,
 495                                         error->message,
 496                                         h->pipe_node_id[i][j],
 497                                         pipe_parent_node_id);
 498
 499                                 return -1;
 500                         }
 501                         shaper_profile_id++;
 502                 }
 503         }
 504         /* Update */
 505         h->n_shapers = shaper_profile_id;
 506
 507         printf("  Pipe nodes added (Start id %u, Count %u, level %u)\n",
 508                 h->pipe_node_id[0][0], NUM_PIPE_NODES, level_id);
 509
 510         return 0;
 511 }
 512
 513 static int
 514 softport_tm_tc_node_add(portid_t port_id, struct tm_hierarchy *h,
 515         struct rte_tm_error *error)
 516 {
 517         uint32_t tc_parent_node_id;
 518         struct rte_tm_node_params tnp;
 519         struct rte_tm_shaper_params tsp, tssp;
 520         uint32_t shared_shaper_profile_id[TC_NODES_PER_PIPE];
 521         uint32_t priority, weight, level_id, shaper_profile_id;
 522         uint32_t pos, n_tc_nodes, i, j, k;
 523
 524         memset(&tsp, 0, sizeof(struct rte_tm_shaper_params));
 525         memset(&tssp, 0, sizeof(struct rte_tm_shaper_params));
 526         memset(&tnp, 0, sizeof(struct rte_tm_node_params));
 527
 528         shaper_profile_id = h->n_shapers;
 529
 530         /* Private Shaper Profile (TC) Parameters */
 531         tsp.peak.rate = h->tc_node_shaper_rate;
 532         tsp.peak.size = TOKEN_BUCKET_SIZE;
 533         tsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 534
 535         /* Shared Shaper Profile (TC) Parameters */
 536         tssp.peak.rate = h->tc_node_shared_shaper_rate;
 537         tssp.peak.size = TOKEN_BUCKET_SIZE;
 538         tssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
 539
 540         /* TC Node Parameters */
 541         weight = 1;
 542         level_id = TM_NODE_LEVEL_TC;
 543         tnp.n_shared_shapers = 1;
 544         tnp.nonleaf.n_sp_priorities = 1;
 545         tnp.stats_mask = STATS_MASK_DEFAULT;
 546
 547         /* Add Shared Shaper Profiles to TM Hierarchy */
 548         for (i = 0; i < TC_NODES_PER_PIPE; i++) {
 549                 shared_shaper_profile_id[i] = shaper_profile_id;
 550
 551                 if (rte_tm_shaper_profile_add(port_id,
 552                         shared_shaper_profile_id[i], &tssp, error)) {
 553                         printf("%s ERROR(%d)-%s!(Shared shaper profileid %u)\n",
 554                                 __func__, error->type, error->message,
 555                                 shared_shaper_profile_id[i]);
 556
 557                         return -1;
 558                 }
 559                 if (rte_tm_shared_shaper_add_update(port_id,  i,
 560                         shared_shaper_profile_id[i], error)) {
 561                         printf("%s ERROR(%d)-%s!(Shared shaper id %u)\n",
 562                                 __func__, error->type, error->message, i);
 563
 564                         return -1;
 565                 }
 566                 shaper_profile_id++;
 567         }
 568
 569         /* Add Shaper Profiles and Nodes to TM Hierarchy */
 570         n_tc_nodes = 0;
 571         for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
 572                 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
 573                         for (k = 0; k < TC_NODES_PER_PIPE ; k++) {
 574                                 priority = k;
 575                                 tc_parent_node_id = h->pipe_node_id[i][j];
 576                                 tnp.shared_shaper_id =
 577                                         (uint32_t *)calloc(1, sizeof(uint32_t));
 578                                 tnp.shared_shaper_id[0] = k;
 579                                 pos = j + (i * PIPE_NODES_PER_SUBPORT);
 580                                 h->tc_node_id[pos][k] =
 581                                         TC_NODES_START_ID + n_tc_nodes;
 582
 583                                 if (rte_tm_shaper_profile_add(port_id,
 584                                         shaper_profile_id, &tsp, error)) {
 585                                         printf("%s ERROR(%d)-%s!(shaper %u)\n",
 586                                                 __func__, error->type,
 587                                                 error->message,
 588                                                 shaper_profile_id);
 589
 590                                         return -1;
 591                                 }
 592                                 tnp.shaper_profile_id = shaper_profile_id;
 593                                 if (rte_tm_node_add(port_id,
 594                                                 h->tc_node_id[pos][k],
 595                                                 tc_parent_node_id,
 596                                                 priority, weight,
 597                                                 level_id,
 598                                                 &tnp, error)) {
 599                                         printf("%s ERROR(%d)-%s!(node id %u)\n",
 600                                                 __func__,
 601                                                 error->type,
 602                                                 error->message,
 603                                                 h->tc_node_id[pos][k]);
 604
 605                                         return -1;
 606                                 }
 607                                 shaper_profile_id++;
 608                                 n_tc_nodes++;
 609                         }
 610                 }
 611         }
 612         /* Update */
 613         h->n_shapers = shaper_profile_id;
 614
 615         printf("  TC nodes added (Start id %u, Count %u, level %u)\n",
 616                 h->tc_node_id[0][0], n_tc_nodes, level_id);
 617
 618         return 0;
 619 }
 620
 621 static int
 622 softport_tm_queue_node_add(portid_t port_id, struct tm_hierarchy *h,
 623         struct rte_tm_error *error)
 624 {
 625         uint32_t queue_parent_node_id;
 626         struct rte_tm_node_params qnp;
 627         uint32_t priority, weight, level_id, pos;
 628         uint32_t n_queue_nodes, i, j, k;
 629
 630         memset(&qnp, 0, sizeof(struct rte_tm_node_params));
 631
 632         /* Queue Node Parameters */
 633         priority = 0;
 634         weight = 1;
 635         level_id = TM_NODE_LEVEL_QUEUE;
 636         qnp.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
 637         qnp.leaf.cman = RTE_TM_CMAN_TAIL_DROP;
 638         qnp.stats_mask = STATS_MASK_QUEUE;
 639
 640         /* Add Queue Nodes to TM Hierarchy */
 641         n_queue_nodes = 0;
 642         for (i = 0; i < NUM_PIPE_NODES; i++) {
 643                 for (j = 0; j < TC_NODES_PER_PIPE; j++) {
 644                         queue_parent_node_id = h->tc_node_id[i][j];
 645                         for (k = 0; k < QUEUE_NODES_PER_TC; k++) {
 646                                 pos = j + (i * TC_NODES_PER_PIPE);
 647                                 h->queue_node_id[pos][k] = n_queue_nodes;
 648                                 if (rte_tm_node_add(port_id,
 649                                                 h->queue_node_id[pos][k],
 650                                                 queue_parent_node_id,
 651                                                 priority,
 652                                                 weight,
 653                                                 level_id,
 654                                                 &qnp, error)) {
 655                                         printf("%s ERROR(%d)-%s!(node %u)\n",
 656                                                 __func__,
 657                                                 error->type,
 658                                                 error->message,
 659                                                 h->queue_node_id[pos][k]);
 660
 661                                         return -1;
 662                                 }
 663                                 n_queue_nodes++;
 664                         }
 665                 }
 666         }
 667         printf("  Queue nodes added (Start id %u, Count %u, level %u)\n",
 668                 h->queue_node_id[0][0], n_queue_nodes, level_id);
 669
 670         return 0;
 671 }
 672
 673 /*
 674  * TM Packet Field Setup
 675  */
 676 static void
 677 softport_tm_pktfield_setup(portid_t port_id)
 678 {
 679         struct rte_port *p = &ports[port_id];
 680         uint64_t pktfield0_mask = 0;
 681         uint64_t pktfield1_mask = 0x0000000FFF000000LLU;
 682         uint64_t pktfield2_mask = 0x00000000000000FCLLU;
 683
 684         p->softport.tm = (struct softnic_port_tm) {
 685                 .n_subports_per_port = SUBPORT_NODES_PER_PORT,
 686                 .n_pipes_per_subport = PIPE_NODES_PER_SUBPORT,
 687
 688                 /* Packet field to identify subport
 689                  *
 690                  * Default configuration assumes only one subport, thus
 691                  * the subport ID is hardcoded to 0
 692                  */
 693                 .tm_pktfield0_slabpos = 0,
 694                 .tm_pktfield0_slabmask = pktfield0_mask,
 695                 .tm_pktfield0_slabshr =
 696                         __builtin_ctzll(pktfield0_mask),
 697
 698                 /* Packet field to identify pipe.
 699                  *
 700                  * Default value assumes Ethernet/IPv4/UDP packets,
 701                  * UDP payload bits 12 .. 23
 702                  */
 703                 .tm_pktfield1_slabpos = 40,
 704                 .tm_pktfield1_slabmask = pktfield1_mask,
 705                 .tm_pktfield1_slabshr =
 706                         __builtin_ctzll(pktfield1_mask),
 707
 708                 /* Packet field used as index into TC translation table
 709                  * to identify the traffic class and queue.
 710                  *
 711                  * Default value assumes Ethernet/IPv4 packets, IPv4
 712                  * DSCP field
 713                  */
 714                 .tm_pktfield2_slabpos = 8,
 715                 .tm_pktfield2_slabmask = pktfield2_mask,
 716                 .tm_pktfield2_slabshr =
 717                         __builtin_ctzll(pktfield2_mask),
 718
 719                 .tm_tc_table = {
 720                         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 721                         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 722                         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 723                         0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 724                 }, /**< TC translation table */
 725         };
 726 }
 727
 728 static int
 729 softport_tm_hierarchy_specify(portid_t port_id, struct rte_tm_error *error)
 730 {
 731
 732         struct tm_hierarchy h;
 733         int status;
 734
 735         memset(&h, 0, sizeof(struct tm_hierarchy));
 736
 737         /* TM hierarchy shapers rate */
 738         set_tm_hiearchy_nodes_shaper_rate(port_id, &h);
 739
 740         /* Add root node (level 0) */
 741         status = softport_tm_root_node_add(port_id, &h, error);
 742         if (status)
 743                 return status;
 744
 745         /* Add subport node (level 1) */
 746         status = softport_tm_subport_node_add(port_id, &h, error);
 747         if (status)
 748                 return status;
 749
 750         /* Add pipe nodes (level 2) */
 751         status = softport_tm_pipe_node_add(port_id, &h, error);
 752         if (status)
 753                 return status;
 754
 755         /* Add traffic class nodes (level 3) */
 756         status = softport_tm_tc_node_add(port_id, &h, error);
 757         if (status)
 758                 return status;
 759
 760         /* Add queue nodes (level 4) */
 761         status = softport_tm_queue_node_add(port_id, &h, error);
 762         if (status)
 763                 return status;
 764
 765         /* TM packet fields setup */
 766         softport_tm_pktfield_setup(port_id);
 767
 768         return 0;
 769 }
 770
 771 /*
 772  * Soft port Init
 773  */
 774 static void
 775 softport_tm_begin(portid_t pi)
 776 {
 777         struct rte_port *port = &ports[pi];
 778
 779         /* Soft port TM flag */
 780         if (port->softport.tm_flag == 1) {
 781                 printf("\n\n  TM feature available on port %u\n", pi);
 782
 783                 /* Soft port TM hierarchy configuration */
 784                 if ((port->softport.tm.hierarchy_config == 0) &&
 785                         (port->softport.tm.default_hierarchy_enable == 1)) {
 786                         struct rte_tm_error error;
 787                         int status;
 788
 789                         /* Stop port */
 790                         rte_eth_dev_stop(pi);
 791
 792                         /* TM hierarchy specification */
 793                         status = softport_tm_hierarchy_specify(pi, &error);
 794                         if (status) {
 795                                 printf("  TM Hierarchy built error(%d) - %s\n",
 796                                         error.type, error.message);
 797                                 return;
 798                         }
 799                         printf("\n  TM Hierarchy Specified!\n\v");
 800
 801                         /* TM hierarchy commit */
 802                         status = rte_tm_hierarchy_commit(pi, 0, &error);
 803                         if (status) {
 804                                 printf("  Hierarchy commit error(%d) - %s\n",
 805                                         error.type, error.message);
 806                                 return;
 807                         }
 808                         printf("  Hierarchy Committed (port %u)!", pi);
 809                         port->softport.tm.hierarchy_config = 1;
 810
 811                         /* Start port */
 812                         status = rte_eth_dev_start(pi);
 813                         if (status) {
 814                                 printf("\n  Port %u start error!\n", pi);
 815                                 return;
 816                         }
 817                         printf("\n  Port %u started!\n", pi);
 818                         return;
 819                 }
 820         }
 821         printf("\n  TM feature not available on port %u", pi);
 822 }
 823
 824 struct fwd_engine softnic_tm_engine = {
 825         .fwd_mode_name  = "tm",
 826         .port_fwd_begin = softport_tm_begin,
 827         .port_fwd_end   = NULL,
 828         .packet_fwd     = softport_packet_fwd,
 829 };
 830
 831 struct fwd_engine softnic_tm_bypass_engine = {
 832         .fwd_mode_name  = "tm-bypass",
 833         .port_fwd_begin = NULL,
 834         .port_fwd_end   = NULL,
 835         .packet_fwd     = softport_packet_fwd,
 836 };