1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
7 #include <rte_cycles.h>
9 #include <rte_malloc.h>
10 #include <rte_ethdev.h>
12 #include <rte_meter.h>
13 #include <rte_eth_softnic.h>
18 #define SUBPORT_NODES_PER_PORT 1
19 #define PIPE_NODES_PER_SUBPORT 4096
20 #define TC_NODES_PER_PIPE 4
21 #define QUEUE_NODES_PER_TC 4
23 #define NUM_PIPE_NODES \
24 (SUBPORT_NODES_PER_PORT * PIPE_NODES_PER_SUBPORT)
26 #define NUM_TC_NODES \
27 (NUM_PIPE_NODES * TC_NODES_PER_PIPE)
29 #define ROOT_NODE_ID 1000000
30 #define SUBPORT_NODES_START_ID 900000
31 #define PIPE_NODES_START_ID 800000
32 #define TC_NODES_START_ID 700000
34 #define STATS_MASK_DEFAULT \
35 (RTE_TM_STATS_N_PKTS | \
36 RTE_TM_STATS_N_BYTES | \
37 RTE_TM_STATS_N_PKTS_GREEN_DROPPED | \
38 RTE_TM_STATS_N_BYTES_GREEN_DROPPED)
40 #define STATS_MASK_QUEUE \
41 (STATS_MASK_DEFAULT | \
42 RTE_TM_STATS_N_PKTS_QUEUED)
44 #define BYTES_IN_MBPS (1000 * 1000 / 8)
45 #define TOKEN_BUCKET_SIZE 1000000
47 /* TM Hierarchy Levels */
48 enum tm_hierarchy_level {
49 TM_NODE_LEVEL_PORT = 0,
50 TM_NODE_LEVEL_SUBPORT,
59 uint32_t root_node_id;
60 uint32_t subport_node_id[SUBPORT_NODES_PER_PORT];
61 uint32_t pipe_node_id[SUBPORT_NODES_PER_PORT][PIPE_NODES_PER_SUBPORT];
62 uint32_t tc_node_id[NUM_PIPE_NODES][TC_NODES_PER_PIPE];
63 uint32_t queue_node_id[NUM_TC_NODES][QUEUE_NODES_PER_TC];
65 /* TM Hierarchy Nodes Shaper Rates */
66 uint32_t root_node_shaper_rate;
67 uint32_t subport_node_shaper_rate;
68 uint32_t pipe_node_shaper_rate;
69 uint32_t tc_node_shaper_rate;
70 uint32_t tc_node_shared_shaper_rate;
75 static struct fwd_lcore *softnic_fwd_lcore;
76 static uint16_t softnic_port_id;
77 struct fwd_engine softnic_fwd_engine;
80 * Softnic packet forward
83 softnic_fwd(struct fwd_stream *fs)
85 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
90 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
96 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
97 start_tsc = rte_rdtsc();
100 /* Packets Receive */
101 nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
102 pkts_burst, nb_pkt_per_burst);
103 fs->rx_packets += nb_rx;
105 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
106 fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
109 nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
112 /* Retry if necessary */
113 if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
115 while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
116 rte_delay_us(burst_tx_delay_time);
117 nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
118 &pkts_burst[nb_tx], nb_rx - nb_tx);
121 fs->tx_packets += nb_tx;
123 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
124 fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
127 if (unlikely(nb_tx < nb_rx)) {
128 fs->fwd_dropped += (nb_rx - nb_tx);
130 rte_pktmbuf_free(pkts_burst[nb_tx]);
131 } while (++nb_tx < nb_rx);
133 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
134 end_tsc = rte_rdtsc();
135 core_cycles = (end_tsc - start_tsc);
136 fs->core_cycles = (uint64_t) (fs->core_cycles + core_cycles);
141 softnic_fwd_run(struct fwd_stream *fs)
143 rte_pmd_softnic_run(softnic_port_id);
151 softnic_begin(void *arg __rte_unused)
154 if (!softnic_fwd_lcore->stopped)
160 rte_pmd_softnic_run(softnic_port_id);
161 } while (!softnic_fwd_lcore->stopped);
167 set_tm_hiearchy_nodes_shaper_rate(portid_t port_id,
168 struct tm_hierarchy *h)
170 struct rte_eth_link link_params;
171 uint64_t tm_port_rate;
174 memset(&link_params, 0, sizeof(link_params));
176 ret = rte_eth_link_get(port_id, &link_params);
178 printf("Error during getting device (port %u) link info: %s\n",
179 port_id, rte_strerror(-ret));
182 tm_port_rate = (uint64_t)ETH_SPEED_NUM_10G * BYTES_IN_MBPS;
184 /* Set tm hierarchy shapers rate */
185 h->root_node_shaper_rate = tm_port_rate;
186 h->subport_node_shaper_rate =
187 tm_port_rate / SUBPORT_NODES_PER_PORT;
188 h->pipe_node_shaper_rate
189 = h->subport_node_shaper_rate / PIPE_NODES_PER_SUBPORT;
190 h->tc_node_shaper_rate = h->pipe_node_shaper_rate;
191 h->tc_node_shared_shaper_rate = h->subport_node_shaper_rate;
197 softport_tm_root_node_add(portid_t port_id, struct tm_hierarchy *h,
198 struct rte_tm_error *error)
200 struct rte_tm_node_params rnp;
201 struct rte_tm_shaper_params rsp;
202 uint32_t priority, weight, level_id, shaper_profile_id;
204 memset(&rsp, 0, sizeof(struct rte_tm_shaper_params));
205 memset(&rnp, 0, sizeof(struct rte_tm_node_params));
207 /* Shaper profile Parameters */
208 rsp.peak.rate = h->root_node_shaper_rate;
209 rsp.peak.size = TOKEN_BUCKET_SIZE;
210 rsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
211 shaper_profile_id = 0;
213 if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
215 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
216 __func__, error->type, error->message,
221 /* Root Node Parameters */
222 h->root_node_id = ROOT_NODE_ID;
225 level_id = TM_NODE_LEVEL_PORT;
226 rnp.shaper_profile_id = shaper_profile_id;
227 rnp.nonleaf.n_sp_priorities = 1;
228 rnp.stats_mask = STATS_MASK_DEFAULT;
230 /* Add Node to TM Hierarchy */
231 if (rte_tm_node_add(port_id, h->root_node_id, RTE_TM_NODE_ID_NULL,
232 priority, weight, level_id, &rnp, error)) {
233 printf("%s ERROR(%d)-%s!(node_id %u, parent_id %u, level %u)\n",
234 __func__, error->type, error->message,
235 h->root_node_id, RTE_TM_NODE_ID_NULL,
242 printf(" Root node added (Start id %u, Count %u, level %u)\n",
243 h->root_node_id, 1, level_id);
249 softport_tm_subport_node_add(portid_t port_id,
250 struct tm_hierarchy *h,
251 struct rte_tm_error *error)
253 uint32_t subport_parent_node_id, subport_node_id = 0;
254 struct rte_tm_node_params snp;
255 struct rte_tm_shaper_params ssp;
256 uint32_t priority, weight, level_id, shaper_profile_id;
259 memset(&ssp, 0, sizeof(struct rte_tm_shaper_params));
260 memset(&snp, 0, sizeof(struct rte_tm_node_params));
262 shaper_profile_id = h->n_shapers;
264 /* Add Shaper Profile to TM Hierarchy */
265 for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
266 ssp.peak.rate = h->subport_node_shaper_rate;
267 ssp.peak.size = TOKEN_BUCKET_SIZE;
268 ssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
270 if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
272 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
273 __func__, error->type, error->message,
278 /* Node Parameters */
279 h->subport_node_id[i] = SUBPORT_NODES_START_ID + i;
280 subport_parent_node_id = h->root_node_id;
283 level_id = TM_NODE_LEVEL_SUBPORT;
284 snp.shaper_profile_id = shaper_profile_id;
285 snp.nonleaf.n_sp_priorities = 1;
286 snp.stats_mask = STATS_MASK_DEFAULT;
288 /* Add Node to TM Hiearchy */
289 if (rte_tm_node_add(port_id,
290 h->subport_node_id[i],
291 subport_parent_node_id,
296 printf("%s ERROR(%d)-%s!(node %u,parent %u,level %u)\n",
300 h->subport_node_id[i],
301 subport_parent_node_id,
309 h->n_shapers = shaper_profile_id;
311 printf(" Subport nodes added (Start id %u, Count %u, level %u)\n",
312 h->subport_node_id[0], SUBPORT_NODES_PER_PORT, level_id);
318 softport_tm_pipe_node_add(portid_t port_id,
319 struct tm_hierarchy *h,
320 struct rte_tm_error *error)
322 uint32_t pipe_parent_node_id;
323 struct rte_tm_node_params pnp;
324 struct rte_tm_shaper_params psp;
325 uint32_t priority, weight, level_id, shaper_profile_id;
328 memset(&psp, 0, sizeof(struct rte_tm_shaper_params));
329 memset(&pnp, 0, sizeof(struct rte_tm_node_params));
331 shaper_profile_id = h->n_shapers;
333 /* Shaper Profile Parameters */
334 psp.peak.rate = h->pipe_node_shaper_rate;
335 psp.peak.size = TOKEN_BUCKET_SIZE;
336 psp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
338 /* Pipe Node Parameters */
341 level_id = TM_NODE_LEVEL_PIPE;
342 pnp.nonleaf.n_sp_priorities = 4;
343 pnp.stats_mask = STATS_MASK_DEFAULT;
345 /* Add Shaper Profiles and Nodes to TM Hierarchy */
346 for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
347 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
348 if (rte_tm_shaper_profile_add(port_id,
349 shaper_profile_id, &psp, error)) {
350 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
351 __func__, error->type, error->message,
355 pnp.shaper_profile_id = shaper_profile_id;
356 pipe_parent_node_id = h->subport_node_id[i];
357 h->pipe_node_id[i][j] = PIPE_NODES_START_ID +
358 (i * PIPE_NODES_PER_SUBPORT) + j;
360 if (rte_tm_node_add(port_id,
361 h->pipe_node_id[i][j],
363 priority, weight, level_id,
366 printf("%s ERROR(%d)-%s!(node %u,parent %u )\n",
370 h->pipe_node_id[i][j],
371 pipe_parent_node_id);
379 h->n_shapers = shaper_profile_id;
381 printf(" Pipe nodes added (Start id %u, Count %u, level %u)\n",
382 h->pipe_node_id[0][0], NUM_PIPE_NODES, level_id);
388 softport_tm_tc_node_add(portid_t port_id,
389 struct tm_hierarchy *h,
390 struct rte_tm_error *error)
392 uint32_t tc_parent_node_id;
393 struct rte_tm_node_params tnp;
394 struct rte_tm_shaper_params tsp, tssp;
395 uint32_t shared_shaper_profile_id[TC_NODES_PER_PIPE];
396 uint32_t priority, weight, level_id, shaper_profile_id;
397 uint32_t pos, n_tc_nodes, i, j, k;
399 memset(&tsp, 0, sizeof(struct rte_tm_shaper_params));
400 memset(&tssp, 0, sizeof(struct rte_tm_shaper_params));
401 memset(&tnp, 0, sizeof(struct rte_tm_node_params));
403 shaper_profile_id = h->n_shapers;
405 /* Private Shaper Profile (TC) Parameters */
406 tsp.peak.rate = h->tc_node_shaper_rate;
407 tsp.peak.size = TOKEN_BUCKET_SIZE;
408 tsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
410 /* Shared Shaper Profile (TC) Parameters */
411 tssp.peak.rate = h->tc_node_shared_shaper_rate;
412 tssp.peak.size = TOKEN_BUCKET_SIZE;
413 tssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
415 /* TC Node Parameters */
417 level_id = TM_NODE_LEVEL_TC;
418 tnp.n_shared_shapers = 1;
419 tnp.nonleaf.n_sp_priorities = 1;
420 tnp.stats_mask = STATS_MASK_DEFAULT;
422 /* Add Shared Shaper Profiles to TM Hierarchy */
423 for (i = 0; i < TC_NODES_PER_PIPE; i++) {
424 shared_shaper_profile_id[i] = shaper_profile_id;
426 if (rte_tm_shaper_profile_add(port_id,
427 shared_shaper_profile_id[i], &tssp, error)) {
428 printf("%s ERROR(%d)-%s!(Shared shaper profileid %u)\n",
429 __func__, error->type, error->message,
430 shared_shaper_profile_id[i]);
434 if (rte_tm_shared_shaper_add_update(port_id, i,
435 shared_shaper_profile_id[i], error)) {
436 printf("%s ERROR(%d)-%s!(Shared shaper id %u)\n",
437 __func__, error->type, error->message, i);
444 /* Add Shaper Profiles and Nodes to TM Hierarchy */
446 for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
447 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
448 for (k = 0; k < TC_NODES_PER_PIPE ; k++) {
450 tc_parent_node_id = h->pipe_node_id[i][j];
451 tnp.shared_shaper_id =
452 (uint32_t *)calloc(1, sizeof(uint32_t));
453 if (tnp.shared_shaper_id == NULL) {
454 printf("Shared shaper mem alloc err\n");
457 tnp.shared_shaper_id[0] = k;
458 pos = j + (i * PIPE_NODES_PER_SUBPORT);
459 h->tc_node_id[pos][k] =
460 TC_NODES_START_ID + n_tc_nodes;
462 if (rte_tm_shaper_profile_add(port_id,
463 shaper_profile_id, &tsp, error)) {
464 printf("%s ERROR(%d)-%s!(shaper %u)\n",
465 __func__, error->type,
469 free(tnp.shared_shaper_id);
472 tnp.shaper_profile_id = shaper_profile_id;
473 if (rte_tm_node_add(port_id,
474 h->tc_node_id[pos][k],
479 printf("%s ERROR(%d)-%s!(node id %u)\n",
483 h->tc_node_id[pos][k]);
485 free(tnp.shared_shaper_id);
494 h->n_shapers = shaper_profile_id;
496 printf(" TC nodes added (Start id %u, Count %u, level %u)\n",
497 h->tc_node_id[0][0], n_tc_nodes, level_id);
503 softport_tm_queue_node_add(portid_t port_id, struct tm_hierarchy *h,
504 struct rte_tm_error *error)
506 uint32_t queue_parent_node_id;
507 struct rte_tm_node_params qnp;
508 uint32_t priority, weight, level_id, pos;
509 uint32_t n_queue_nodes, i, j, k;
511 memset(&qnp, 0, sizeof(struct rte_tm_node_params));
513 /* Queue Node Parameters */
516 level_id = TM_NODE_LEVEL_QUEUE;
517 qnp.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
518 qnp.leaf.cman = RTE_TM_CMAN_TAIL_DROP;
519 qnp.stats_mask = STATS_MASK_QUEUE;
521 /* Add Queue Nodes to TM Hierarchy */
523 for (i = 0; i < NUM_PIPE_NODES; i++) {
524 for (j = 0; j < TC_NODES_PER_PIPE; j++) {
525 queue_parent_node_id = h->tc_node_id[i][j];
526 for (k = 0; k < QUEUE_NODES_PER_TC; k++) {
527 pos = j + (i * TC_NODES_PER_PIPE);
528 h->queue_node_id[pos][k] = n_queue_nodes;
529 if (rte_tm_node_add(port_id,
530 h->queue_node_id[pos][k],
531 queue_parent_node_id,
536 printf("%s ERROR(%d)-%s!(node %u)\n",
540 h->queue_node_id[pos][k]);
548 printf(" Queue nodes added (Start id %u, Count %u, level %u)\n",
549 h->queue_node_id[0][0], n_queue_nodes, level_id);
555 softport_tm_hierarchy_specify(portid_t port_id,
556 struct rte_tm_error *error)
559 struct tm_hierarchy h;
562 memset(&h, 0, sizeof(struct tm_hierarchy));
564 /* TM hierarchy shapers rate */
565 status = set_tm_hiearchy_nodes_shaper_rate(port_id, &h);
569 /* Add root node (level 0) */
570 status = softport_tm_root_node_add(port_id, &h, error);
574 /* Add subport node (level 1) */
575 status = softport_tm_subport_node_add(port_id, &h, error);
579 /* Add pipe nodes (level 2) */
580 status = softport_tm_pipe_node_add(port_id, &h, error);
584 /* Add traffic class nodes (level 3) */
585 status = softport_tm_tc_node_add(port_id, &h, error);
589 /* Add queue nodes (level 4) */
590 status = softport_tm_queue_node_add(port_id, &h, error);
598 * Softnic TM default configuration
601 softnic_tm_default_config(portid_t pi)
603 struct rte_port *port = &ports[pi];
604 struct rte_tm_error error;
608 rte_eth_dev_stop(pi);
610 /* TM hierarchy specification */
611 status = softport_tm_hierarchy_specify(pi, &error);
613 printf(" TM Hierarchy built error(%d) - %s\n",
614 error.type, error.message);
617 printf("\n TM Hierarchy Specified!\n");
619 /* TM hierarchy commit */
620 status = rte_tm_hierarchy_commit(pi, 0, &error);
622 printf(" Hierarchy commit error(%d) - %s\n",
623 error.type, error.message);
626 printf(" Hierarchy Committed (port %u)!\n", pi);
629 status = rte_eth_dev_start(pi);
631 printf("\n Port %u start error!\n", pi);
635 /* Reset the default hierarchy flag */
636 port->softport.default_tm_hierarchy_enable = 0;
640 * Softnic forwarding init
643 softnic_fwd_begin(portid_t pi)
645 struct rte_port *port = &ports[pi];
646 uint32_t lcore, fwd_core_present = 0, softnic_run_launch = 0;
649 softnic_fwd_lcore = port->softport.fwd_lcore_arg[0];
650 softnic_port_id = pi;
652 /* Launch softnic_run function on lcores */
653 for (lcore = 0; lcore < RTE_MAX_LCORE; lcore++) {
654 if (!rte_lcore_is_enabled(lcore))
657 if (lcore == rte_get_master_lcore())
660 if (fwd_core_present == 0) {
665 status = rte_eal_remote_launch(softnic_begin, NULL, lcore);
667 printf("softnic launch on lcore %u failed (%d)\n",
670 softnic_run_launch = 1;
673 if (!softnic_run_launch)
674 softnic_fwd_engine.packet_fwd = softnic_fwd_run;
676 /* Softnic TM default configuration */
677 if (port->softport.default_tm_hierarchy_enable == 1)
678 softnic_tm_default_config(pi);
681 struct fwd_engine softnic_fwd_engine = {
682 .fwd_mode_name = "softnic",
683 .port_fwd_begin = softnic_fwd_begin,
684 .port_fwd_end = NULL,
685 .packet_fwd = softnic_fwd,