1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
7 #include <rte_cycles.h>
9 #include <rte_malloc.h>
10 #include <rte_ethdev.h>
12 #include <rte_meter.h>
13 #include <rte_eth_softnic.h>
18 #define SUBPORT_NODES_PER_PORT 1
19 #define PIPE_NODES_PER_SUBPORT 4096
20 #define TC_NODES_PER_PIPE 4
21 #define QUEUE_NODES_PER_TC 4
23 #define NUM_PIPE_NODES \
24 (SUBPORT_NODES_PER_PORT * PIPE_NODES_PER_SUBPORT)
26 #define NUM_TC_NODES \
27 (NUM_PIPE_NODES * TC_NODES_PER_PIPE)
29 #define ROOT_NODE_ID 1000000
30 #define SUBPORT_NODES_START_ID 900000
31 #define PIPE_NODES_START_ID 800000
32 #define TC_NODES_START_ID 700000
34 #define STATS_MASK_DEFAULT \
35 (RTE_TM_STATS_N_PKTS | \
36 RTE_TM_STATS_N_BYTES | \
37 RTE_TM_STATS_N_PKTS_GREEN_DROPPED | \
38 RTE_TM_STATS_N_BYTES_GREEN_DROPPED)
40 #define STATS_MASK_QUEUE \
41 (STATS_MASK_DEFAULT | \
42 RTE_TM_STATS_N_PKTS_QUEUED)
44 #define BYTES_IN_MBPS (1000 * 1000 / 8)
45 #define TOKEN_BUCKET_SIZE 1000000
47 /* TM Hierarchy Levels */
48 enum tm_hierarchy_level {
49 TM_NODE_LEVEL_PORT = 0,
50 TM_NODE_LEVEL_SUBPORT,
59 uint32_t root_node_id;
60 uint32_t subport_node_id[SUBPORT_NODES_PER_PORT];
61 uint32_t pipe_node_id[SUBPORT_NODES_PER_PORT][PIPE_NODES_PER_SUBPORT];
62 uint32_t tc_node_id[NUM_PIPE_NODES][TC_NODES_PER_PIPE];
63 uint32_t queue_node_id[NUM_TC_NODES][QUEUE_NODES_PER_TC];
65 /* TM Hierarchy Nodes Shaper Rates */
66 uint32_t root_node_shaper_rate;
67 uint32_t subport_node_shaper_rate;
68 uint32_t pipe_node_shaper_rate;
69 uint32_t tc_node_shaper_rate;
70 uint32_t tc_node_shared_shaper_rate;
75 static struct fwd_lcore *softnic_fwd_lcore;
76 static uint16_t softnic_port_id;
77 struct fwd_engine softnic_fwd_engine;
80 * Softnic packet forward
83 softnic_fwd(struct fwd_stream *fs)
85 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
90 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
96 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
97 start_tsc = rte_rdtsc();
100 /* Packets Receive */
101 nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
102 pkts_burst, nb_pkt_per_burst);
103 fs->rx_packets += nb_rx;
105 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
106 fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
109 nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
112 /* Retry if necessary */
113 if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
115 while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
116 rte_delay_us(burst_tx_delay_time);
117 nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
118 &pkts_burst[nb_tx], nb_rx - nb_tx);
121 fs->tx_packets += nb_tx;
123 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
124 fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
127 if (unlikely(nb_tx < nb_rx)) {
128 fs->fwd_dropped += (nb_rx - nb_tx);
130 rte_pktmbuf_free(pkts_burst[nb_tx]);
131 } while (++nb_tx < nb_rx);
133 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
134 end_tsc = rte_rdtsc();
135 core_cycles = (end_tsc - start_tsc);
136 fs->core_cycles = (uint64_t) (fs->core_cycles + core_cycles);
141 softnic_fwd_run(struct fwd_stream *fs)
143 rte_pmd_softnic_run(softnic_port_id);
151 softnic_begin(void *arg __rte_unused)
154 if (!softnic_fwd_lcore->stopped)
160 rte_pmd_softnic_run(softnic_port_id);
161 } while (!softnic_fwd_lcore->stopped);
167 set_tm_hiearchy_nodes_shaper_rate(portid_t port_id,
168 struct tm_hierarchy *h)
170 struct rte_eth_link link_params;
171 uint64_t tm_port_rate;
173 memset(&link_params, 0, sizeof(link_params));
175 rte_eth_link_get(port_id, &link_params);
176 tm_port_rate = (uint64_t)ETH_SPEED_NUM_10G * BYTES_IN_MBPS;
178 if (tm_port_rate > UINT32_MAX)
179 tm_port_rate = UINT32_MAX;
181 /* Set tm hierarchy shapers rate */
182 h->root_node_shaper_rate = tm_port_rate;
183 h->subport_node_shaper_rate =
184 tm_port_rate / SUBPORT_NODES_PER_PORT;
185 h->pipe_node_shaper_rate
186 = h->subport_node_shaper_rate / PIPE_NODES_PER_SUBPORT;
187 h->tc_node_shaper_rate = h->pipe_node_shaper_rate;
188 h->tc_node_shared_shaper_rate = h->subport_node_shaper_rate;
192 softport_tm_root_node_add(portid_t port_id, struct tm_hierarchy *h,
193 struct rte_tm_error *error)
195 struct rte_tm_node_params rnp;
196 struct rte_tm_shaper_params rsp;
197 uint32_t priority, weight, level_id, shaper_profile_id;
199 memset(&rsp, 0, sizeof(struct rte_tm_shaper_params));
200 memset(&rnp, 0, sizeof(struct rte_tm_node_params));
202 /* Shaper profile Parameters */
203 rsp.peak.rate = h->root_node_shaper_rate;
204 rsp.peak.size = TOKEN_BUCKET_SIZE;
205 rsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
206 shaper_profile_id = 0;
208 if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
210 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
211 __func__, error->type, error->message,
216 /* Root Node Parameters */
217 h->root_node_id = ROOT_NODE_ID;
220 level_id = TM_NODE_LEVEL_PORT;
221 rnp.shaper_profile_id = shaper_profile_id;
222 rnp.nonleaf.n_sp_priorities = 1;
223 rnp.stats_mask = STATS_MASK_DEFAULT;
225 /* Add Node to TM Hierarchy */
226 if (rte_tm_node_add(port_id, h->root_node_id, RTE_TM_NODE_ID_NULL,
227 priority, weight, level_id, &rnp, error)) {
228 printf("%s ERROR(%d)-%s!(node_id %u, parent_id %u, level %u)\n",
229 __func__, error->type, error->message,
230 h->root_node_id, RTE_TM_NODE_ID_NULL,
237 printf(" Root node added (Start id %u, Count %u, level %u)\n",
238 h->root_node_id, 1, level_id);
244 softport_tm_subport_node_add(portid_t port_id,
245 struct tm_hierarchy *h,
246 struct rte_tm_error *error)
248 uint32_t subport_parent_node_id, subport_node_id = 0;
249 struct rte_tm_node_params snp;
250 struct rte_tm_shaper_params ssp;
251 uint32_t priority, weight, level_id, shaper_profile_id;
254 memset(&ssp, 0, sizeof(struct rte_tm_shaper_params));
255 memset(&snp, 0, sizeof(struct rte_tm_node_params));
257 shaper_profile_id = h->n_shapers;
259 /* Add Shaper Profile to TM Hierarchy */
260 for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
261 ssp.peak.rate = h->subport_node_shaper_rate;
262 ssp.peak.size = TOKEN_BUCKET_SIZE;
263 ssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
265 if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
267 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
268 __func__, error->type, error->message,
273 /* Node Parameters */
274 h->subport_node_id[i] = SUBPORT_NODES_START_ID + i;
275 subport_parent_node_id = h->root_node_id;
278 level_id = TM_NODE_LEVEL_SUBPORT;
279 snp.shaper_profile_id = shaper_profile_id;
280 snp.nonleaf.n_sp_priorities = 1;
281 snp.stats_mask = STATS_MASK_DEFAULT;
283 /* Add Node to TM Hiearchy */
284 if (rte_tm_node_add(port_id,
285 h->subport_node_id[i],
286 subport_parent_node_id,
291 printf("%s ERROR(%d)-%s!(node %u,parent %u,level %u)\n",
295 h->subport_node_id[i],
296 subport_parent_node_id,
304 h->n_shapers = shaper_profile_id;
306 printf(" Subport nodes added (Start id %u, Count %u, level %u)\n",
307 h->subport_node_id[0], SUBPORT_NODES_PER_PORT, level_id);
313 softport_tm_pipe_node_add(portid_t port_id,
314 struct tm_hierarchy *h,
315 struct rte_tm_error *error)
317 uint32_t pipe_parent_node_id;
318 struct rte_tm_node_params pnp;
319 struct rte_tm_shaper_params psp;
320 uint32_t priority, weight, level_id, shaper_profile_id;
323 memset(&psp, 0, sizeof(struct rte_tm_shaper_params));
324 memset(&pnp, 0, sizeof(struct rte_tm_node_params));
326 shaper_profile_id = h->n_shapers;
328 /* Shaper Profile Parameters */
329 psp.peak.rate = h->pipe_node_shaper_rate;
330 psp.peak.size = TOKEN_BUCKET_SIZE;
331 psp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
333 /* Pipe Node Parameters */
336 level_id = TM_NODE_LEVEL_PIPE;
337 pnp.nonleaf.n_sp_priorities = 4;
338 pnp.stats_mask = STATS_MASK_DEFAULT;
340 /* Add Shaper Profiles and Nodes to TM Hierarchy */
341 for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
342 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
343 if (rte_tm_shaper_profile_add(port_id,
344 shaper_profile_id, &psp, error)) {
345 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
346 __func__, error->type, error->message,
350 pnp.shaper_profile_id = shaper_profile_id;
351 pipe_parent_node_id = h->subport_node_id[i];
352 h->pipe_node_id[i][j] = PIPE_NODES_START_ID +
353 (i * PIPE_NODES_PER_SUBPORT) + j;
355 if (rte_tm_node_add(port_id,
356 h->pipe_node_id[i][j],
358 priority, weight, level_id,
361 printf("%s ERROR(%d)-%s!(node %u,parent %u )\n",
365 h->pipe_node_id[i][j],
366 pipe_parent_node_id);
374 h->n_shapers = shaper_profile_id;
376 printf(" Pipe nodes added (Start id %u, Count %u, level %u)\n",
377 h->pipe_node_id[0][0], NUM_PIPE_NODES, level_id);
383 softport_tm_tc_node_add(portid_t port_id,
384 struct tm_hierarchy *h,
385 struct rte_tm_error *error)
387 uint32_t tc_parent_node_id;
388 struct rte_tm_node_params tnp;
389 struct rte_tm_shaper_params tsp, tssp;
390 uint32_t shared_shaper_profile_id[TC_NODES_PER_PIPE];
391 uint32_t priority, weight, level_id, shaper_profile_id;
392 uint32_t pos, n_tc_nodes, i, j, k;
394 memset(&tsp, 0, sizeof(struct rte_tm_shaper_params));
395 memset(&tssp, 0, sizeof(struct rte_tm_shaper_params));
396 memset(&tnp, 0, sizeof(struct rte_tm_node_params));
398 shaper_profile_id = h->n_shapers;
400 /* Private Shaper Profile (TC) Parameters */
401 tsp.peak.rate = h->tc_node_shaper_rate;
402 tsp.peak.size = TOKEN_BUCKET_SIZE;
403 tsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
405 /* Shared Shaper Profile (TC) Parameters */
406 tssp.peak.rate = h->tc_node_shared_shaper_rate;
407 tssp.peak.size = TOKEN_BUCKET_SIZE;
408 tssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
410 /* TC Node Parameters */
412 level_id = TM_NODE_LEVEL_TC;
413 tnp.n_shared_shapers = 1;
414 tnp.nonleaf.n_sp_priorities = 1;
415 tnp.stats_mask = STATS_MASK_DEFAULT;
417 /* Add Shared Shaper Profiles to TM Hierarchy */
418 for (i = 0; i < TC_NODES_PER_PIPE; i++) {
419 shared_shaper_profile_id[i] = shaper_profile_id;
421 if (rte_tm_shaper_profile_add(port_id,
422 shared_shaper_profile_id[i], &tssp, error)) {
423 printf("%s ERROR(%d)-%s!(Shared shaper profileid %u)\n",
424 __func__, error->type, error->message,
425 shared_shaper_profile_id[i]);
429 if (rte_tm_shared_shaper_add_update(port_id, i,
430 shared_shaper_profile_id[i], error)) {
431 printf("%s ERROR(%d)-%s!(Shared shaper id %u)\n",
432 __func__, error->type, error->message, i);
439 /* Add Shaper Profiles and Nodes to TM Hierarchy */
441 for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
442 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
443 for (k = 0; k < TC_NODES_PER_PIPE ; k++) {
445 tc_parent_node_id = h->pipe_node_id[i][j];
446 tnp.shared_shaper_id =
447 (uint32_t *)calloc(1, sizeof(uint32_t));
448 if (tnp.shared_shaper_id == NULL) {
449 printf("Shared shaper mem alloc err\n");
452 tnp.shared_shaper_id[0] = k;
453 pos = j + (i * PIPE_NODES_PER_SUBPORT);
454 h->tc_node_id[pos][k] =
455 TC_NODES_START_ID + n_tc_nodes;
457 if (rte_tm_shaper_profile_add(port_id,
458 shaper_profile_id, &tsp, error)) {
459 printf("%s ERROR(%d)-%s!(shaper %u)\n",
460 __func__, error->type,
466 tnp.shaper_profile_id = shaper_profile_id;
467 if (rte_tm_node_add(port_id,
468 h->tc_node_id[pos][k],
473 printf("%s ERROR(%d)-%s!(node id %u)\n",
477 h->tc_node_id[pos][k]);
487 h->n_shapers = shaper_profile_id;
489 printf(" TC nodes added (Start id %u, Count %u, level %u)\n",
490 h->tc_node_id[0][0], n_tc_nodes, level_id);
496 softport_tm_queue_node_add(portid_t port_id, struct tm_hierarchy *h,
497 struct rte_tm_error *error)
499 uint32_t queue_parent_node_id;
500 struct rte_tm_node_params qnp;
501 uint32_t priority, weight, level_id, pos;
502 uint32_t n_queue_nodes, i, j, k;
504 memset(&qnp, 0, sizeof(struct rte_tm_node_params));
506 /* Queue Node Parameters */
509 level_id = TM_NODE_LEVEL_QUEUE;
510 qnp.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
511 qnp.leaf.cman = RTE_TM_CMAN_TAIL_DROP;
512 qnp.stats_mask = STATS_MASK_QUEUE;
514 /* Add Queue Nodes to TM Hierarchy */
516 for (i = 0; i < NUM_PIPE_NODES; i++) {
517 for (j = 0; j < TC_NODES_PER_PIPE; j++) {
518 queue_parent_node_id = h->tc_node_id[i][j];
519 for (k = 0; k < QUEUE_NODES_PER_TC; k++) {
520 pos = j + (i * TC_NODES_PER_PIPE);
521 h->queue_node_id[pos][k] = n_queue_nodes;
522 if (rte_tm_node_add(port_id,
523 h->queue_node_id[pos][k],
524 queue_parent_node_id,
529 printf("%s ERROR(%d)-%s!(node %u)\n",
533 h->queue_node_id[pos][k]);
541 printf(" Queue nodes added (Start id %u, Count %u, level %u)\n",
542 h->queue_node_id[0][0], n_queue_nodes, level_id);
548 softport_tm_hierarchy_specify(portid_t port_id,
549 struct rte_tm_error *error)
552 struct tm_hierarchy h;
555 memset(&h, 0, sizeof(struct tm_hierarchy));
557 /* TM hierarchy shapers rate */
558 set_tm_hiearchy_nodes_shaper_rate(port_id, &h);
560 /* Add root node (level 0) */
561 status = softport_tm_root_node_add(port_id, &h, error);
565 /* Add subport node (level 1) */
566 status = softport_tm_subport_node_add(port_id, &h, error);
570 /* Add pipe nodes (level 2) */
571 status = softport_tm_pipe_node_add(port_id, &h, error);
575 /* Add traffic class nodes (level 3) */
576 status = softport_tm_tc_node_add(port_id, &h, error);
580 /* Add queue nodes (level 4) */
581 status = softport_tm_queue_node_add(port_id, &h, error);
589 * Softnic TM default configuration
592 softnic_tm_default_config(portid_t pi)
594 struct rte_port *port = &ports[pi];
595 struct rte_tm_error error;
599 rte_eth_dev_stop(pi);
601 /* TM hierarchy specification */
602 status = softport_tm_hierarchy_specify(pi, &error);
604 printf(" TM Hierarchy built error(%d) - %s\n",
605 error.type, error.message);
608 printf("\n TM Hierarchy Specified!\n");
610 /* TM hierarchy commit */
611 status = rte_tm_hierarchy_commit(pi, 0, &error);
613 printf(" Hierarchy commit error(%d) - %s\n",
614 error.type, error.message);
617 printf(" Hierarchy Committed (port %u)!\n", pi);
620 status = rte_eth_dev_start(pi);
622 printf("\n Port %u start error!\n", pi);
626 /* Reset the default hierarchy flag */
627 port->softport.default_tm_hierarchy_enable = 0;
631 * Softnic forwarding init
634 softnic_fwd_begin(portid_t pi)
636 struct rte_port *port = &ports[pi];
637 uint32_t lcore, fwd_core_present = 0, softnic_run_launch = 0;
640 softnic_fwd_lcore = port->softport.fwd_lcore_arg[0];
641 softnic_port_id = pi;
643 /* Launch softnic_run function on lcores */
644 for (lcore = 0; lcore < RTE_MAX_LCORE; lcore++) {
645 if (!rte_lcore_is_enabled(lcore))
648 if (lcore == rte_get_master_lcore())
651 if (fwd_core_present == 0) {
656 status = rte_eal_remote_launch(softnic_begin, NULL, lcore);
658 printf("softnic launch on lcore %u failed (%d)\n",
661 softnic_run_launch = 1;
664 if (!softnic_run_launch)
665 softnic_fwd_engine.packet_fwd = softnic_fwd_run;
667 /* Softnic TM default configuration */
668 if (port->softport.default_tm_hierarchy_enable == 1)
669 softnic_tm_default_config(pi);
672 struct fwd_engine softnic_fwd_engine = {
673 .fwd_mode_name = "softnic",
674 .port_fwd_begin = softnic_fwd_begin,
675 .port_fwd_end = NULL,
676 .packet_fwd = softnic_fwd,