1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
7 #include <rte_cycles.h>
9 #include <rte_malloc.h>
10 #include <rte_ethdev.h>
12 #include <rte_meter.h>
13 #include <rte_eth_softnic.h>
18 #define SUBPORT_NODES_PER_PORT 1
19 #define PIPE_NODES_PER_SUBPORT 4096
20 #define TC_NODES_PER_PIPE 4
21 #define QUEUE_NODES_PER_TC 4
23 #define NUM_PIPE_NODES \
24 (SUBPORT_NODES_PER_PORT * PIPE_NODES_PER_SUBPORT)
26 #define NUM_TC_NODES \
27 (NUM_PIPE_NODES * TC_NODES_PER_PIPE)
29 #define ROOT_NODE_ID 1000000
30 #define SUBPORT_NODES_START_ID 900000
31 #define PIPE_NODES_START_ID 800000
32 #define TC_NODES_START_ID 700000
34 #define STATS_MASK_DEFAULT \
35 (RTE_TM_STATS_N_PKTS | \
36 RTE_TM_STATS_N_BYTES | \
37 RTE_TM_STATS_N_PKTS_GREEN_DROPPED | \
38 RTE_TM_STATS_N_BYTES_GREEN_DROPPED)
40 #define STATS_MASK_QUEUE \
41 (STATS_MASK_DEFAULT | \
42 RTE_TM_STATS_N_PKTS_QUEUED)
44 #define BYTES_IN_MBPS (1000 * 1000 / 8)
45 #define TOKEN_BUCKET_SIZE 1000000
47 /* TM Hierarchy Levels */
48 enum tm_hierarchy_level {
49 TM_NODE_LEVEL_PORT = 0,
50 TM_NODE_LEVEL_SUBPORT,
59 uint32_t root_node_id;
60 uint32_t subport_node_id[SUBPORT_NODES_PER_PORT];
61 uint32_t pipe_node_id[SUBPORT_NODES_PER_PORT][PIPE_NODES_PER_SUBPORT];
62 uint32_t tc_node_id[NUM_PIPE_NODES][TC_NODES_PER_PIPE];
63 uint32_t queue_node_id[NUM_TC_NODES][QUEUE_NODES_PER_TC];
65 /* TM Hierarchy Nodes Shaper Rates */
66 uint32_t root_node_shaper_rate;
67 uint32_t subport_node_shaper_rate;
68 uint32_t pipe_node_shaper_rate;
69 uint32_t tc_node_shaper_rate;
70 uint32_t tc_node_shared_shaper_rate;
75 static struct fwd_lcore *softnic_fwd_lcore;
76 static uint16_t softnic_port_id;
77 struct fwd_engine softnic_fwd_engine;
80 * Softnic packet forward
83 softnic_fwd(struct fwd_stream *fs)
85 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
90 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
96 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
97 start_tsc = rte_rdtsc();
100 /* Packets Receive */
101 nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
102 pkts_burst, nb_pkt_per_burst);
103 fs->rx_packets += nb_rx;
105 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
106 fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
109 nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
112 /* Retry if necessary */
113 if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
115 while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
116 rte_delay_us(burst_tx_delay_time);
117 nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
118 &pkts_burst[nb_tx], nb_rx - nb_tx);
121 fs->tx_packets += nb_tx;
123 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
124 fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
127 if (unlikely(nb_tx < nb_rx)) {
128 fs->fwd_dropped += (nb_rx - nb_tx);
130 rte_pktmbuf_free(pkts_burst[nb_tx]);
131 } while (++nb_tx < nb_rx);
133 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
134 end_tsc = rte_rdtsc();
135 core_cycles = (end_tsc - start_tsc);
136 fs->core_cycles = (uint64_t) (fs->core_cycles + core_cycles);
141 softnic_fwd_run(struct fwd_stream *fs)
143 rte_pmd_softnic_run(softnic_port_id);
151 softnic_begin(void *arg __rte_unused)
154 if (!softnic_fwd_lcore->stopped)
160 rte_pmd_softnic_run(softnic_port_id);
161 } while (!softnic_fwd_lcore->stopped);
167 set_tm_hiearchy_nodes_shaper_rate(portid_t port_id,
168 struct tm_hierarchy *h)
170 struct rte_eth_link link_params;
171 uint64_t tm_port_rate;
173 memset(&link_params, 0, sizeof(link_params));
175 rte_eth_link_get(port_id, &link_params);
176 tm_port_rate = (uint64_t)ETH_SPEED_NUM_10G * BYTES_IN_MBPS;
178 /* Set tm hierarchy shapers rate */
179 h->root_node_shaper_rate = tm_port_rate;
180 h->subport_node_shaper_rate =
181 tm_port_rate / SUBPORT_NODES_PER_PORT;
182 h->pipe_node_shaper_rate
183 = h->subport_node_shaper_rate / PIPE_NODES_PER_SUBPORT;
184 h->tc_node_shaper_rate = h->pipe_node_shaper_rate;
185 h->tc_node_shared_shaper_rate = h->subport_node_shaper_rate;
189 softport_tm_root_node_add(portid_t port_id, struct tm_hierarchy *h,
190 struct rte_tm_error *error)
192 struct rte_tm_node_params rnp;
193 struct rte_tm_shaper_params rsp;
194 uint32_t priority, weight, level_id, shaper_profile_id;
196 memset(&rsp, 0, sizeof(struct rte_tm_shaper_params));
197 memset(&rnp, 0, sizeof(struct rte_tm_node_params));
199 /* Shaper profile Parameters */
200 rsp.peak.rate = h->root_node_shaper_rate;
201 rsp.peak.size = TOKEN_BUCKET_SIZE;
202 rsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
203 shaper_profile_id = 0;
205 if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
207 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
208 __func__, error->type, error->message,
213 /* Root Node Parameters */
214 h->root_node_id = ROOT_NODE_ID;
217 level_id = TM_NODE_LEVEL_PORT;
218 rnp.shaper_profile_id = shaper_profile_id;
219 rnp.nonleaf.n_sp_priorities = 1;
220 rnp.stats_mask = STATS_MASK_DEFAULT;
222 /* Add Node to TM Hierarchy */
223 if (rte_tm_node_add(port_id, h->root_node_id, RTE_TM_NODE_ID_NULL,
224 priority, weight, level_id, &rnp, error)) {
225 printf("%s ERROR(%d)-%s!(node_id %u, parent_id %u, level %u)\n",
226 __func__, error->type, error->message,
227 h->root_node_id, RTE_TM_NODE_ID_NULL,
234 printf(" Root node added (Start id %u, Count %u, level %u)\n",
235 h->root_node_id, 1, level_id);
241 softport_tm_subport_node_add(portid_t port_id,
242 struct tm_hierarchy *h,
243 struct rte_tm_error *error)
245 uint32_t subport_parent_node_id, subport_node_id = 0;
246 struct rte_tm_node_params snp;
247 struct rte_tm_shaper_params ssp;
248 uint32_t priority, weight, level_id, shaper_profile_id;
251 memset(&ssp, 0, sizeof(struct rte_tm_shaper_params));
252 memset(&snp, 0, sizeof(struct rte_tm_node_params));
254 shaper_profile_id = h->n_shapers;
256 /* Add Shaper Profile to TM Hierarchy */
257 for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
258 ssp.peak.rate = h->subport_node_shaper_rate;
259 ssp.peak.size = TOKEN_BUCKET_SIZE;
260 ssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
262 if (rte_tm_shaper_profile_add(port_id, shaper_profile_id,
264 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
265 __func__, error->type, error->message,
270 /* Node Parameters */
271 h->subport_node_id[i] = SUBPORT_NODES_START_ID + i;
272 subport_parent_node_id = h->root_node_id;
275 level_id = TM_NODE_LEVEL_SUBPORT;
276 snp.shaper_profile_id = shaper_profile_id;
277 snp.nonleaf.n_sp_priorities = 1;
278 snp.stats_mask = STATS_MASK_DEFAULT;
280 /* Add Node to TM Hiearchy */
281 if (rte_tm_node_add(port_id,
282 h->subport_node_id[i],
283 subport_parent_node_id,
288 printf("%s ERROR(%d)-%s!(node %u,parent %u,level %u)\n",
292 h->subport_node_id[i],
293 subport_parent_node_id,
301 h->n_shapers = shaper_profile_id;
303 printf(" Subport nodes added (Start id %u, Count %u, level %u)\n",
304 h->subport_node_id[0], SUBPORT_NODES_PER_PORT, level_id);
310 softport_tm_pipe_node_add(portid_t port_id,
311 struct tm_hierarchy *h,
312 struct rte_tm_error *error)
314 uint32_t pipe_parent_node_id;
315 struct rte_tm_node_params pnp;
316 struct rte_tm_shaper_params psp;
317 uint32_t priority, weight, level_id, shaper_profile_id;
320 memset(&psp, 0, sizeof(struct rte_tm_shaper_params));
321 memset(&pnp, 0, sizeof(struct rte_tm_node_params));
323 shaper_profile_id = h->n_shapers;
325 /* Shaper Profile Parameters */
326 psp.peak.rate = h->pipe_node_shaper_rate;
327 psp.peak.size = TOKEN_BUCKET_SIZE;
328 psp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
330 /* Pipe Node Parameters */
333 level_id = TM_NODE_LEVEL_PIPE;
334 pnp.nonleaf.n_sp_priorities = 4;
335 pnp.stats_mask = STATS_MASK_DEFAULT;
337 /* Add Shaper Profiles and Nodes to TM Hierarchy */
338 for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
339 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
340 if (rte_tm_shaper_profile_add(port_id,
341 shaper_profile_id, &psp, error)) {
342 printf("%s ERROR(%d)-%s!(shaper_id %u)\n ",
343 __func__, error->type, error->message,
347 pnp.shaper_profile_id = shaper_profile_id;
348 pipe_parent_node_id = h->subport_node_id[i];
349 h->pipe_node_id[i][j] = PIPE_NODES_START_ID +
350 (i * PIPE_NODES_PER_SUBPORT) + j;
352 if (rte_tm_node_add(port_id,
353 h->pipe_node_id[i][j],
355 priority, weight, level_id,
358 printf("%s ERROR(%d)-%s!(node %u,parent %u )\n",
362 h->pipe_node_id[i][j],
363 pipe_parent_node_id);
371 h->n_shapers = shaper_profile_id;
373 printf(" Pipe nodes added (Start id %u, Count %u, level %u)\n",
374 h->pipe_node_id[0][0], NUM_PIPE_NODES, level_id);
380 softport_tm_tc_node_add(portid_t port_id,
381 struct tm_hierarchy *h,
382 struct rte_tm_error *error)
384 uint32_t tc_parent_node_id;
385 struct rte_tm_node_params tnp;
386 struct rte_tm_shaper_params tsp, tssp;
387 uint32_t shared_shaper_profile_id[TC_NODES_PER_PIPE];
388 uint32_t priority, weight, level_id, shaper_profile_id;
389 uint32_t pos, n_tc_nodes, i, j, k;
391 memset(&tsp, 0, sizeof(struct rte_tm_shaper_params));
392 memset(&tssp, 0, sizeof(struct rte_tm_shaper_params));
393 memset(&tnp, 0, sizeof(struct rte_tm_node_params));
395 shaper_profile_id = h->n_shapers;
397 /* Private Shaper Profile (TC) Parameters */
398 tsp.peak.rate = h->tc_node_shaper_rate;
399 tsp.peak.size = TOKEN_BUCKET_SIZE;
400 tsp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
402 /* Shared Shaper Profile (TC) Parameters */
403 tssp.peak.rate = h->tc_node_shared_shaper_rate;
404 tssp.peak.size = TOKEN_BUCKET_SIZE;
405 tssp.pkt_length_adjust = RTE_TM_ETH_FRAMING_OVERHEAD_FCS;
407 /* TC Node Parameters */
409 level_id = TM_NODE_LEVEL_TC;
410 tnp.n_shared_shapers = 1;
411 tnp.nonleaf.n_sp_priorities = 1;
412 tnp.stats_mask = STATS_MASK_DEFAULT;
414 /* Add Shared Shaper Profiles to TM Hierarchy */
415 for (i = 0; i < TC_NODES_PER_PIPE; i++) {
416 shared_shaper_profile_id[i] = shaper_profile_id;
418 if (rte_tm_shaper_profile_add(port_id,
419 shared_shaper_profile_id[i], &tssp, error)) {
420 printf("%s ERROR(%d)-%s!(Shared shaper profileid %u)\n",
421 __func__, error->type, error->message,
422 shared_shaper_profile_id[i]);
426 if (rte_tm_shared_shaper_add_update(port_id, i,
427 shared_shaper_profile_id[i], error)) {
428 printf("%s ERROR(%d)-%s!(Shared shaper id %u)\n",
429 __func__, error->type, error->message, i);
436 /* Add Shaper Profiles and Nodes to TM Hierarchy */
438 for (i = 0; i < SUBPORT_NODES_PER_PORT; i++) {
439 for (j = 0; j < PIPE_NODES_PER_SUBPORT; j++) {
440 for (k = 0; k < TC_NODES_PER_PIPE ; k++) {
442 tc_parent_node_id = h->pipe_node_id[i][j];
443 tnp.shared_shaper_id =
444 (uint32_t *)calloc(1, sizeof(uint32_t));
445 if (tnp.shared_shaper_id == NULL) {
446 printf("Shared shaper mem alloc err\n");
449 tnp.shared_shaper_id[0] = k;
450 pos = j + (i * PIPE_NODES_PER_SUBPORT);
451 h->tc_node_id[pos][k] =
452 TC_NODES_START_ID + n_tc_nodes;
454 if (rte_tm_shaper_profile_add(port_id,
455 shaper_profile_id, &tsp, error)) {
456 printf("%s ERROR(%d)-%s!(shaper %u)\n",
457 __func__, error->type,
461 free(tnp.shared_shaper_id);
464 tnp.shaper_profile_id = shaper_profile_id;
465 if (rte_tm_node_add(port_id,
466 h->tc_node_id[pos][k],
471 printf("%s ERROR(%d)-%s!(node id %u)\n",
475 h->tc_node_id[pos][k]);
477 free(tnp.shared_shaper_id);
486 h->n_shapers = shaper_profile_id;
488 printf(" TC nodes added (Start id %u, Count %u, level %u)\n",
489 h->tc_node_id[0][0], n_tc_nodes, level_id);
495 softport_tm_queue_node_add(portid_t port_id, struct tm_hierarchy *h,
496 struct rte_tm_error *error)
498 uint32_t queue_parent_node_id;
499 struct rte_tm_node_params qnp;
500 uint32_t priority, weight, level_id, pos;
501 uint32_t n_queue_nodes, i, j, k;
503 memset(&qnp, 0, sizeof(struct rte_tm_node_params));
505 /* Queue Node Parameters */
508 level_id = TM_NODE_LEVEL_QUEUE;
509 qnp.shaper_profile_id = RTE_TM_SHAPER_PROFILE_ID_NONE;
510 qnp.leaf.cman = RTE_TM_CMAN_TAIL_DROP;
511 qnp.stats_mask = STATS_MASK_QUEUE;
513 /* Add Queue Nodes to TM Hierarchy */
515 for (i = 0; i < NUM_PIPE_NODES; i++) {
516 for (j = 0; j < TC_NODES_PER_PIPE; j++) {
517 queue_parent_node_id = h->tc_node_id[i][j];
518 for (k = 0; k < QUEUE_NODES_PER_TC; k++) {
519 pos = j + (i * TC_NODES_PER_PIPE);
520 h->queue_node_id[pos][k] = n_queue_nodes;
521 if (rte_tm_node_add(port_id,
522 h->queue_node_id[pos][k],
523 queue_parent_node_id,
528 printf("%s ERROR(%d)-%s!(node %u)\n",
532 h->queue_node_id[pos][k]);
540 printf(" Queue nodes added (Start id %u, Count %u, level %u)\n",
541 h->queue_node_id[0][0], n_queue_nodes, level_id);
547 softport_tm_hierarchy_specify(portid_t port_id,
548 struct rte_tm_error *error)
551 struct tm_hierarchy h;
554 memset(&h, 0, sizeof(struct tm_hierarchy));
556 /* TM hierarchy shapers rate */
557 set_tm_hiearchy_nodes_shaper_rate(port_id, &h);
559 /* Add root node (level 0) */
560 status = softport_tm_root_node_add(port_id, &h, error);
564 /* Add subport node (level 1) */
565 status = softport_tm_subport_node_add(port_id, &h, error);
569 /* Add pipe nodes (level 2) */
570 status = softport_tm_pipe_node_add(port_id, &h, error);
574 /* Add traffic class nodes (level 3) */
575 status = softport_tm_tc_node_add(port_id, &h, error);
579 /* Add queue nodes (level 4) */
580 status = softport_tm_queue_node_add(port_id, &h, error);
588 * Softnic TM default configuration
591 softnic_tm_default_config(portid_t pi)
593 struct rte_port *port = &ports[pi];
594 struct rte_tm_error error;
598 rte_eth_dev_stop(pi);
600 /* TM hierarchy specification */
601 status = softport_tm_hierarchy_specify(pi, &error);
603 printf(" TM Hierarchy built error(%d) - %s\n",
604 error.type, error.message);
607 printf("\n TM Hierarchy Specified!\n");
609 /* TM hierarchy commit */
610 status = rte_tm_hierarchy_commit(pi, 0, &error);
612 printf(" Hierarchy commit error(%d) - %s\n",
613 error.type, error.message);
616 printf(" Hierarchy Committed (port %u)!\n", pi);
619 status = rte_eth_dev_start(pi);
621 printf("\n Port %u start error!\n", pi);
625 /* Reset the default hierarchy flag */
626 port->softport.default_tm_hierarchy_enable = 0;
630 * Softnic forwarding init
633 softnic_fwd_begin(portid_t pi)
635 struct rte_port *port = &ports[pi];
636 uint32_t lcore, fwd_core_present = 0, softnic_run_launch = 0;
639 softnic_fwd_lcore = port->softport.fwd_lcore_arg[0];
640 softnic_port_id = pi;
642 /* Launch softnic_run function on lcores */
643 for (lcore = 0; lcore < RTE_MAX_LCORE; lcore++) {
644 if (!rte_lcore_is_enabled(lcore))
647 if (lcore == rte_get_master_lcore())
650 if (fwd_core_present == 0) {
655 status = rte_eal_remote_launch(softnic_begin, NULL, lcore);
657 printf("softnic launch on lcore %u failed (%d)\n",
660 softnic_run_launch = 1;
663 if (!softnic_run_launch)
664 softnic_fwd_engine.packet_fwd = softnic_fwd_run;
666 /* Softnic TM default configuration */
667 if (port->softport.default_tm_hierarchy_enable == 1)
668 softnic_tm_default_config(pi);
671 struct fwd_engine softnic_fwd_engine = {
672 .fwd_mode_name = "softnic",
673 .port_fwd_begin = softnic_fwd_begin,
674 .port_fwd_end = NULL,
675 .packet_fwd = softnic_fwd,