1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
9 #include <rte_cycles.h>
10 #include <rte_launch.h>
11 #include <rte_pause.h>
12 #include <rte_stack.h>
16 #define STACK_NAME "STACK_PERF"
18 #define STACK_SIZE (RTE_MAX_LCORE * MAX_BURST)
21 * Push/pop bulk sizes, marked volatile so they aren't treated as compile-time
24 static volatile unsigned int bulk_sizes[] = {8, MAX_BURST};
26 static uint32_t lcore_barrier;
34 get_two_hyperthreads(struct lcore_pair *lcp)
36 unsigned int socket[2];
40 RTE_LCORE_FOREACH(id[0]) {
41 RTE_LCORE_FOREACH(id[1]) {
44 core[0] = rte_lcore_to_cpu_id(id[0]);
45 core[1] = rte_lcore_to_cpu_id(id[1]);
46 socket[0] = rte_lcore_to_socket_id(id[0]);
47 socket[1] = rte_lcore_to_socket_id(id[1]);
48 if ((core[0] == core[1]) && (socket[0] == socket[1])) {
60 get_two_cores(struct lcore_pair *lcp)
62 unsigned int socket[2];
66 RTE_LCORE_FOREACH(id[0]) {
67 RTE_LCORE_FOREACH(id[1]) {
70 core[0] = rte_lcore_to_cpu_id(id[0]);
71 core[1] = rte_lcore_to_cpu_id(id[1]);
72 socket[0] = rte_lcore_to_socket_id(id[0]);
73 socket[1] = rte_lcore_to_socket_id(id[1]);
74 if ((core[0] != core[1]) && (socket[0] == socket[1])) {
86 get_two_sockets(struct lcore_pair *lcp)
88 unsigned int socket[2];
91 RTE_LCORE_FOREACH(id[0]) {
92 RTE_LCORE_FOREACH(id[1]) {
95 socket[0] = rte_lcore_to_socket_id(id[0]);
96 socket[1] = rte_lcore_to_socket_id(id[1]);
97 if (socket[0] != socket[1]) {
108 /* Measure the cycle cost of popping an empty stack. */
110 test_empty_pop(struct rte_stack *s)
112 unsigned int iterations = 100000000;
113 void *objs[MAX_BURST];
116 uint64_t start = rte_rdtsc();
118 for (i = 0; i < iterations; i++)
119 rte_stack_pop(s, objs, bulk_sizes[0]);
121 uint64_t end = rte_rdtsc();
123 printf("Stack empty pop: %.2F\n",
124 (double)(end - start) / iterations);
133 /* Measure the average per-pointer cycle cost of stack push and pop */
135 bulk_push_pop(void *p)
137 unsigned int iterations = 1000000;
138 struct thread_args *args = p;
139 void *objs[MAX_BURST] = {0};
140 unsigned int size, i;
146 __atomic_fetch_sub(&lcore_barrier, 1, __ATOMIC_RELAXED);
147 rte_wait_until_equal_32(&lcore_barrier, 0, __ATOMIC_RELAXED);
149 uint64_t start = rte_rdtsc();
151 for (i = 0; i < iterations; i++) {
152 rte_stack_push(s, objs, size);
153 rte_stack_pop(s, objs, size);
156 uint64_t end = rte_rdtsc();
158 args->avg = ((double)(end - start))/(iterations * size);
164 * Run bulk_push_pop() simultaneously on pairs of cores, to measure stack
165 * perf when between hyperthread siblings, cores on the same socket, and cores
166 * on different sockets.
169 run_on_core_pair(struct lcore_pair *cores, struct rte_stack *s,
172 struct thread_args args[2];
175 for (i = 0; i < RTE_DIM(bulk_sizes); i++) {
176 __atomic_store_n(&lcore_barrier, 2, __ATOMIC_RELAXED);
178 args[0].sz = args[1].sz = bulk_sizes[i];
179 args[0].s = args[1].s = s;
181 if (cores->c1 == rte_get_main_lcore()) {
182 rte_eal_remote_launch(fn, &args[1], cores->c2);
184 rte_eal_wait_lcore(cores->c2);
186 rte_eal_remote_launch(fn, &args[0], cores->c1);
187 rte_eal_remote_launch(fn, &args[1], cores->c2);
188 rte_eal_wait_lcore(cores->c1);
189 rte_eal_wait_lcore(cores->c2);
192 printf("Average cycles per object push/pop (bulk size: %u): %.2F\n",
193 bulk_sizes[i], (args[0].avg + args[1].avg) / 2);
197 /* Run bulk_push_pop() simultaneously on 1+ cores. */
199 run_on_n_cores(struct rte_stack *s, lcore_function_t fn, int n)
201 struct thread_args args[RTE_MAX_LCORE];
204 for (i = 0; i < RTE_DIM(bulk_sizes); i++) {
205 unsigned int lcore_id;
209 __atomic_store_n(&lcore_barrier, n, __ATOMIC_RELAXED);
211 RTE_LCORE_FOREACH_WORKER(lcore_id) {
215 args[lcore_id].s = s;
216 args[lcore_id].sz = bulk_sizes[i];
218 if (rte_eal_remote_launch(fn, &args[lcore_id],
220 rte_panic("Failed to launch lcore %d\n",
224 lcore_id = rte_lcore_id();
226 args[lcore_id].s = s;
227 args[lcore_id].sz = bulk_sizes[i];
231 rte_eal_mp_wait_lcore();
233 avg = args[rte_lcore_id()].avg;
236 RTE_LCORE_FOREACH_WORKER(lcore_id) {
239 avg += args[lcore_id].avg;
242 printf("Average cycles per object push/pop (bulk size: %u): %.2F\n",
243 bulk_sizes[i], avg / n);
248 * Measure the cycle cost of pushing and popping a single pointer on a single
252 test_single_push_pop(struct rte_stack *s)
254 unsigned int iterations = 16000000;
258 uint64_t start = rte_rdtsc();
260 for (i = 0; i < iterations; i++) {
261 rte_stack_push(s, &obj, 1);
262 rte_stack_pop(s, &obj, 1);
265 uint64_t end = rte_rdtsc();
267 printf("Average cycles per single object push/pop: %.2F\n",
268 ((double)(end - start)) / iterations);
271 /* Measure the cycle cost of bulk pushing and popping on a single lcore. */
273 test_bulk_push_pop(struct rte_stack *s)
275 unsigned int iterations = 8000000;
276 void *objs[MAX_BURST];
279 for (sz = 0; sz < RTE_DIM(bulk_sizes); sz++) {
280 uint64_t start = rte_rdtsc();
282 for (i = 0; i < iterations; i++) {
283 rte_stack_push(s, objs, bulk_sizes[sz]);
284 rte_stack_pop(s, objs, bulk_sizes[sz]);
287 uint64_t end = rte_rdtsc();
289 double avg = ((double)(end - start) /
290 (iterations * bulk_sizes[sz]));
292 printf("Average cycles per object push/pop (bulk size: %u): %.2F\n",
293 bulk_sizes[sz], avg);
298 __test_stack_perf(uint32_t flags)
300 struct lcore_pair cores;
303 __atomic_store_n(&lcore_barrier, 0, __ATOMIC_RELAXED);
305 s = rte_stack_create(STACK_NAME, STACK_SIZE, rte_socket_id(), flags);
307 printf("[%s():%u] failed to create a stack\n",
312 printf("### Testing single element push/pop ###\n");
313 test_single_push_pop(s);
315 printf("\n### Testing empty pop ###\n");
318 printf("\n### Testing using a single lcore ###\n");
319 test_bulk_push_pop(s);
321 if (get_two_hyperthreads(&cores) == 0) {
322 printf("\n### Testing using two hyperthreads ###\n");
323 run_on_core_pair(&cores, s, bulk_push_pop);
325 if (get_two_cores(&cores) == 0) {
326 printf("\n### Testing using two physical cores ###\n");
327 run_on_core_pair(&cores, s, bulk_push_pop);
329 if (get_two_sockets(&cores) == 0) {
330 printf("\n### Testing using two NUMA nodes ###\n");
331 run_on_core_pair(&cores, s, bulk_push_pop);
334 printf("\n### Testing on all %u lcores ###\n", rte_lcore_count());
335 run_on_n_cores(s, bulk_push_pop, rte_lcore_count());
342 test_stack_perf(void)
344 return __test_stack_perf(0);
348 test_lf_stack_perf(void)
350 #if defined(RTE_STACK_LF_SUPPORTED)
351 return __test_stack_perf(RTE_STACK_F_LF);
357 REGISTER_TEST_COMMAND(stack_perf_autotest, test_stack_perf);
358 REGISTER_TEST_COMMAND(stack_lf_perf_autotest, test_lf_stack_perf);