app/testeventdev: use service cores

Use service cores for offloading event scheduling in case of
centralized scheduling instead of calling the schedule api directly.
This removes the dependency on dedicated scheduler core specified by
giving command line option --slcore.

Signed-off-by: Pavan Nikhilesh <pbhagavatula@caviumnetworks.com>
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

diff --git a/app/test-eventdev/evt_common.h b/app/test-eventdev/evt_common.h
index ee896a2..0fadab4 100644 (file)
--- a/app/test-eventdev/evt_common.h
+++ b/app/test-eventdev/evt_common.h
@@ -36,6 +36,7 @@
 #include <rte_common.h>
 #include <rte_debug.h>
 #include <rte_eventdev.h>
+#include <rte_service.h>
 
 #define CLNRM  "\x1b[0m"
 #define CLRED  "\x1b[31m"
@@ -92,4 +93,43 @@ evt_has_all_types_queue(uint8_t dev_id)
                        true : false;
 }
 
+static inline int
+evt_service_setup(uint8_t dev_id)
+{
+       uint32_t service_id;
+       int32_t core_cnt;
+       unsigned int lcore = 0;
+       uint32_t core_array[RTE_MAX_LCORE];
+       uint8_t cnt;
+       uint8_t min_cnt = UINT8_MAX;
+
+       if (evt_has_distributed_sched(dev_id))
+               return 0;
+
+       if (!rte_service_lcore_count())
+               return -ENOENT;
+
+       if (!rte_event_dev_service_id_get(dev_id, &service_id)) {
+               core_cnt = rte_service_lcore_list(core_array,
+                               RTE_MAX_LCORE);
+               if (core_cnt < 0)
+                       return -ENOENT;
+               /* Get the core which has least number of services running. */
+               while (core_cnt--) {
+                       /* Reset default mapping */
+                       rte_service_map_lcore_set(service_id,
+                                       core_array[core_cnt], 0);
+                       cnt = rte_service_lcore_count_services(
+                                       core_array[core_cnt]);
+                       if (cnt < min_cnt) {
+                               lcore = core_array[core_cnt];
+                               min_cnt = cnt;
+                       }
+               }
+               if (rte_service_map_lcore_set(service_id, lcore, 1))
+                       return -ENOENT;
+       }
+       return 0;
+}
+
 #endif /*  _EVT_COMMON_*/

diff --git a/app/test-eventdev/evt_options.c b/app/test-eventdev/evt_options.c
index 65e22f8..e2187df 100644 (file)
--- a/app/test-eventdev/evt_options.c
+++ b/app/test-eventdev/evt_options.c
@@ -113,13 +113,6 @@ evt_parse_test_name(struct evt_options *opt, const char *arg)
        return 0;
 }
 
-static int
-evt_parse_slcore(struct evt_options *opt, const char *arg)
-{
-       opt->slcore = atoi(arg);
-       return 0;
-}
-
 static int
 evt_parse_socket_id(struct evt_options *opt, const char *arg)
 {
@@ -188,7 +181,6 @@ usage(char *program)
                "\t--test             : name of the test application to run\n"
                "\t--socket_id        : socket_id of application resources\n"
                "\t--pool_sz          : pool size of the mempool\n"
-               "\t--slcore           : lcore id of the scheduler\n"
                "\t--plcores          : list of lcore ids for producers\n"
                "\t--wlcores          : list of lcore ids for workers\n"
                "\t--stlist           : list of scheduled types of the stages\n"
@@ -254,7 +246,6 @@ static struct option lgopts[] = {
        { EVT_POOL_SZ,          1, 0, 0 },
        { EVT_NB_PKTS,          1, 0, 0 },
        { EVT_WKR_DEQ_DEP,      1, 0, 0 },
-       { EVT_SCHED_LCORE,      1, 0, 0 },
        { EVT_SCHED_TYPE_LIST,  1, 0, 0 },
        { EVT_FWD_LATENCY,      0, 0, 0 },
        { EVT_QUEUE_PRIORITY,   0, 0, 0 },
@@ -278,7 +269,6 @@ evt_opts_parse_long(int opt_idx, struct evt_options *opt)
                { EVT_POOL_SZ, evt_parse_pool_sz},
                { EVT_NB_PKTS, evt_parse_nb_pkts},
                { EVT_WKR_DEQ_DEP, evt_parse_wkr_deq_dep},
-               { EVT_SCHED_LCORE, evt_parse_slcore},
                { EVT_SCHED_TYPE_LIST, evt_parse_sched_type_list},
                { EVT_FWD_LATENCY, evt_parse_fwd_latency},
                { EVT_QUEUE_PRIORITY, evt_parse_queue_priority},

diff --git a/app/test-eventdev/evt_options.h b/app/test-eventdev/evt_options.h
index d8a9fdc..a9a9125 100644 (file)
--- a/app/test-eventdev/evt_options.h
+++ b/app/test-eventdev/evt_options.h
@@ -47,7 +47,6 @@
 #define EVT_VERBOSE              ("verbose")
 #define EVT_DEVICE               ("dev")
 #define EVT_TEST                 ("test")
-#define EVT_SCHED_LCORE          ("slcore")
 #define EVT_PROD_LCORES          ("plcores")
 #define EVT_WORK_LCORES          ("wlcores")
 #define EVT_NB_FLOWS             ("nb_flows")
@@ -67,7 +66,6 @@ struct evt_options {
        bool plcores[RTE_MAX_LCORE];
        bool wlcores[RTE_MAX_LCORE];
        uint8_t sched_type_list[EVT_MAX_STAGES];
-       int slcore;
        uint32_t nb_flows;
        int socket_id;
        int pool_sz;
@@ -218,12 +216,6 @@ evt_dump_nb_flows(struct evt_options *opt)
        evt_dump("nb_flows", "%d", opt->nb_flows);
 }
 
-static inline void
-evt_dump_scheduler_lcore(struct evt_options *opt)
-{
-       evt_dump("scheduler lcore", "%d", opt->slcore);
-}
-
 static inline void
 evt_dump_worker_dequeue_depth(struct evt_options *opt)
 {

diff --git a/app/test-eventdev/test_order_atq.c b/app/test-eventdev/test_order_atq.c
index 7e6c67d..4ee0dea 100644 (file)
--- a/app/test-eventdev/test_order_atq.c
+++ b/app/test-eventdev/test_order_atq.c
@@ -179,6 +179,12 @@ order_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
        if (ret)
                return ret;
 
+       ret = evt_service_setup(opt->dev_id);
+       if (ret) {
+               evt_err("No service lcore found to run event dev.");
+               return ret;
+       }
+
        ret = rte_event_dev_start(opt->dev_id);
        if (ret) {
                evt_err("failed to start eventdev %d", opt->dev_id);

diff --git a/app/test-eventdev/test_order_common.c b/app/test-eventdev/test_order_common.c
index 80e14c0..7cfe7fa 100644 (file)
--- a/app/test-eventdev/test_order_common.c
+++ b/app/test-eventdev/test_order_common.c
@@ -292,9 +292,6 @@ order_launch_lcores(struct evt_test *test, struct evt_options *opt,
        int64_t old_remaining  = -1;
 
        while (t->err == false) {
-
-               rte_event_schedule(opt->dev_id);
-
                uint64_t new_cycles = rte_get_timer_cycles();
                int64_t remaining = rte_atomic64_read(&t->outstand_pkts);
 

diff --git a/app/test-eventdev/test_order_queue.c b/app/test-eventdev/test_order_queue.c
index 1fa4082..eef69a4 100644 (file)
--- a/app/test-eventdev/test_order_queue.c
+++ b/app/test-eventdev/test_order_queue.c
@@ -192,6 +192,12 @@ order_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
        if (ret)
                return ret;
 
+       ret = evt_service_setup(opt->dev_id);
+       if (ret) {
+               evt_err("No service lcore found to run event dev.");
+               return ret;
+       }
+
        ret = rte_event_dev_start(opt->dev_id);
        if (ret) {
                evt_err("failed to start eventdev %d", opt->dev_id);

diff --git a/app/test-eventdev/test_perf_atq.c b/app/test-eventdev/test_perf_atq.c
index 9c3efa3..0e9f2db 100644 (file)
--- a/app/test-eventdev/test_perf_atq.c
+++ b/app/test-eventdev/test_perf_atq.c
@@ -221,6 +221,12 @@ perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
        if (ret)
                return ret;
 
+       ret = evt_service_setup(opt->dev_id);
+       if (ret) {
+               evt_err("No service lcore found to run event dev.");
+               return ret;
+       }
+
        ret = rte_event_dev_start(opt->dev_id);
        if (ret) {
                evt_err("failed to start eventdev %d", opt->dev_id);

diff --git a/app/test-eventdev/test_perf_common.c b/app/test-eventdev/test_perf_common.c
index 7b09299..e77b472 100644 (file)
--- a/app/test-eventdev/test_perf_common.c
+++ b/app/test-eventdev/test_perf_common.c
@@ -88,18 +88,6 @@ perf_producer(void *arg)
        return 0;
 }
 
-static inline int
-scheduler(void *arg)
-{
-       struct test_perf *t = arg;
-       const uint8_t dev_id = t->opt->dev_id;
-
-       while (t->done == false)
-               rte_event_schedule(dev_id);
-
-       return 0;
-}
-
 static inline uint64_t
 processed_pkts(struct test_perf *t)
 {
@@ -163,15 +151,6 @@ perf_launch_lcores(struct evt_test *test, struct evt_options *opt,
                port_idx++;
        }
 
-       /* launch scheduler */
-       if (!evt_has_distributed_sched(opt->dev_id)) {
-               ret = rte_eal_remote_launch(scheduler, t, opt->slcore);
-               if (ret) {
-                       evt_err("failed to launch sched %d", opt->slcore);
-                       return ret;
-               }
-       }
-
        const uint64_t total_pkts = opt->nb_pkts *
                        evt_nr_active_lcores(opt->plcores);
 
@@ -307,10 +286,9 @@ int
 perf_opt_check(struct evt_options *opt, uint64_t nb_queues)
 {
        unsigned int lcores;
-       bool need_slcore = !evt_has_distributed_sched(opt->dev_id);
 
-       /* N producer + N worker + 1 scheduler(based on dev capa) + 1 master */
-       lcores = need_slcore ? 4 : 3;
+       /* N producer + N worker + 1 master */
+       lcores = 3;
 
        if (rte_lcore_count() < lcores) {
                evt_err("test need minimum %d lcores", lcores);
@@ -322,10 +300,6 @@ perf_opt_check(struct evt_options *opt, uint64_t nb_queues)
                evt_err("worker lcores overlaps with master lcore");
                return -1;
        }
-       if (need_slcore && evt_lcores_has_overlap(opt->wlcores, opt->slcore)) {
-               evt_err("worker lcores overlaps with scheduler lcore");
-               return -1;
-       }
        if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) {
                evt_err("worker lcores overlaps producer lcores");
                return -1;
@@ -344,10 +318,6 @@ perf_opt_check(struct evt_options *opt, uint64_t nb_queues)
                evt_err("producer lcores overlaps with master lcore");
                return -1;
        }
-       if (need_slcore && evt_lcores_has_overlap(opt->plcores, opt->slcore)) {
-               evt_err("producer lcores overlaps with scheduler lcore");
-               return -1;
-       }
        if (evt_has_disabled_lcore(opt->plcores)) {
                evt_err("one or more producer lcores are not enabled");
                return -1;
@@ -357,17 +327,6 @@ perf_opt_check(struct evt_options *opt, uint64_t nb_queues)
                return -1;
        }
 
-       /* Validate scheduler lcore */
-       if (!evt_has_distributed_sched(opt->dev_id) &&
-                       opt->slcore == (int)rte_get_master_lcore()) {
-               evt_err("scheduler lcore and master lcore should be different");
-               return -1;
-       }
-       if (need_slcore && !rte_lcore_is_enabled(opt->slcore)) {
-               evt_err("scheduler lcore is not enabled");
-               return -1;
-       }
-
        if (evt_has_invalid_stage(opt))
                return -1;
 
@@ -405,8 +364,6 @@ perf_opt_dump(struct evt_options *opt, uint8_t nb_queues)
        evt_dump_producer_lcores(opt);
        evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores));
        evt_dump_worker_lcores(opt);
-       if (!evt_has_distributed_sched(opt->dev_id))
-               evt_dump_scheduler_lcore(opt);
        evt_dump_nb_stages(opt);
        evt_dump("nb_evdev_ports", "%d", perf_nb_event_ports(opt));
        evt_dump("nb_evdev_queues", "%d", nb_queues);

diff --git a/app/test-eventdev/test_perf_common.h b/app/test-eventdev/test_perf_common.h
index 4956586..c6fc70c 100644 (file)
--- a/app/test-eventdev/test_perf_common.h
+++ b/app/test-eventdev/test_perf_common.h
@@ -159,6 +159,7 @@ int perf_test_setup(struct evt_test *test, struct evt_options *opt);
 int perf_mempool_setup(struct evt_test *test, struct evt_options *opt);
 int perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt,
                                uint8_t stride, uint8_t nb_queues);
+int perf_event_dev_service_setup(uint8_t dev_id);
 int perf_launch_lcores(struct evt_test *test, struct evt_options *opt,
                int (*worker)(void *));
 void perf_opt_dump(struct evt_options *opt, uint8_t nb_queues);

diff --git a/app/test-eventdev/test_perf_queue.c b/app/test-eventdev/test_perf_queue.c
index a7a2b1f..d843eea 100644 (file)
--- a/app/test-eventdev/test_perf_queue.c
+++ b/app/test-eventdev/test_perf_queue.c
@@ -232,6 +232,12 @@ perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt)
        if (ret)
                return ret;
 
+       ret = evt_service_setup(opt->dev_id);
+       if (ret) {
+               evt_err("No service lcore found to run event dev.");
+               return ret;
+       }
+
        ret = rte_event_dev_start(opt->dev_id);
        if (ret) {
                evt_err("failed to start eventdev %d", opt->dev_id);