From 314bcf58ca8fcd3e013f36729f9e6323a412cd1a Mon Sep 17 00:00:00 2001 From: Pavan Nikhilesh Date: Tue, 16 Jan 2018 23:16:02 +0530 Subject: [PATCH] app/eventdev: add pipeline queue worker functions Signed-off-by: Pavan Nikhilesh Acked-by: Harry van Haaren --- app/test-eventdev/test_pipeline_common.h | 72 ++++++ app/test-eventdev/test_pipeline_queue.c | 288 ++++++++++++++++++++++- 2 files changed, 359 insertions(+), 1 deletion(-) diff --git a/app/test-eventdev/test_pipeline_common.h b/app/test-eventdev/test_pipeline_common.h index b6923f3bef..5fb91607d5 100644 --- a/app/test-eventdev/test_pipeline_common.h +++ b/app/test-eventdev/test_pipeline_common.h @@ -65,6 +65,78 @@ struct test_pipeline { #define BURST_SIZE 16 +#define PIPELINE_WROKER_SINGLE_STAGE_INIT \ + struct worker_data *w = arg; \ + struct test_pipeline *t = w->t; \ + const uint8_t dev = w->dev_id; \ + const uint8_t port = w->port_id; \ + struct rte_event ev + +#define PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT \ + int i; \ + struct worker_data *w = arg; \ + struct test_pipeline *t = w->t; \ + const uint8_t dev = w->dev_id; \ + const uint8_t port = w->port_id; \ + struct rte_event ev[BURST_SIZE + 1] + +#define PIPELINE_WROKER_MULTI_STAGE_INIT \ + struct worker_data *w = arg; \ + struct test_pipeline *t = w->t; \ + uint8_t cq_id; \ + const uint8_t dev = w->dev_id; \ + const uint8_t port = w->port_id; \ + const uint8_t last_queue = t->opt->nb_stages - 1; \ + uint8_t *const sched_type_list = &t->sched_type_list[0]; \ + struct rte_event ev + +#define PIPELINE_WROKER_MULTI_STAGE_BURST_INIT \ + int i; \ + struct worker_data *w = arg; \ + struct test_pipeline *t = w->t; \ + uint8_t cq_id; \ + const uint8_t dev = w->dev_id; \ + const uint8_t port = w->port_id; \ + const uint8_t last_queue = t->opt->nb_stages - 1; \ + uint8_t *const sched_type_list = &t->sched_type_list[0]; \ + struct rte_event ev[BURST_SIZE + 1] + +static __rte_always_inline void +pipeline_fwd_event(struct rte_event *ev, uint8_t sched) +{ + ev->event_type = RTE_EVENT_TYPE_CPU; + ev->op = RTE_EVENT_OP_FORWARD; + ev->sched_type = sched; +} + +static __rte_always_inline void +pipeline_event_enqueue(const uint8_t dev, const uint8_t port, + struct rte_event *ev) +{ + while (rte_event_enqueue_burst(dev, port, ev, 1) != 1) + rte_pause(); +} + +static __rte_always_inline void +pipeline_event_enqueue_burst(const uint8_t dev, const uint8_t port, + struct rte_event *ev, const uint16_t nb_rx) +{ + uint16_t enq; + + enq = rte_event_enqueue_burst(dev, port, ev, nb_rx); + while (enq < nb_rx) { + enq += rte_event_enqueue_burst(dev, port, + ev + enq, nb_rx - enq); + } +} + +static __rte_always_inline void +pipeline_tx_pkt(struct rte_mbuf *mbuf) +{ + while (rte_eth_tx_burst(mbuf->port, 0, &mbuf, 1) != 1) + rte_pause(); +} + static inline int pipeline_nb_event_ports(struct evt_options *opt) { diff --git a/app/test-eventdev/test_pipeline_queue.c b/app/test-eventdev/test_pipeline_queue.c index 773c3ecaa3..835fe0782f 100644 --- a/app/test-eventdev/test_pipeline_queue.c +++ b/app/test-eventdev/test_pipeline_queue.c @@ -15,10 +15,296 @@ pipeline_queue_nb_event_queues(struct evt_options *opt) return (eth_count * opt->nb_stages) + eth_count; } +static int +pipeline_queue_worker_single_stage_tx(void *arg) +{ + PIPELINE_WROKER_SINGLE_STAGE_INIT; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) { + pipeline_tx_pkt(ev.mbuf); + w->processed_pkts++; + } else { + ev.queue_id++; + pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + pipeline_event_enqueue(dev, port, &ev); + } + } + + return 0; +} + +static int +pipeline_queue_worker_single_stage_fwd(void *arg) +{ + PIPELINE_WROKER_SINGLE_STAGE_INIT; + const uint8_t tx_queue = t->tx_service.queue_id; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + ev.queue_id = tx_queue; + pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + pipeline_event_enqueue(dev, port, &ev); + w->processed_pkts++; + } + + return 0; +} + +static int +pipeline_queue_worker_single_stage_burst_tx(void *arg) +{ + PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) { + + pipeline_tx_pkt(ev[i].mbuf); + ev[i].op = RTE_EVENT_OP_RELEASE; + w->processed_pkts++; + } else { + ev[i].queue_id++; + pipeline_fwd_event(&ev[i], + RTE_SCHED_TYPE_ATOMIC); + } + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + } + + return 0; +} + +static int +pipeline_queue_worker_single_stage_burst_fwd(void *arg) +{ + PIPELINE_WROKER_SINGLE_STAGE_BURST_INIT; + const uint8_t tx_queue = t->tx_service.queue_id; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + ev[i].queue_id = tx_queue; + pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC); + w->processed_pkts++; + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + } + + return 0; +} + + +static int +pipeline_queue_worker_multi_stage_tx(void *arg) +{ + PIPELINE_WROKER_MULTI_STAGE_INIT; + const uint8_t nb_stages = t->opt->nb_stages + 1; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + cq_id = ev.queue_id % nb_stages; + + if (cq_id >= last_queue) { + if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) { + + pipeline_tx_pkt(ev.mbuf); + w->processed_pkts++; + continue; + } + ev.queue_id += (cq_id == last_queue) ? 1 : 0; + pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + } else { + ev.queue_id++; + pipeline_fwd_event(&ev, sched_type_list[cq_id]); + } + + pipeline_event_enqueue(dev, port, &ev); + } + return 0; +} + +static int +pipeline_queue_worker_multi_stage_fwd(void *arg) +{ + PIPELINE_WROKER_MULTI_STAGE_INIT; + const uint8_t nb_stages = t->opt->nb_stages + 1; + const uint8_t tx_queue = t->tx_service.queue_id; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + cq_id = ev.queue_id % nb_stages; + + if (cq_id == last_queue) { + ev.queue_id = tx_queue; + pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + w->processed_pkts++; + } else { + ev.queue_id++; + pipeline_fwd_event(&ev, sched_type_list[cq_id]); + } + + pipeline_event_enqueue(dev, port, &ev); + } + return 0; +} + +static int +pipeline_queue_worker_multi_stage_burst_tx(void *arg) +{ + PIPELINE_WROKER_MULTI_STAGE_BURST_INIT; + const uint8_t nb_stages = t->opt->nb_stages + 1; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + cq_id = ev[i].queue_id % nb_stages; + + if (cq_id >= last_queue) { + if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) { + + pipeline_tx_pkt(ev[i].mbuf); + ev[i].op = RTE_EVENT_OP_RELEASE; + w->processed_pkts++; + continue; + } + + ev[i].queue_id += (cq_id == last_queue) ? 1 : 0; + pipeline_fwd_event(&ev[i], + RTE_SCHED_TYPE_ATOMIC); + } else { + ev[i].queue_id++; + pipeline_fwd_event(&ev[i], + sched_type_list[cq_id]); + } + + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + } + return 0; +} + +static int +pipeline_queue_worker_multi_stage_burst_fwd(void *arg) +{ + PIPELINE_WROKER_MULTI_STAGE_BURST_INIT; + const uint8_t nb_stages = t->opt->nb_stages + 1; + const uint8_t tx_queue = t->tx_service.queue_id; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + cq_id = ev[i].queue_id % nb_stages; + + if (cq_id == last_queue) { + ev[i].queue_id = tx_queue; + pipeline_fwd_event(&ev[i], + RTE_SCHED_TYPE_ATOMIC); + w->processed_pkts++; + } else { + ev[i].queue_id++; + pipeline_fwd_event(&ev[i], + sched_type_list[cq_id]); + } + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + } + return 0; +} + static int worker_wrapper(void *arg) { - RTE_SET_USED(arg); + struct worker_data *w = arg; + struct evt_options *opt = w->t->opt; + const bool burst = evt_has_burst_mode(w->dev_id); + const bool mt_safe = !w->t->mt_unsafe; + const uint8_t nb_stages = opt->nb_stages; + RTE_SET_USED(opt); + + if (nb_stages == 1) { + if (!burst && mt_safe) + return pipeline_queue_worker_single_stage_tx(arg); + else if (!burst && !mt_safe) + return pipeline_queue_worker_single_stage_fwd(arg); + else if (burst && mt_safe) + return pipeline_queue_worker_single_stage_burst_tx(arg); + else if (burst && !mt_safe) + return pipeline_queue_worker_single_stage_burst_fwd( + arg); + } else { + if (!burst && mt_safe) + return pipeline_queue_worker_multi_stage_tx(arg); + else if (!burst && !mt_safe) + return pipeline_queue_worker_multi_stage_fwd(arg); + else if (burst && mt_safe) + return pipeline_queue_worker_multi_stage_burst_tx(arg); + else if (burst && !mt_safe) + return pipeline_queue_worker_multi_stage_burst_fwd(arg); + + } rte_panic("invalid worker\n"); } -- 2.20.1