+
+ return 0;
+}
+
+static __rte_noinline int
+pipeline_atq_worker_multi_stage_tx_vector(void *arg)
+{
+ PIPELINE_WORKER_MULTI_STAGE_INIT;
+ uint16_t vector_sz;
+
+ while (!t->done) {
+ uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
+
+ if (!event) {
+ rte_pause();
+ continue;
+ }
+
+ cq_id = ev.sub_event_type % nb_stages;
+
+ if (cq_id == last_queue) {
+ vector_sz = ev.vec->nb_elem;
+ pipeline_event_tx_vector(dev, port, &ev);
+ w->processed_pkts += vector_sz;
+ continue;
+ }
+
+ ev.sub_event_type++;
+ pipeline_fwd_event_vector(&ev, sched_type_list[cq_id]);
+ pipeline_event_enqueue(dev, port, &ev);
+ }
+
+ return 0;
+}
+
+static __rte_noinline int
+pipeline_atq_worker_multi_stage_fwd_vector(void *arg)
+{
+ PIPELINE_WORKER_MULTI_STAGE_INIT;
+ const uint8_t *tx_queue = t->tx_evqueue_id;
+ uint16_t vector_sz;
+
+ while (!t->done) {
+ uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
+
+ if (!event) {
+ rte_pause();
+ continue;
+ }
+
+ cq_id = ev.sub_event_type % nb_stages;
+
+ if (cq_id == last_queue) {
+ ev.queue_id = tx_queue[ev.vec->port];
+ ev.vec->queue = 0;
+ vector_sz = ev.vec->nb_elem;
+ pipeline_fwd_event_vector(&ev, RTE_SCHED_TYPE_ATOMIC);
+ pipeline_event_enqueue(dev, port, &ev);
+ w->processed_pkts += vector_sz;
+ } else {
+ ev.sub_event_type++;
+ pipeline_fwd_event_vector(&ev, sched_type_list[cq_id]);
+ pipeline_event_enqueue(dev, port, &ev);
+ }
+ }
+
+ return 0;
+}
+
+static __rte_noinline int
+pipeline_atq_worker_multi_stage_burst_tx_vector(void *arg)
+{
+ PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
+ uint16_t vector_sz;
+
+ while (!t->done) {
+ uint16_t nb_rx =
+ rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
+
+ if (!nb_rx) {
+ rte_pause();
+ continue;
+ }
+
+ for (i = 0; i < nb_rx; i++) {
+ cq_id = ev[i].sub_event_type % nb_stages;
+
+ if (cq_id == last_queue) {
+ vector_sz = ev[i].vec->nb_elem;
+ pipeline_event_tx_vector(dev, port, &ev[i]);
+ ev[i].op = RTE_EVENT_OP_RELEASE;
+ w->processed_pkts += vector_sz;
+ continue;
+ }
+
+ ev[i].sub_event_type++;
+ pipeline_fwd_event_vector(&ev[i],
+ sched_type_list[cq_id]);
+ }
+
+ pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
+ }
+
+ return 0;
+}
+
+static __rte_noinline int
+pipeline_atq_worker_multi_stage_burst_fwd_vector(void *arg)
+{
+ PIPELINE_WORKER_MULTI_STAGE_BURST_INIT;
+ const uint8_t *tx_queue = t->tx_evqueue_id;
+ uint16_t vector_sz;
+
+ while (!t->done) {
+ uint16_t nb_rx =
+ rte_event_dequeue_burst(dev, port, ev, BURST_SIZE, 0);
+
+ if (!nb_rx) {
+ rte_pause();
+ continue;
+ }
+
+ for (i = 0; i < nb_rx; i++) {
+ cq_id = ev[i].sub_event_type % nb_stages;
+
+ if (cq_id == last_queue) {
+ vector_sz = ev[i].vec->nb_elem;
+ ev[i].queue_id = tx_queue[ev[i].vec->port];
+ ev[i].vec->queue = 0;
+ pipeline_fwd_event_vector(
+ &ev[i], RTE_SCHED_TYPE_ATOMIC);
+ w->processed_pkts += vector_sz;
+ } else {
+ ev[i].sub_event_type++;
+ pipeline_fwd_event_vector(
+ &ev[i], sched_type_list[cq_id]);
+ }
+ }
+
+ pipeline_event_enqueue_burst(dev, port, ev, nb_rx);
+ }
+