bus/pci: reference driver structure before mapping
[dpdk.git] / drivers / event / opdl / opdl_evdev_init.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Intel Corporation
3  */
4
5 #include <inttypes.h>
6 #include <string.h>
7
8 #include <rte_bus_vdev.h>
9 #include <rte_errno.h>
10 #include <rte_cycles.h>
11 #include <rte_memzone.h>
12
13 #include "opdl_evdev.h"
14 #include "opdl_ring.h"
15 #include "opdl_log.h"
16
17
18 static __rte_always_inline uint32_t
19 enqueue_check(struct opdl_port *p,
20                 const struct rte_event ev[],
21                 uint16_t num,
22                 uint16_t num_events)
23 {
24         uint16_t i;
25
26         if (p->opdl->do_validation) {
27
28                 for (i = 0; i < num; i++) {
29                         if (ev[i].queue_id != p->next_external_qid) {
30                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
31                                              "ERROR - port:[%u] - event wants"
32                                              " to enq to q_id[%u],"
33                                              " but should be [%u]",
34                                              opdl_pmd_dev_id(p->opdl),
35                                              p->id,
36                                              ev[i].queue_id,
37                                              p->next_external_qid);
38                                 rte_errno = -EINVAL;
39                                 return 0;
40                         }
41                 }
42
43                 /* Stats */
44                 if (p->p_type == OPDL_PURE_RX_PORT ||
45                                 p->p_type == OPDL_ASYNC_PORT) {
46                         /* Stats */
47                         if (num_events) {
48                                 p->port_stat[claim_pkts_requested] += num;
49                                 p->port_stat[claim_pkts_granted] += num_events;
50                                 p->port_stat[claim_non_empty]++;
51                                 p->start_cycles = rte_rdtsc();
52                         } else {
53                                 p->port_stat[claim_empty]++;
54                                 p->start_cycles = 0;
55                         }
56                 } else {
57                         if (p->start_cycles) {
58                                 uint64_t end_cycles = rte_rdtsc();
59                                 p->port_stat[total_cycles] +=
60                                         end_cycles - p->start_cycles;
61                         }
62                 }
63         } else {
64                 if (num > 0 &&
65                                 ev[0].queue_id != p->next_external_qid) {
66                         rte_errno = -EINVAL;
67                         return 0;
68                 }
69         }
70
71         return num;
72 }
73
74 static __rte_always_inline void
75 update_on_dequeue(struct opdl_port *p,
76                 struct rte_event ev[],
77                 uint16_t num,
78                 uint16_t num_events)
79 {
80         if (p->opdl->do_validation) {
81                 int16_t i;
82                 for (i = 0; i < num; i++)
83                         ev[i].queue_id =
84                                 p->opdl->queue[p->queue_id].external_qid;
85
86                 /* Stats */
87                 if (num_events) {
88                         p->port_stat[claim_pkts_requested] += num;
89                         p->port_stat[claim_pkts_granted] += num_events;
90                         p->port_stat[claim_non_empty]++;
91                         p->start_cycles = rte_rdtsc();
92                 } else {
93                         p->port_stat[claim_empty]++;
94                         p->start_cycles = 0;
95                 }
96         } else {
97                 if (num > 0)
98                         ev[0].queue_id =
99                                 p->opdl->queue[p->queue_id].external_qid;
100         }
101 }
102
103
104 /*
105  * Error RX enqueue:
106  *
107  *
108  */
109
110 static uint16_t
111 opdl_rx_error_enqueue(struct opdl_port *p,
112                 const struct rte_event ev[],
113                 uint16_t num)
114 {
115         RTE_SET_USED(p);
116         RTE_SET_USED(ev);
117         RTE_SET_USED(num);
118
119         rte_errno = -ENOSPC;
120
121         return 0;
122 }
123
124 /*
125  * RX enqueue:
126  *
127  * This function handles enqueue for a single input stage_inst with
128  *      threadsafe disabled or enabled. eg 1 thread using a stage_inst or
129  *      multiple threads sharing a stage_inst
130  */
131
132 static uint16_t
133 opdl_rx_enqueue(struct opdl_port *p,
134                 const struct rte_event ev[],
135                 uint16_t num)
136 {
137         uint16_t enqueued = 0;
138
139         enqueued = opdl_ring_input(opdl_stage_get_opdl_ring(p->enq_stage_inst),
140                                    ev,
141                                    num,
142                                    false);
143         if (!enqueue_check(p, ev, num, enqueued))
144                 return 0;
145
146
147         if (enqueued < num)
148                 rte_errno = -ENOSPC;
149
150         return enqueued;
151 }
152
153 /*
154  * Error TX handler
155  *
156  */
157
158 static uint16_t
159 opdl_tx_error_dequeue(struct opdl_port *p,
160                 struct rte_event ev[],
161                 uint16_t num)
162 {
163         RTE_SET_USED(p);
164         RTE_SET_USED(ev);
165         RTE_SET_USED(num);
166
167         rte_errno = -ENOSPC;
168
169         return 0;
170 }
171
172 /*
173  * TX single threaded claim
174  *
175  * This function handles dequeue for a single worker stage_inst with
176  *      threadsafe disabled. eg 1 thread using an stage_inst
177  */
178
179 static uint16_t
180 opdl_tx_dequeue_single_thread(struct opdl_port *p,
181                         struct rte_event ev[],
182                         uint16_t num)
183 {
184         uint16_t returned;
185
186         struct opdl_ring  *ring;
187
188         ring = opdl_stage_get_opdl_ring(p->deq_stage_inst);
189
190         returned = opdl_ring_copy_to_burst(ring,
191                                            p->deq_stage_inst,
192                                            ev,
193                                            num,
194                                            false);
195
196         update_on_dequeue(p, ev, num, returned);
197
198         return returned;
199 }
200
201 /*
202  * TX multi threaded claim
203  *
204  * This function handles dequeue for multiple worker stage_inst with
205  *      threadsafe disabled. eg multiple stage_inst each with its own instance
206  */
207
208 static uint16_t
209 opdl_tx_dequeue_multi_inst(struct opdl_port *p,
210                         struct rte_event ev[],
211                         uint16_t num)
212 {
213         uint32_t num_events = 0;
214
215         num_events = opdl_stage_claim(p->deq_stage_inst,
216                                     (void *)ev,
217                                     num,
218                                     NULL,
219                                     false,
220                                     false);
221
222         update_on_dequeue(p, ev, num, num_events);
223
224         return opdl_stage_disclaim(p->deq_stage_inst, num_events, false);
225 }
226
227
228 /*
229  * Worker thread claim
230  *
231  */
232
233 static uint16_t
234 opdl_claim(struct opdl_port *p, struct rte_event ev[], uint16_t num)
235 {
236         uint32_t num_events = 0;
237
238         if (unlikely(num > MAX_OPDL_CONS_Q_DEPTH)) {
239                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
240                              "Attempt to dequeue num of events larger than port (%d) max",
241                              opdl_pmd_dev_id(p->opdl),
242                              p->id);
243                 rte_errno = -EINVAL;
244                 return 0;
245         }
246
247
248         num_events = opdl_stage_claim(p->deq_stage_inst,
249                         (void *)ev,
250                         num,
251                         NULL,
252                         false,
253                         p->atomic_claim);
254
255
256         update_on_dequeue(p, ev, num, num_events);
257
258         return num_events;
259 }
260
261 /*
262  * Worker thread disclaim
263  */
264
265 static uint16_t
266 opdl_disclaim(struct opdl_port *p, const struct rte_event ev[], uint16_t num)
267 {
268         uint16_t enqueued = 0;
269
270         uint32_t i = 0;
271
272         for (i = 0; i < num; i++)
273                 opdl_ring_cas_slot(p->enq_stage_inst, &ev[i],
274                                 i, p->atomic_claim);
275
276         enqueued = opdl_stage_disclaim(p->enq_stage_inst,
277                                        num,
278                                        false);
279
280         return enqueue_check(p, ev, num, enqueued);
281 }
282
283 static __rte_always_inline struct opdl_stage *
284 stage_for_port(struct opdl_queue *q, unsigned int i)
285 {
286         if (q->q_pos == OPDL_Q_POS_START || q->q_pos == OPDL_Q_POS_MIDDLE)
287                 return q->ports[i]->enq_stage_inst;
288         else
289                 return q->ports[i]->deq_stage_inst;
290 }
291
292 static int opdl_add_deps(struct opdl_evdev *device,
293                          int q_id,
294                          int deps_q_id)
295 {
296         unsigned int i, j;
297         int status;
298         struct opdl_ring  *ring;
299         struct opdl_queue *queue = &device->queue[q_id];
300         struct opdl_queue *queue_deps = &device->queue[deps_q_id];
301         struct opdl_stage *dep_stages[OPDL_PORTS_MAX];
302
303         /* sanity check that all stages are for same opdl ring */
304         for (i = 0; i < queue->nb_ports; i++) {
305                 struct opdl_ring *r =
306                         opdl_stage_get_opdl_ring(stage_for_port(queue, i));
307                 for (j = 0; j < queue_deps->nb_ports; j++) {
308                         struct opdl_ring *rj =
309                                 opdl_stage_get_opdl_ring(
310                                                 stage_for_port(queue_deps, j));
311                         if (r != rj) {
312                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
313                                              "Stages and dependents"
314                                              " are not for same opdl ring",
315                                              opdl_pmd_dev_id(device));
316                                 uint32_t k;
317                                 for (k = 0; k < device->nb_opdls; k++) {
318                                         opdl_ring_dump(device->opdl[k],
319                                                         stdout);
320                                 }
321                                 return -EINVAL;
322                         }
323                 }
324         }
325
326         /* Gather all stages instance in deps */
327         for (i = 0; i < queue_deps->nb_ports; i++)
328                 dep_stages[i] = stage_for_port(queue_deps, i);
329
330
331         /* Add all deps for each port->stage_inst in this queue */
332         for (i = 0; i < queue->nb_ports; i++) {
333
334                 ring = opdl_stage_get_opdl_ring(stage_for_port(queue, i));
335
336                 status = opdl_stage_deps_add(ring,
337                                 stage_for_port(queue, i),
338                                 queue->ports[i]->num_instance,
339                                 queue->ports[i]->instance_id,
340                                 dep_stages,
341                                 queue_deps->nb_ports);
342                 if (status < 0)
343                         return -EINVAL;
344         }
345
346         return 0;
347 }
348
349 int
350 opdl_add_event_handlers(struct rte_eventdev *dev)
351 {
352         int err = 0;
353
354         struct opdl_evdev *device = opdl_pmd_priv(dev);
355         unsigned int i;
356
357         for (i = 0; i < device->max_port_nb; i++) {
358
359                 struct opdl_port *port = &device->ports[i];
360
361                 if (port->configured) {
362                         if (port->p_type == OPDL_PURE_RX_PORT) {
363                                 port->enq = opdl_rx_enqueue;
364                                 port->deq = opdl_tx_error_dequeue;
365
366                         } else if (port->p_type == OPDL_PURE_TX_PORT) {
367
368                                 port->enq = opdl_rx_error_enqueue;
369
370                                 if (port->num_instance == 1)
371                                         port->deq =
372                                                 opdl_tx_dequeue_single_thread;
373                                 else
374                                         port->deq = opdl_tx_dequeue_multi_inst;
375
376                         } else if (port->p_type == OPDL_REGULAR_PORT) {
377
378                                 port->enq = opdl_disclaim;
379                                 port->deq = opdl_claim;
380
381                         } else if (port->p_type == OPDL_ASYNC_PORT) {
382
383                                 port->enq = opdl_rx_enqueue;
384
385                                 /* Always single instance */
386                                 port->deq = opdl_tx_dequeue_single_thread;
387                         } else {
388                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
389                                              "port:[%u] has invalid port type - ",
390                                              opdl_pmd_dev_id(port->opdl),
391                                              port->id);
392                                 err = -EINVAL;
393                                 break;
394                         }
395                         port->initialized = 1;
396                 }
397         }
398
399         if (!err)
400                 fprintf(stdout, "Success - enqueue/dequeue handler(s) added\n");
401         return err;
402 }
403
404 int
405 build_all_dependencies(struct rte_eventdev *dev)
406 {
407
408         int err = 0;
409         unsigned int i;
410         struct opdl_evdev *device = opdl_pmd_priv(dev);
411
412         uint8_t start_qid = 0;
413
414         for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
415                 struct opdl_queue *queue = &device->queue[i];
416                 if (!queue->initialized)
417                         break;
418
419                 if (queue->q_pos == OPDL_Q_POS_START) {
420                         start_qid = i;
421                         continue;
422                 }
423
424                 if (queue->q_pos == OPDL_Q_POS_MIDDLE) {
425                         err = opdl_add_deps(device, i, i-1);
426                         if (err < 0) {
427                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
428                                              "dependency addition for queue:[%u] - FAILED",
429                                              dev->data->dev_id,
430                                              queue->external_qid);
431                                 break;
432                         }
433                 }
434
435                 if (queue->q_pos == OPDL_Q_POS_END) {
436                         /* Add this dependency */
437                         err = opdl_add_deps(device, i, i-1);
438                         if (err < 0) {
439                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
440                                              "dependency addition for queue:[%u] - FAILED",
441                                              dev->data->dev_id,
442                                              queue->external_qid);
443                                 break;
444                         }
445                         /* Add dependency for rx on tx */
446                         err = opdl_add_deps(device, start_qid, i);
447                         if (err < 0) {
448                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
449                                              "dependency addition for queue:[%u] - FAILED",
450                                              dev->data->dev_id,
451                                              queue->external_qid);
452                                 break;
453                         }
454                 }
455         }
456
457         if (!err)
458                 fprintf(stdout, "Success - dependencies built\n");
459
460         return err;
461 }
462 int
463 check_queues_linked(struct rte_eventdev *dev)
464 {
465
466         int err = 0;
467         unsigned int i;
468         struct opdl_evdev *device = opdl_pmd_priv(dev);
469         uint32_t nb_iq = 0;
470
471         for (i = 0; i < RTE_EVENT_MAX_QUEUES_PER_DEV; i++) {
472                 struct opdl_queue *queue = &device->queue[i];
473
474                 if (!queue->initialized)
475                         break;
476
477                 if (queue->external_qid == OPDL_INVALID_QID)
478                         nb_iq++;
479
480                 if (queue->nb_ports == 0) {
481                         PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
482                                      "queue:[%u] has no associated ports",
483                                      dev->data->dev_id,
484                                      i);
485                         err = -EINVAL;
486                         break;
487                 }
488         }
489         if (!err) {
490                 if ((i - nb_iq) != device->max_queue_nb) {
491                         PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
492                                      "%u queues counted but should be %u",
493                                      dev->data->dev_id,
494                                      i - nb_iq,
495                                      device->max_queue_nb);
496                         err = -1;
497                 }
498
499         }
500         return err;
501 }
502
503 void
504 destroy_queues_and_rings(struct rte_eventdev *dev)
505 {
506         struct opdl_evdev *device = opdl_pmd_priv(dev);
507         uint32_t i;
508
509         for (i = 0; i < device->nb_opdls; i++) {
510                 if (device->opdl[i])
511                         opdl_ring_free(device->opdl[i]);
512         }
513
514         memset(&device->queue,
515                         0,
516                         sizeof(struct opdl_queue)
517                         * RTE_EVENT_MAX_QUEUES_PER_DEV);
518 }
519
520 #define OPDL_ID(d)(d->nb_opdls - 1)
521
522 static __rte_always_inline void
523 initialise_queue(struct opdl_evdev *device,
524                 enum queue_pos pos,
525                 int32_t i)
526 {
527         struct opdl_queue *queue = &device->queue[device->nb_queues];
528
529         if (i == -1) {
530                 queue->q_type = OPDL_Q_TYPE_ORDERED;
531                 queue->external_qid = OPDL_INVALID_QID;
532         } else {
533                 queue->q_type = device->q_md[i].type;
534                 queue->external_qid = device->q_md[i].ext_id;
535                 /* Add ex->in for queues setup */
536                 device->q_map_ex_to_in[queue->external_qid] = device->nb_queues;
537         }
538         queue->opdl_id = OPDL_ID(device);
539         queue->q_pos = pos;
540         queue->nb_ports = 0;
541         queue->configured = 1;
542
543         device->nb_queues++;
544 }
545
546
547 static __rte_always_inline int
548 create_opdl(struct opdl_evdev *device)
549 {
550         int err = 0;
551
552         char name[RTE_MEMZONE_NAMESIZE];
553
554         snprintf(name, RTE_MEMZONE_NAMESIZE,
555                         "%s_%u", device->service_name, device->nb_opdls);
556
557         device->opdl[device->nb_opdls] =
558                 opdl_ring_create(name,
559                                 device->nb_events_limit,
560                                 sizeof(struct rte_event),
561                                 device->max_port_nb * 2,
562                                 device->socket);
563
564         if (!device->opdl[device->nb_opdls]) {
565                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
566                              "opdl ring %u creation - FAILED",
567                              opdl_pmd_dev_id(device),
568                              device->nb_opdls);
569                 err = -EINVAL;
570         } else {
571                 device->nb_opdls++;
572         }
573         return err;
574 }
575
576 static __rte_always_inline int
577 create_link_opdl(struct opdl_evdev *device, uint32_t index)
578 {
579
580         int err = 0;
581
582         if (device->q_md[index + 1].type !=
583                         OPDL_Q_TYPE_SINGLE_LINK) {
584
585                 /* async queue with regular
586                  * queue following it
587                  */
588
589                 /* create a new opdl ring */
590                 err = create_opdl(device);
591                 if (!err) {
592                         /* create an initial
593                          * dummy queue for new opdl
594                          */
595                         initialise_queue(device,
596                                         OPDL_Q_POS_START,
597                                         -1);
598                 } else {
599                         err = -EINVAL;
600                 }
601         } else {
602                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
603                              "queue %u, two consecutive"
604                              " SINGLE_LINK queues, not allowed",
605                              opdl_pmd_dev_id(device),
606                              index);
607                 err = -EINVAL;
608         }
609
610         return err;
611 }
612
613 int
614 create_queues_and_rings(struct rte_eventdev *dev)
615 {
616         int err = 0;
617
618         struct opdl_evdev *device = opdl_pmd_priv(dev);
619
620         device->nb_queues = 0;
621
622         if (device->nb_ports != device->max_port_nb) {
623                 PMD_DRV_LOG(ERR, "Number ports setup:%u NOT EQUAL to max port"
624                                 " number:%u for this device",
625                                 device->nb_ports,
626                                 device->max_port_nb);
627                 err = -1;
628         }
629
630         if (!err) {
631                 /* We will have at least one opdl so create it now */
632                 err = create_opdl(device);
633         }
634
635         if (!err) {
636
637                 /* Create 1st "dummy" queue */
638                 initialise_queue(device,
639                                  OPDL_Q_POS_START,
640                                  -1);
641
642                 uint32_t i;
643                 for (i = 0; i < device->nb_q_md; i++) {
644
645                         /* Check */
646                         if (!device->q_md[i].setup) {
647
648                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
649                                              "queue meta data slot %u"
650                                              " not setup - FAILING",
651                                              dev->data->dev_id,
652                                              i);
653                                 err = -EINVAL;
654                                 break;
655                         } else if (device->q_md[i].type !=
656                                         OPDL_Q_TYPE_SINGLE_LINK) {
657
658                                 if (!device->q_md[i + 1].setup) {
659                                         /* Create a simple ORDERED/ATOMIC
660                                          * queue at the end
661                                          */
662                                         initialise_queue(device,
663                                                         OPDL_Q_POS_END,
664                                                         i);
665
666                                 } else {
667                                         /* Create a simple ORDERED/ATOMIC
668                                          * queue in the middle
669                                          */
670                                         initialise_queue(device,
671                                                         OPDL_Q_POS_MIDDLE,
672                                                         i);
673                                 }
674                         } else if (device->q_md[i].type ==
675                                         OPDL_Q_TYPE_SINGLE_LINK) {
676
677                                 /* create last queue for this opdl */
678                                 initialise_queue(device,
679                                                 OPDL_Q_POS_END,
680                                                 i);
681
682                                 err = create_link_opdl(device, i);
683
684                                 if (err)
685                                         break;
686
687
688                         }
689                 }
690         }
691         if (err)
692                 destroy_queues_and_rings(dev);
693
694         return err;
695 }
696
697
698 int
699 initialise_all_other_ports(struct rte_eventdev *dev)
700 {
701         int err = 0;
702         struct opdl_stage *stage_inst = NULL;
703
704         struct opdl_evdev *device = opdl_pmd_priv(dev);
705
706         uint32_t i;
707         for (i = 0; i < device->nb_ports; i++) {
708                 struct opdl_port *port = &device->ports[i];
709                 struct opdl_queue *queue = &device->queue[port->queue_id];
710
711                 if (port->queue_id == 0) {
712                         continue;
713                 } else if (queue->q_type != OPDL_Q_TYPE_SINGLE_LINK) {
714
715                         if (queue->q_pos == OPDL_Q_POS_MIDDLE) {
716
717                                 /* Regular port with claim/disclaim */
718                                 stage_inst = opdl_stage_add(
719                                         device->opdl[queue->opdl_id],
720                                                 false,
721                                                 false);
722                                 port->deq_stage_inst = stage_inst;
723                                 port->enq_stage_inst = stage_inst;
724
725                                 if (queue->q_type == OPDL_Q_TYPE_ATOMIC)
726                                         port->atomic_claim = true;
727                                 else
728                                         port->atomic_claim = false;
729
730                                 port->p_type =  OPDL_REGULAR_PORT;
731
732                                 /* Add the port to the queue array of ports */
733                                 queue->ports[queue->nb_ports] = port;
734                                 port->instance_id = queue->nb_ports;
735                                 queue->nb_ports++;
736                                 opdl_stage_set_queue_id(stage_inst,
737                                                 port->queue_id);
738
739                         } else if (queue->q_pos == OPDL_Q_POS_END) {
740
741                                 /* tx port  */
742                                 stage_inst = opdl_stage_add(
743                                         device->opdl[queue->opdl_id],
744                                                 false,
745                                                 false);
746                                 port->deq_stage_inst = stage_inst;
747                                 port->enq_stage_inst = NULL;
748                                 port->p_type = OPDL_PURE_TX_PORT;
749
750                                 /* Add the port to the queue array of ports */
751                                 queue->ports[queue->nb_ports] = port;
752                                 port->instance_id = queue->nb_ports;
753                                 queue->nb_ports++;
754                         } else {
755
756                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
757                                              "port %u:, linked incorrectly"
758                                              " to a q_pos START/INVALID %u",
759                                              opdl_pmd_dev_id(port->opdl),
760                                              port->id,
761                                              queue->q_pos);
762                                 err = -EINVAL;
763                                 break;
764                         }
765
766                 } else if (queue->q_type == OPDL_Q_TYPE_SINGLE_LINK) {
767
768                         port->p_type = OPDL_ASYNC_PORT;
769
770                         /* -- tx -- */
771                         stage_inst = opdl_stage_add(
772                                 device->opdl[queue->opdl_id],
773                                         false,
774                                         false); /* First stage */
775                         port->deq_stage_inst = stage_inst;
776
777                         /* Add the port to the queue array of ports */
778                         queue->ports[queue->nb_ports] = port;
779                         port->instance_id = queue->nb_ports;
780                         queue->nb_ports++;
781
782                         if (queue->nb_ports > 1) {
783                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
784                                              "queue %u:, setup as SINGLE_LINK"
785                                              " but has more than one port linked",
786                                              opdl_pmd_dev_id(port->opdl),
787                                              queue->external_qid);
788                                 err = -EINVAL;
789                                 break;
790                         }
791
792                         /* -- single instance rx for next opdl -- */
793                         uint8_t next_qid =
794                                 device->q_map_ex_to_in[queue->external_qid] + 1;
795                         if (next_qid < RTE_EVENT_MAX_QUEUES_PER_DEV &&
796                                         device->queue[next_qid].configured) {
797
798                                 /* Remap the queue */
799                                 queue = &device->queue[next_qid];
800
801                                 stage_inst = opdl_stage_add(
802                                         device->opdl[queue->opdl_id],
803                                                 false,
804                                                 true);
805                                 port->enq_stage_inst = stage_inst;
806
807                                 /* Add the port to the queue array of ports */
808                                 queue->ports[queue->nb_ports] = port;
809                                 port->instance_id = queue->nb_ports;
810                                 queue->nb_ports++;
811                                 if (queue->nb_ports > 1) {
812                                         PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
813                                                 "dummy queue %u: for "
814                                                 "port %u, "
815                                                 "SINGLE_LINK but has more "
816                                                 "than one port linked",
817                                                 opdl_pmd_dev_id(port->opdl),
818                                                 next_qid,
819                                                 port->id);
820                                         err = -EINVAL;
821                                         break;
822                                 }
823                                 /* Set this queue to initialized as it is never
824                                  * referenced by any ports
825                                  */
826                                 queue->initialized = 1;
827                         }
828                 }
829         }
830
831         /* Now that all ports are initialised we need to
832          * setup the last bit of stage md
833          */
834         if (!err) {
835                 for (i = 0; i < device->nb_ports; i++) {
836                         struct opdl_port *port = &device->ports[i];
837                         struct opdl_queue *queue =
838                                 &device->queue[port->queue_id];
839
840                         if (port->configured &&
841                                         (port->queue_id != OPDL_INVALID_QID)) {
842                                 if (queue->nb_ports == 0) {
843                                         PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
844                                                 "queue:[%u] has no ports"
845                                                 " linked to it",
846                                                 opdl_pmd_dev_id(port->opdl),
847                                                 port->id);
848                                         err = -EINVAL;
849                                         break;
850                                 }
851
852                                 port->num_instance = queue->nb_ports;
853                                 port->initialized = 1;
854                                 queue->initialized = 1;
855                         } else {
856                                 PMD_DRV_LOG(ERR, "DEV_ID:[%02d] : "
857                                              "Port:[%u] not configured  invalid"
858                                              " queue configuration",
859                                              opdl_pmd_dev_id(port->opdl),
860                                              port->id);
861                                 err = -EINVAL;
862                                 break;
863                         }
864                 }
865         }
866         return err;
867 }
868
869 int
870 initialise_queue_zero_ports(struct rte_eventdev *dev)
871 {
872         int err = 0;
873         uint8_t mt_rx = 0;
874         struct opdl_stage *stage_inst = NULL;
875         struct opdl_queue *queue = NULL;
876
877         struct opdl_evdev *device = opdl_pmd_priv(dev);
878
879         /* Assign queue zero and figure out how many Q0 ports we have */
880         uint32_t i;
881         for (i = 0; i < device->nb_ports; i++) {
882                 struct opdl_port *port = &device->ports[i];
883                 if (port->queue_id == OPDL_INVALID_QID) {
884                         port->queue_id = 0;
885                         port->external_qid = OPDL_INVALID_QID;
886                         port->p_type = OPDL_PURE_RX_PORT;
887                         mt_rx++;
888                 }
889         }
890
891         /* Create the stage */
892         stage_inst = opdl_stage_add(device->opdl[0],
893                         (mt_rx > 1 ? true : false),
894                         true);
895         if (stage_inst) {
896
897                 /* Assign the new created input stage to all relevant ports */
898                 for (i = 0; i < device->nb_ports; i++) {
899                         struct opdl_port *port = &device->ports[i];
900                         if (port->queue_id == 0) {
901                                 queue = &device->queue[port->queue_id];
902                                 port->enq_stage_inst = stage_inst;
903                                 port->deq_stage_inst = NULL;
904                                 port->configured = 1;
905                                 port->initialized = 1;
906
907                                 queue->ports[queue->nb_ports] = port;
908                                 port->instance_id = queue->nb_ports;
909                                 queue->nb_ports++;
910                         }
911                 }
912         } else {
913                 err = -1;
914         }
915         return err;
916 }
917
918 int
919 assign_internal_queue_ids(struct rte_eventdev *dev)
920 {
921         int err = 0;
922         struct opdl_evdev *device = opdl_pmd_priv(dev);
923         uint32_t i;
924
925         for (i = 0; i < device->nb_ports; i++) {
926                 struct opdl_port *port = &device->ports[i];
927                 if (port->external_qid != OPDL_INVALID_QID) {
928                         port->queue_id =
929                                 device->q_map_ex_to_in[port->external_qid];
930
931                         /* Now do the external_qid of the next queue */
932                         struct opdl_queue *queue =
933                                 &device->queue[port->queue_id];
934                         if (queue->q_pos == OPDL_Q_POS_END)
935                                 port->next_external_qid =
936                                 device->queue[port->queue_id + 2].external_qid;
937                         else
938                                 port->next_external_qid =
939                                 device->queue[port->queue_id + 1].external_qid;
940                 }
941         }
942         return err;
943 }