eventdev: express DLB/DLB2 PMD constraints
[dpdk.git] / drivers / event / dpaa2 / dpaa2_eventdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2017,2019 NXP
3  */
4
5 #include <assert.h>
6 #include <stdio.h>
7 #include <stdbool.h>
8 #include <errno.h>
9 #include <stdint.h>
10 #include <string.h>
11 #include <sys/epoll.h>
12
13 #include <rte_atomic.h>
14 #include <rte_byteorder.h>
15 #include <rte_common.h>
16 #include <rte_debug.h>
17 #include <rte_dev.h>
18 #include <rte_eal.h>
19 #include <rte_fslmc.h>
20 #include <rte_lcore.h>
21 #include <rte_log.h>
22 #include <rte_malloc.h>
23 #include <rte_memcpy.h>
24 #include <rte_memory.h>
25 #include <rte_pci.h>
26 #include <rte_bus_vdev.h>
27 #include <rte_ethdev_driver.h>
28 #include <rte_cryptodev.h>
29 #include <rte_event_eth_rx_adapter.h>
30 #include <rte_event_eth_tx_adapter.h>
31
32 #include <fslmc_vfio.h>
33 #include <dpaa2_hw_pvt.h>
34 #include <dpaa2_hw_mempool.h>
35 #include <dpaa2_hw_dpio.h>
36 #include <dpaa2_ethdev.h>
37 #include <dpaa2_sec_event.h>
38 #include "dpaa2_eventdev.h"
39 #include "dpaa2_eventdev_logs.h"
40 #include <portal/dpaa2_hw_pvt.h>
41 #include <mc/fsl_dpci.h>
42
43 /* Clarifications
44  * Evendev = SoC Instance
45  * Eventport = DPIO Instance
46  * Eventqueue = DPCON Instance
47  * 1 Eventdev can have N Eventqueue
48  * Soft Event Flow is DPCI Instance
49  */
50
51 #define DPAA2_EV_TX_RETRY_COUNT 10000
52
53 static uint16_t
54 dpaa2_eventdev_enqueue_burst(void *port, const struct rte_event ev[],
55                              uint16_t nb_events)
56 {
57
58         struct dpaa2_port *dpaa2_portal = port;
59         struct dpaa2_dpio_dev *dpio_dev;
60         uint32_t queue_id = ev[0].queue_id;
61         struct dpaa2_eventq *evq_info;
62         uint32_t fqid, retry_count;
63         struct qbman_swp *swp;
64         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
65         uint32_t loop, frames_to_send;
66         struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS];
67         uint16_t num_tx = 0;
68         int i, n, ret;
69         uint8_t channel_index;
70
71         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
72                 /* Affine current thread context to a qman portal */
73                 ret = dpaa2_affine_qbman_swp();
74                 if (ret < 0) {
75                         DPAA2_EVENTDEV_ERR(
76                                 "Failed to allocate IO portal, tid: %d\n",
77                                 rte_gettid());
78                         return 0;
79                 }
80         }
81         /* todo - dpaa2_portal shall have dpio_dev - no per thread variable */
82         dpio_dev = DPAA2_PER_LCORE_DPIO;
83         swp = DPAA2_PER_LCORE_PORTAL;
84
85         if (likely(dpaa2_portal->is_port_linked))
86                 goto skip_linking;
87
88         /* Create mapping between portal and channel to receive packets */
89         for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) {
90                 evq_info = &dpaa2_portal->evq_info[i];
91                 if (!evq_info->event_port)
92                         continue;
93
94                 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio,
95                                                       CMD_PRI_LOW,
96                                                       dpio_dev->token,
97                                                       evq_info->dpcon->dpcon_id,
98                                                       &channel_index);
99                 if (ret < 0) {
100                         DPAA2_EVENTDEV_ERR(
101                                 "Static dequeue config failed: err(%d)", ret);
102                         goto err;
103                 }
104
105                 qbman_swp_push_set(swp, channel_index, 1);
106                 evq_info->dpcon->channel_index = channel_index;
107         }
108         dpaa2_portal->is_port_linked = true;
109
110 skip_linking:
111         evq_info = &dpaa2_portal->evq_info[queue_id];
112
113         while (nb_events) {
114                 frames_to_send = (nb_events > dpaa2_eqcr_size) ?
115                         dpaa2_eqcr_size : nb_events;
116
117                 for (loop = 0; loop < frames_to_send; loop++) {
118                         const struct rte_event *event = &ev[num_tx + loop];
119
120                         if (event->sched_type != RTE_SCHED_TYPE_ATOMIC)
121                                 fqid = evq_info->dpci->rx_queue[
122                                         DPAA2_EVENT_DPCI_PARALLEL_QUEUE].fqid;
123                         else
124                                 fqid = evq_info->dpci->rx_queue[
125                                         DPAA2_EVENT_DPCI_ATOMIC_QUEUE].fqid;
126
127                         /* Prepare enqueue descriptor */
128                         qbman_eq_desc_clear(&eqdesc[loop]);
129                         qbman_eq_desc_set_fq(&eqdesc[loop], fqid);
130                         qbman_eq_desc_set_no_orp(&eqdesc[loop], 0);
131                         qbman_eq_desc_set_response(&eqdesc[loop], 0, 0);
132
133                         if (event->sched_type == RTE_SCHED_TYPE_ATOMIC
134                                 && event->mbuf->seqn) {
135                                 uint8_t dqrr_index = event->mbuf->seqn - 1;
136
137                                 qbman_eq_desc_set_dca(&eqdesc[loop], 1,
138                                                       dqrr_index, 0);
139                                 DPAA2_PER_LCORE_DQRR_SIZE--;
140                                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
141                         }
142
143                         memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
144
145                         /*
146                          * todo - need to align with hw context data
147                          * to avoid copy
148                          */
149                         struct rte_event *ev_temp = rte_malloc(NULL,
150                                                 sizeof(struct rte_event), 0);
151
152                         if (!ev_temp) {
153                                 if (!loop)
154                                         return num_tx;
155                                 frames_to_send = loop;
156                                 DPAA2_EVENTDEV_ERR(
157                                         "Unable to allocate event object");
158                                 goto send_partial;
159                         }
160                         rte_memcpy(ev_temp, event, sizeof(struct rte_event));
161                         DPAA2_SET_FD_ADDR((&fd_arr[loop]), (size_t)ev_temp);
162                         DPAA2_SET_FD_LEN((&fd_arr[loop]),
163                                          sizeof(struct rte_event));
164                 }
165 send_partial:
166                 loop = 0;
167                 retry_count = 0;
168                 while (loop < frames_to_send) {
169                         ret = qbman_swp_enqueue_multiple_desc(swp,
170                                         &eqdesc[loop], &fd_arr[loop],
171                                         frames_to_send - loop);
172                         if (unlikely(ret < 0)) {
173                                 retry_count++;
174                                 if (retry_count > DPAA2_EV_TX_RETRY_COUNT) {
175                                         num_tx += loop;
176                                         nb_events -= loop;
177                                         return num_tx + loop;
178                                 }
179                         } else {
180                                 loop += ret;
181                                 retry_count = 0;
182                         }
183                 }
184                 num_tx += loop;
185                 nb_events -= loop;
186         }
187
188         return num_tx;
189 err:
190         for (n = 0; n < i; n++) {
191                 evq_info = &dpaa2_portal->evq_info[n];
192                 if (!evq_info->event_port)
193                         continue;
194                 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0);
195                 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0,
196                                                 dpio_dev->token,
197                                                 evq_info->dpcon->dpcon_id);
198         }
199         return 0;
200
201 }
202
203 static uint16_t
204 dpaa2_eventdev_enqueue(void *port, const struct rte_event *ev)
205 {
206         return dpaa2_eventdev_enqueue_burst(port, ev, 1);
207 }
208
209 static void dpaa2_eventdev_dequeue_wait(uint64_t timeout_ticks)
210 {
211         struct epoll_event epoll_ev;
212
213         qbman_swp_interrupt_clear_status(DPAA2_PER_LCORE_PORTAL,
214                                          QBMAN_SWP_INTERRUPT_DQRI);
215
216         epoll_wait(DPAA2_PER_LCORE_DPIO->epoll_fd,
217                          &epoll_ev, 1, timeout_ticks);
218 }
219
220 static void dpaa2_eventdev_process_parallel(struct qbman_swp *swp,
221                                             const struct qbman_fd *fd,
222                                             const struct qbman_result *dq,
223                                             struct dpaa2_queue *rxq,
224                                             struct rte_event *ev)
225 {
226         struct rte_event *ev_temp =
227                 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd);
228
229         RTE_SET_USED(rxq);
230
231         rte_memcpy(ev, ev_temp, sizeof(struct rte_event));
232         rte_free(ev_temp);
233
234         qbman_swp_dqrr_consume(swp, dq);
235 }
236
237 static void dpaa2_eventdev_process_atomic(struct qbman_swp *swp,
238                                           const struct qbman_fd *fd,
239                                           const struct qbman_result *dq,
240                                           struct dpaa2_queue *rxq,
241                                           struct rte_event *ev)
242 {
243         struct rte_event *ev_temp =
244                 (struct rte_event *)(size_t)DPAA2_GET_FD_ADDR(fd);
245         uint8_t dqrr_index = qbman_get_dqrr_idx(dq);
246
247         RTE_SET_USED(swp);
248         RTE_SET_USED(rxq);
249
250         rte_memcpy(ev, ev_temp, sizeof(struct rte_event));
251         rte_free(ev_temp);
252         ev->mbuf->seqn = dqrr_index + 1;
253         DPAA2_PER_LCORE_DQRR_SIZE++;
254         DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
255         DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf;
256 }
257
258 static uint16_t
259 dpaa2_eventdev_dequeue_burst(void *port, struct rte_event ev[],
260                              uint16_t nb_events, uint64_t timeout_ticks)
261 {
262         const struct qbman_result *dq;
263         struct dpaa2_dpio_dev *dpio_dev = NULL;
264         struct dpaa2_port *dpaa2_portal = port;
265         struct dpaa2_eventq *evq_info;
266         struct qbman_swp *swp;
267         const struct qbman_fd *fd;
268         struct dpaa2_queue *rxq;
269         int num_pkts = 0, ret, i = 0, n;
270         uint8_t channel_index;
271
272         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
273                 /* Affine current thread context to a qman portal */
274                 ret = dpaa2_affine_qbman_swp();
275                 if (ret < 0) {
276                         DPAA2_EVENTDEV_ERR(
277                                 "Failed to allocate IO portal, tid: %d\n",
278                                 rte_gettid());
279                         return 0;
280                 }
281         }
282
283         dpio_dev = DPAA2_PER_LCORE_DPIO;
284         swp = DPAA2_PER_LCORE_PORTAL;
285
286         if (likely(dpaa2_portal->is_port_linked))
287                 goto skip_linking;
288
289         /* Create mapping between portal and channel to receive packets */
290         for (i = 0; i < DPAA2_EVENT_MAX_QUEUES; i++) {
291                 evq_info = &dpaa2_portal->evq_info[i];
292                 if (!evq_info->event_port)
293                         continue;
294
295                 ret = dpio_add_static_dequeue_channel(dpio_dev->dpio,
296                                                       CMD_PRI_LOW,
297                                                       dpio_dev->token,
298                                                       evq_info->dpcon->dpcon_id,
299                                                       &channel_index);
300                 if (ret < 0) {
301                         DPAA2_EVENTDEV_ERR(
302                                 "Static dequeue config failed: err(%d)", ret);
303                         goto err;
304                 }
305
306                 qbman_swp_push_set(swp, channel_index, 1);
307                 evq_info->dpcon->channel_index = channel_index;
308         }
309         dpaa2_portal->is_port_linked = true;
310
311 skip_linking:
312         /* Check if there are atomic contexts to be released */
313         while (DPAA2_PER_LCORE_DQRR_SIZE) {
314                 if (DPAA2_PER_LCORE_DQRR_HELD & (1 << i)) {
315                         qbman_swp_dqrr_idx_consume(swp, i);
316                         DPAA2_PER_LCORE_DQRR_SIZE--;
317                         DPAA2_PER_LCORE_DQRR_MBUF(i)->seqn =
318                                 DPAA2_INVALID_MBUF_SEQN;
319                 }
320                 i++;
321         }
322         DPAA2_PER_LCORE_DQRR_HELD = 0;
323
324         do {
325                 dq = qbman_swp_dqrr_next(swp);
326                 if (!dq) {
327                         if (!num_pkts && timeout_ticks) {
328                                 dpaa2_eventdev_dequeue_wait(timeout_ticks);
329                                 timeout_ticks = 0;
330                                 continue;
331                         }
332                         return num_pkts;
333                 }
334                 qbman_swp_prefetch_dqrr_next(swp);
335
336                 fd = qbman_result_DQ_fd(dq);
337                 rxq = (struct dpaa2_queue *)(size_t)qbman_result_DQ_fqd_ctx(dq);
338                 if (rxq) {
339                         rxq->cb(swp, fd, dq, rxq, &ev[num_pkts]);
340                 } else {
341                         qbman_swp_dqrr_consume(swp, dq);
342                         DPAA2_EVENTDEV_ERR("Null Return VQ received");
343                         return 0;
344                 }
345
346                 num_pkts++;
347         } while (num_pkts < nb_events);
348
349         return num_pkts;
350 err:
351         for (n = 0; n < i; n++) {
352                 evq_info = &dpaa2_portal->evq_info[n];
353                 if (!evq_info->event_port)
354                         continue;
355
356                 qbman_swp_push_set(swp, evq_info->dpcon->channel_index, 0);
357                 dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0,
358                                                         dpio_dev->token,
359                                                 evq_info->dpcon->dpcon_id);
360         }
361         return 0;
362 }
363
364 static uint16_t
365 dpaa2_eventdev_dequeue(void *port, struct rte_event *ev,
366                        uint64_t timeout_ticks)
367 {
368         return dpaa2_eventdev_dequeue_burst(port, ev, 1, timeout_ticks);
369 }
370
371 static void
372 dpaa2_eventdev_info_get(struct rte_eventdev *dev,
373                         struct rte_event_dev_info *dev_info)
374 {
375         struct dpaa2_eventdev *priv = dev->data->dev_private;
376
377         EVENTDEV_INIT_FUNC_TRACE();
378
379         RTE_SET_USED(dev);
380
381         memset(dev_info, 0, sizeof(struct rte_event_dev_info));
382         dev_info->min_dequeue_timeout_ns =
383                 DPAA2_EVENT_MIN_DEQUEUE_TIMEOUT;
384         dev_info->max_dequeue_timeout_ns =
385                 DPAA2_EVENT_MAX_DEQUEUE_TIMEOUT;
386         dev_info->dequeue_timeout_ns =
387                 DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS;
388         dev_info->max_event_queues = priv->max_event_queues;
389         dev_info->max_event_queue_flows =
390                 DPAA2_EVENT_MAX_QUEUE_FLOWS;
391         dev_info->max_event_queue_priority_levels =
392                 DPAA2_EVENT_MAX_QUEUE_PRIORITY_LEVELS;
393         dev_info->max_event_priority_levels =
394                 DPAA2_EVENT_MAX_EVENT_PRIORITY_LEVELS;
395         dev_info->max_event_ports = rte_fslmc_get_device_count(DPAA2_IO);
396         /* we only support dpio up to number of cores */
397         if (dev_info->max_event_ports > rte_lcore_count())
398                 dev_info->max_event_ports = rte_lcore_count();
399         dev_info->max_event_port_dequeue_depth =
400                 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
401         dev_info->max_event_port_enqueue_depth =
402                 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH;
403         dev_info->max_num_events = DPAA2_EVENT_MAX_NUM_EVENTS;
404         dev_info->event_dev_cap = RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
405                 RTE_EVENT_DEV_CAP_BURST_MODE|
406                 RTE_EVENT_DEV_CAP_RUNTIME_PORT_LINK |
407                 RTE_EVENT_DEV_CAP_MULTIPLE_QUEUE_PORT |
408                 RTE_EVENT_DEV_CAP_NONSEQ_MODE |
409                 RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES |
410                 RTE_EVENT_DEV_CAP_CARRY_FLOW_ID;
411
412 }
413
414 static int
415 dpaa2_eventdev_configure(const struct rte_eventdev *dev)
416 {
417         struct dpaa2_eventdev *priv = dev->data->dev_private;
418         struct rte_event_dev_config *conf = &dev->data->dev_conf;
419
420         EVENTDEV_INIT_FUNC_TRACE();
421
422         priv->nb_event_queues = conf->nb_event_queues;
423         priv->nb_event_ports = conf->nb_event_ports;
424         priv->nb_event_queue_flows = conf->nb_event_queue_flows;
425         priv->nb_event_port_dequeue_depth = conf->nb_event_port_dequeue_depth;
426         priv->nb_event_port_enqueue_depth = conf->nb_event_port_enqueue_depth;
427         priv->event_dev_cfg = conf->event_dev_cfg;
428
429         /* Check dequeue timeout method is per dequeue or global */
430         if (priv->event_dev_cfg & RTE_EVENT_DEV_CFG_PER_DEQUEUE_TIMEOUT) {
431                 /*
432                  * Use timeout value as given in dequeue operation.
433                  * So invalidating this timeout value.
434                  */
435                 priv->dequeue_timeout_ns = 0;
436
437         } else if (conf->dequeue_timeout_ns == 0) {
438                 priv->dequeue_timeout_ns = DPAA2_EVENT_PORT_DEQUEUE_TIMEOUT_NS;
439         } else {
440                 priv->dequeue_timeout_ns = conf->dequeue_timeout_ns;
441         }
442
443         DPAA2_EVENTDEV_DEBUG("Configured eventdev devid=%d",
444                              dev->data->dev_id);
445         return 0;
446 }
447
448 static int
449 dpaa2_eventdev_start(struct rte_eventdev *dev)
450 {
451         EVENTDEV_INIT_FUNC_TRACE();
452
453         RTE_SET_USED(dev);
454
455         return 0;
456 }
457
458 static void
459 dpaa2_eventdev_stop(struct rte_eventdev *dev)
460 {
461         EVENTDEV_INIT_FUNC_TRACE();
462
463         RTE_SET_USED(dev);
464 }
465
466 static int
467 dpaa2_eventdev_close(struct rte_eventdev *dev)
468 {
469         EVENTDEV_INIT_FUNC_TRACE();
470
471         RTE_SET_USED(dev);
472
473         return 0;
474 }
475
476 static void
477 dpaa2_eventdev_queue_def_conf(struct rte_eventdev *dev, uint8_t queue_id,
478                               struct rte_event_queue_conf *queue_conf)
479 {
480         EVENTDEV_INIT_FUNC_TRACE();
481
482         RTE_SET_USED(dev);
483         RTE_SET_USED(queue_id);
484
485         queue_conf->nb_atomic_flows = DPAA2_EVENT_QUEUE_ATOMIC_FLOWS;
486         queue_conf->nb_atomic_order_sequences =
487                                 DPAA2_EVENT_QUEUE_ORDER_SEQUENCES;
488         queue_conf->schedule_type = RTE_SCHED_TYPE_PARALLEL;
489         queue_conf->priority = RTE_EVENT_DEV_PRIORITY_NORMAL;
490 }
491
492 static int
493 dpaa2_eventdev_queue_setup(struct rte_eventdev *dev, uint8_t queue_id,
494                            const struct rte_event_queue_conf *queue_conf)
495 {
496         struct dpaa2_eventdev *priv = dev->data->dev_private;
497         struct dpaa2_eventq *evq_info = &priv->evq_info[queue_id];
498
499         EVENTDEV_INIT_FUNC_TRACE();
500
501         switch (queue_conf->schedule_type) {
502         case RTE_SCHED_TYPE_PARALLEL:
503         case RTE_SCHED_TYPE_ATOMIC:
504         case RTE_SCHED_TYPE_ORDERED:
505                 break;
506         default:
507                 DPAA2_EVENTDEV_ERR("Schedule type is not supported.");
508                 return -1;
509         }
510         evq_info->event_queue_cfg = queue_conf->event_queue_cfg;
511         evq_info->event_queue_id = queue_id;
512
513         return 0;
514 }
515
516 static void
517 dpaa2_eventdev_queue_release(struct rte_eventdev *dev, uint8_t queue_id)
518 {
519         EVENTDEV_INIT_FUNC_TRACE();
520
521         RTE_SET_USED(dev);
522         RTE_SET_USED(queue_id);
523 }
524
525 static void
526 dpaa2_eventdev_port_def_conf(struct rte_eventdev *dev, uint8_t port_id,
527                              struct rte_event_port_conf *port_conf)
528 {
529         EVENTDEV_INIT_FUNC_TRACE();
530
531         RTE_SET_USED(dev);
532         RTE_SET_USED(port_id);
533
534         port_conf->new_event_threshold =
535                 DPAA2_EVENT_MAX_NUM_EVENTS;
536         port_conf->dequeue_depth =
537                 DPAA2_EVENT_MAX_PORT_DEQUEUE_DEPTH;
538         port_conf->enqueue_depth =
539                 DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH;
540         port_conf->event_port_cfg = 0;
541 }
542
543 static int
544 dpaa2_eventdev_port_setup(struct rte_eventdev *dev, uint8_t port_id,
545                           const struct rte_event_port_conf *port_conf)
546 {
547         char event_port_name[32];
548         struct dpaa2_port *portal;
549
550         EVENTDEV_INIT_FUNC_TRACE();
551
552         RTE_SET_USED(port_conf);
553
554         sprintf(event_port_name, "event-port-%d", port_id);
555         portal = rte_malloc(event_port_name, sizeof(struct dpaa2_port), 0);
556         if (!portal) {
557                 DPAA2_EVENTDEV_ERR("Memory allocation failure");
558                 return -ENOMEM;
559         }
560
561         memset(portal, 0, sizeof(struct dpaa2_port));
562         dev->data->ports[port_id] = portal;
563         return 0;
564 }
565
566 static void
567 dpaa2_eventdev_port_release(void *port)
568 {
569         struct dpaa2_port *portal = port;
570
571         EVENTDEV_INIT_FUNC_TRACE();
572
573         if (portal == NULL)
574                 return;
575
576         /* TODO: Cleanup is required when ports are in linked state. */
577         if (portal->is_port_linked)
578                 DPAA2_EVENTDEV_WARN("Event port must be unlinked before release");
579
580         rte_free(portal);
581 }
582
583 static int
584 dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port,
585                          const uint8_t queues[], const uint8_t priorities[],
586                         uint16_t nb_links)
587 {
588         struct dpaa2_eventdev *priv = dev->data->dev_private;
589         struct dpaa2_port *dpaa2_portal = port;
590         struct dpaa2_eventq *evq_info;
591         uint16_t i;
592
593         EVENTDEV_INIT_FUNC_TRACE();
594
595         RTE_SET_USED(priorities);
596
597         for (i = 0; i < nb_links; i++) {
598                 evq_info = &priv->evq_info[queues[i]];
599                 memcpy(&dpaa2_portal->evq_info[queues[i]], evq_info,
600                            sizeof(struct dpaa2_eventq));
601                 dpaa2_portal->evq_info[queues[i]].event_port = port;
602                 dpaa2_portal->num_linked_evq++;
603         }
604
605         return (int)nb_links;
606 }
607
608 static int
609 dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port,
610                            uint8_t queues[], uint16_t nb_unlinks)
611 {
612         struct dpaa2_port *dpaa2_portal = port;
613         int i;
614         struct dpaa2_dpio_dev *dpio_dev = NULL;
615         struct dpaa2_eventq *evq_info;
616         struct qbman_swp *swp;
617
618         EVENTDEV_INIT_FUNC_TRACE();
619
620         RTE_SET_USED(dev);
621         RTE_SET_USED(queues);
622
623         for (i = 0; i < nb_unlinks; i++) {
624                 evq_info = &dpaa2_portal->evq_info[queues[i]];
625
626                 if (DPAA2_PER_LCORE_DPIO && evq_info->dpcon) {
627                         /* todo dpaa2_portal shall have dpio_dev-no per lcore*/
628                         dpio_dev = DPAA2_PER_LCORE_DPIO;
629                         swp = DPAA2_PER_LCORE_PORTAL;
630
631                         qbman_swp_push_set(swp,
632                                         evq_info->dpcon->channel_index, 0);
633                         dpio_remove_static_dequeue_channel(dpio_dev->dpio, 0,
634                                                 dpio_dev->token,
635                                                 evq_info->dpcon->dpcon_id);
636                 }
637                 memset(evq_info, 0, sizeof(struct dpaa2_eventq));
638                 if (dpaa2_portal->num_linked_evq)
639                         dpaa2_portal->num_linked_evq--;
640         }
641
642         if (!dpaa2_portal->num_linked_evq)
643                 dpaa2_portal->is_port_linked = false;
644
645         return (int)nb_unlinks;
646 }
647
648
649 static int
650 dpaa2_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
651                              uint64_t *timeout_ticks)
652 {
653         uint32_t scale = 1000*1000;
654
655         EVENTDEV_INIT_FUNC_TRACE();
656
657         RTE_SET_USED(dev);
658         *timeout_ticks = ns / scale;
659
660         return 0;
661 }
662
663 static void
664 dpaa2_eventdev_dump(struct rte_eventdev *dev, FILE *f)
665 {
666         EVENTDEV_INIT_FUNC_TRACE();
667
668         RTE_SET_USED(dev);
669         RTE_SET_USED(f);
670 }
671
672 static int
673 dpaa2_eventdev_eth_caps_get(const struct rte_eventdev *dev,
674                             const struct rte_eth_dev *eth_dev,
675                             uint32_t *caps)
676 {
677         const char *ethdev_driver = eth_dev->device->driver->name;
678
679         EVENTDEV_INIT_FUNC_TRACE();
680
681         RTE_SET_USED(dev);
682
683         if (!strcmp(ethdev_driver, "net_dpaa2"))
684                 *caps = RTE_EVENT_ETH_RX_ADAPTER_DPAA2_CAP;
685         else
686                 *caps = RTE_EVENT_ETH_RX_ADAPTER_SW_CAP;
687
688         return 0;
689 }
690
691 static int
692 dpaa2_eventdev_eth_queue_add_all(const struct rte_eventdev *dev,
693                 const struct rte_eth_dev *eth_dev,
694                 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
695 {
696         struct dpaa2_eventdev *priv = dev->data->dev_private;
697         uint8_t ev_qid = queue_conf->ev.queue_id;
698         struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon;
699         int i, ret;
700
701         EVENTDEV_INIT_FUNC_TRACE();
702
703         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
704                 ret = dpaa2_eth_eventq_attach(eth_dev, i,
705                                               dpcon, queue_conf);
706                 if (ret) {
707                         DPAA2_EVENTDEV_ERR(
708                                 "Event queue attach failed: err(%d)", ret);
709                         goto fail;
710                 }
711         }
712         return 0;
713 fail:
714         for (i = (i - 1); i >= 0 ; i--)
715                 dpaa2_eth_eventq_detach(eth_dev, i);
716
717         return ret;
718 }
719
720 static int
721 dpaa2_eventdev_eth_queue_add(const struct rte_eventdev *dev,
722                 const struct rte_eth_dev *eth_dev,
723                 int32_t rx_queue_id,
724                 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
725 {
726         struct dpaa2_eventdev *priv = dev->data->dev_private;
727         uint8_t ev_qid = queue_conf->ev.queue_id;
728         struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon;
729         int ret;
730
731         EVENTDEV_INIT_FUNC_TRACE();
732
733         if (rx_queue_id == -1)
734                 return dpaa2_eventdev_eth_queue_add_all(dev,
735                                 eth_dev, queue_conf);
736
737         ret = dpaa2_eth_eventq_attach(eth_dev, rx_queue_id,
738                                       dpcon, queue_conf);
739         if (ret) {
740                 DPAA2_EVENTDEV_ERR(
741                         "Event queue attach failed: err(%d)", ret);
742                 return ret;
743         }
744         return 0;
745 }
746
747 static int
748 dpaa2_eventdev_eth_queue_del_all(const struct rte_eventdev *dev,
749                              const struct rte_eth_dev *eth_dev)
750 {
751         int i, ret;
752
753         EVENTDEV_INIT_FUNC_TRACE();
754
755         RTE_SET_USED(dev);
756
757         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
758                 ret = dpaa2_eth_eventq_detach(eth_dev, i);
759                 if (ret) {
760                         DPAA2_EVENTDEV_ERR(
761                                 "Event queue detach failed: err(%d)", ret);
762                         return ret;
763                 }
764         }
765
766         return 0;
767 }
768
769 static int
770 dpaa2_eventdev_eth_queue_del(const struct rte_eventdev *dev,
771                              const struct rte_eth_dev *eth_dev,
772                              int32_t rx_queue_id)
773 {
774         int ret;
775
776         EVENTDEV_INIT_FUNC_TRACE();
777
778         if (rx_queue_id == -1)
779                 return dpaa2_eventdev_eth_queue_del_all(dev, eth_dev);
780
781         ret = dpaa2_eth_eventq_detach(eth_dev, rx_queue_id);
782         if (ret) {
783                 DPAA2_EVENTDEV_ERR(
784                         "Event queue detach failed: err(%d)", ret);
785                 return ret;
786         }
787
788         return 0;
789 }
790
791 static int
792 dpaa2_eventdev_eth_start(const struct rte_eventdev *dev,
793                          const struct rte_eth_dev *eth_dev)
794 {
795         EVENTDEV_INIT_FUNC_TRACE();
796
797         RTE_SET_USED(dev);
798         RTE_SET_USED(eth_dev);
799
800         return 0;
801 }
802
803 static int
804 dpaa2_eventdev_eth_stop(const struct rte_eventdev *dev,
805                         const struct rte_eth_dev *eth_dev)
806 {
807         EVENTDEV_INIT_FUNC_TRACE();
808
809         RTE_SET_USED(dev);
810         RTE_SET_USED(eth_dev);
811
812         return 0;
813 }
814
815 static int
816 dpaa2_eventdev_crypto_caps_get(const struct rte_eventdev *dev,
817                             const struct rte_cryptodev *cdev,
818                             uint32_t *caps)
819 {
820         const char *name = cdev->data->name;
821
822         EVENTDEV_INIT_FUNC_TRACE();
823
824         RTE_SET_USED(dev);
825
826         if (!strncmp(name, "dpsec-", 6))
827                 *caps = RTE_EVENT_CRYPTO_ADAPTER_DPAA2_CAP;
828         else
829                 return -1;
830
831         return 0;
832 }
833
834 static int
835 dpaa2_eventdev_crypto_queue_add_all(const struct rte_eventdev *dev,
836                 const struct rte_cryptodev *cryptodev,
837                 const struct rte_event *ev)
838 {
839         struct dpaa2_eventdev *priv = dev->data->dev_private;
840         uint8_t ev_qid = ev->queue_id;
841         struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon;
842         int i, ret;
843
844         EVENTDEV_INIT_FUNC_TRACE();
845
846         for (i = 0; i < cryptodev->data->nb_queue_pairs; i++) {
847                 ret = dpaa2_sec_eventq_attach(cryptodev, i, dpcon, ev);
848                 if (ret) {
849                         DPAA2_EVENTDEV_ERR("dpaa2_sec_eventq_attach failed: ret %d\n",
850                                     ret);
851                         goto fail;
852                 }
853         }
854         return 0;
855 fail:
856         for (i = (i - 1); i >= 0 ; i--)
857                 dpaa2_sec_eventq_detach(cryptodev, i);
858
859         return ret;
860 }
861
862 static int
863 dpaa2_eventdev_crypto_queue_add(const struct rte_eventdev *dev,
864                 const struct rte_cryptodev *cryptodev,
865                 int32_t rx_queue_id,
866                 const struct rte_event *ev)
867 {
868         struct dpaa2_eventdev *priv = dev->data->dev_private;
869         uint8_t ev_qid = ev->queue_id;
870         struct dpaa2_dpcon_dev *dpcon = priv->evq_info[ev_qid].dpcon;
871         int ret;
872
873         EVENTDEV_INIT_FUNC_TRACE();
874
875         if (rx_queue_id == -1)
876                 return dpaa2_eventdev_crypto_queue_add_all(dev,
877                                 cryptodev, ev);
878
879         ret = dpaa2_sec_eventq_attach(cryptodev, rx_queue_id,
880                                       dpcon, ev);
881         if (ret) {
882                 DPAA2_EVENTDEV_ERR(
883                         "dpaa2_sec_eventq_attach failed: ret: %d\n", ret);
884                 return ret;
885         }
886         return 0;
887 }
888
889 static int
890 dpaa2_eventdev_crypto_queue_del_all(const struct rte_eventdev *dev,
891                              const struct rte_cryptodev *cdev)
892 {
893         int i, ret;
894
895         EVENTDEV_INIT_FUNC_TRACE();
896
897         RTE_SET_USED(dev);
898
899         for (i = 0; i < cdev->data->nb_queue_pairs; i++) {
900                 ret = dpaa2_sec_eventq_detach(cdev, i);
901                 if (ret) {
902                         DPAA2_EVENTDEV_ERR(
903                                 "dpaa2_sec_eventq_detach failed:ret %d\n", ret);
904                         return ret;
905                 }
906         }
907
908         return 0;
909 }
910
911 static int
912 dpaa2_eventdev_crypto_queue_del(const struct rte_eventdev *dev,
913                              const struct rte_cryptodev *cryptodev,
914                              int32_t rx_queue_id)
915 {
916         int ret;
917
918         EVENTDEV_INIT_FUNC_TRACE();
919
920         if (rx_queue_id == -1)
921                 return dpaa2_eventdev_crypto_queue_del_all(dev, cryptodev);
922
923         ret = dpaa2_sec_eventq_detach(cryptodev, rx_queue_id);
924         if (ret) {
925                 DPAA2_EVENTDEV_ERR(
926                         "dpaa2_sec_eventq_detach failed: ret: %d\n", ret);
927                 return ret;
928         }
929
930         return 0;
931 }
932
933 static int
934 dpaa2_eventdev_crypto_start(const struct rte_eventdev *dev,
935                             const struct rte_cryptodev *cryptodev)
936 {
937         EVENTDEV_INIT_FUNC_TRACE();
938
939         RTE_SET_USED(dev);
940         RTE_SET_USED(cryptodev);
941
942         return 0;
943 }
944
945 static int
946 dpaa2_eventdev_crypto_stop(const struct rte_eventdev *dev,
947                            const struct rte_cryptodev *cryptodev)
948 {
949         EVENTDEV_INIT_FUNC_TRACE();
950
951         RTE_SET_USED(dev);
952         RTE_SET_USED(cryptodev);
953
954         return 0;
955 }
956
957 static int
958 dpaa2_eventdev_tx_adapter_create(uint8_t id,
959                                  const struct rte_eventdev *dev)
960 {
961         RTE_SET_USED(id);
962         RTE_SET_USED(dev);
963
964         /* Nothing to do. Simply return. */
965         return 0;
966 }
967
968 static int
969 dpaa2_eventdev_tx_adapter_caps(const struct rte_eventdev *dev,
970                                const struct rte_eth_dev *eth_dev,
971                                uint32_t *caps)
972 {
973         RTE_SET_USED(dev);
974         RTE_SET_USED(eth_dev);
975
976         *caps = RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT;
977         return 0;
978 }
979
980 static uint16_t
981 dpaa2_eventdev_txa_enqueue_same_dest(void *port,
982                                      struct rte_event ev[],
983                                      uint16_t nb_events)
984 {
985         struct rte_mbuf *m[DPAA2_EVENT_MAX_PORT_ENQUEUE_DEPTH], *m0;
986         uint8_t qid, i;
987
988         RTE_SET_USED(port);
989
990         m0 = (struct rte_mbuf *)ev[0].mbuf;
991         qid = rte_event_eth_tx_adapter_txq_get(m0);
992
993         for (i = 0; i < nb_events; i++)
994                 m[i] = (struct rte_mbuf *)ev[i].mbuf;
995
996         return rte_eth_tx_burst(m0->port, qid, m, nb_events);
997 }
998
999 static uint16_t
1000 dpaa2_eventdev_txa_enqueue(void *port,
1001                            struct rte_event ev[],
1002                            uint16_t nb_events)
1003 {
1004         struct rte_mbuf *m = (struct rte_mbuf *)ev[0].mbuf;
1005         uint8_t qid, i;
1006
1007         RTE_SET_USED(port);
1008
1009         for (i = 0; i < nb_events; i++) {
1010                 qid = rte_event_eth_tx_adapter_txq_get(m);
1011                 rte_eth_tx_burst(m->port, qid, &m, 1);
1012         }
1013
1014         return nb_events;
1015 }
1016
1017 static struct rte_eventdev_ops dpaa2_eventdev_ops = {
1018         .dev_infos_get    = dpaa2_eventdev_info_get,
1019         .dev_configure    = dpaa2_eventdev_configure,
1020         .dev_start        = dpaa2_eventdev_start,
1021         .dev_stop         = dpaa2_eventdev_stop,
1022         .dev_close        = dpaa2_eventdev_close,
1023         .queue_def_conf   = dpaa2_eventdev_queue_def_conf,
1024         .queue_setup      = dpaa2_eventdev_queue_setup,
1025         .queue_release    = dpaa2_eventdev_queue_release,
1026         .port_def_conf    = dpaa2_eventdev_port_def_conf,
1027         .port_setup       = dpaa2_eventdev_port_setup,
1028         .port_release     = dpaa2_eventdev_port_release,
1029         .port_link        = dpaa2_eventdev_port_link,
1030         .port_unlink      = dpaa2_eventdev_port_unlink,
1031         .timeout_ticks    = dpaa2_eventdev_timeout_ticks,
1032         .dump             = dpaa2_eventdev_dump,
1033         .dev_selftest     = test_eventdev_dpaa2,
1034         .eth_rx_adapter_caps_get        = dpaa2_eventdev_eth_caps_get,
1035         .eth_rx_adapter_queue_add       = dpaa2_eventdev_eth_queue_add,
1036         .eth_rx_adapter_queue_del       = dpaa2_eventdev_eth_queue_del,
1037         .eth_rx_adapter_start           = dpaa2_eventdev_eth_start,
1038         .eth_rx_adapter_stop            = dpaa2_eventdev_eth_stop,
1039         .eth_tx_adapter_caps_get        = dpaa2_eventdev_tx_adapter_caps,
1040         .eth_tx_adapter_create          = dpaa2_eventdev_tx_adapter_create,
1041         .crypto_adapter_caps_get        = dpaa2_eventdev_crypto_caps_get,
1042         .crypto_adapter_queue_pair_add  = dpaa2_eventdev_crypto_queue_add,
1043         .crypto_adapter_queue_pair_del  = dpaa2_eventdev_crypto_queue_del,
1044         .crypto_adapter_start           = dpaa2_eventdev_crypto_start,
1045         .crypto_adapter_stop            = dpaa2_eventdev_crypto_stop,
1046 };
1047
1048 static int
1049 dpaa2_eventdev_setup_dpci(struct dpaa2_dpci_dev *dpci_dev,
1050                           struct dpaa2_dpcon_dev *dpcon_dev)
1051 {
1052         struct dpci_rx_queue_cfg rx_queue_cfg;
1053         int ret, i;
1054
1055         /*Do settings to get the frame on a DPCON object*/
1056         rx_queue_cfg.options = DPCI_QUEUE_OPT_DEST |
1057                   DPCI_QUEUE_OPT_USER_CTX;
1058         rx_queue_cfg.dest_cfg.dest_type = DPCI_DEST_DPCON;
1059         rx_queue_cfg.dest_cfg.dest_id = dpcon_dev->dpcon_id;
1060         rx_queue_cfg.dest_cfg.priority = DPAA2_EVENT_DEFAULT_DPCI_PRIO;
1061
1062         dpci_dev->rx_queue[DPAA2_EVENT_DPCI_PARALLEL_QUEUE].cb =
1063                 dpaa2_eventdev_process_parallel;
1064         dpci_dev->rx_queue[DPAA2_EVENT_DPCI_ATOMIC_QUEUE].cb =
1065                 dpaa2_eventdev_process_atomic;
1066
1067         for (i = 0 ; i < DPAA2_EVENT_DPCI_MAX_QUEUES; i++) {
1068                 rx_queue_cfg.user_ctx = (size_t)(&dpci_dev->rx_queue[i]);
1069                 ret = dpci_set_rx_queue(&dpci_dev->dpci,
1070                                         CMD_PRI_LOW,
1071                                         dpci_dev->token, i,
1072                                         &rx_queue_cfg);
1073                 if (ret) {
1074                         DPAA2_EVENTDEV_ERR(
1075                                 "DPCI Rx queue setup failed: err(%d)",
1076                                 ret);
1077                         return ret;
1078                 }
1079         }
1080         return 0;
1081 }
1082
1083 static int
1084 dpaa2_eventdev_create(const char *name)
1085 {
1086         struct rte_eventdev *eventdev;
1087         struct dpaa2_eventdev *priv;
1088         struct dpaa2_dpcon_dev *dpcon_dev = NULL;
1089         struct dpaa2_dpci_dev *dpci_dev = NULL;
1090         int ret;
1091
1092         eventdev = rte_event_pmd_vdev_init(name,
1093                                            sizeof(struct dpaa2_eventdev),
1094                                            rte_socket_id());
1095         if (eventdev == NULL) {
1096                 DPAA2_EVENTDEV_ERR("Failed to create Event device %s", name);
1097                 goto fail;
1098         }
1099
1100         eventdev->dev_ops       = &dpaa2_eventdev_ops;
1101         eventdev->enqueue       = dpaa2_eventdev_enqueue;
1102         eventdev->enqueue_burst = dpaa2_eventdev_enqueue_burst;
1103         eventdev->enqueue_new_burst = dpaa2_eventdev_enqueue_burst;
1104         eventdev->enqueue_forward_burst = dpaa2_eventdev_enqueue_burst;
1105         eventdev->dequeue       = dpaa2_eventdev_dequeue;
1106         eventdev->dequeue_burst = dpaa2_eventdev_dequeue_burst;
1107         eventdev->txa_enqueue   = dpaa2_eventdev_txa_enqueue;
1108         eventdev->txa_enqueue_same_dest = dpaa2_eventdev_txa_enqueue_same_dest;
1109
1110         /* For secondary processes, the primary has done all the work */
1111         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1112                 return 0;
1113
1114         priv = eventdev->data->dev_private;
1115         priv->max_event_queues = 0;
1116
1117         do {
1118                 dpcon_dev = rte_dpaa2_alloc_dpcon_dev();
1119                 if (!dpcon_dev)
1120                         break;
1121                 priv->evq_info[priv->max_event_queues].dpcon = dpcon_dev;
1122
1123                 dpci_dev = rte_dpaa2_alloc_dpci_dev();
1124                 if (!dpci_dev) {
1125                         rte_dpaa2_free_dpcon_dev(dpcon_dev);
1126                         break;
1127                 }
1128                 priv->evq_info[priv->max_event_queues].dpci = dpci_dev;
1129
1130                 ret = dpaa2_eventdev_setup_dpci(dpci_dev, dpcon_dev);
1131                 if (ret) {
1132                         DPAA2_EVENTDEV_ERR(
1133                                     "DPCI setup failed: err(%d)", ret);
1134                         return ret;
1135                 }
1136                 priv->max_event_queues++;
1137         } while (dpcon_dev && dpci_dev);
1138
1139         RTE_LOG(INFO, PMD, "%s eventdev created\n", name);
1140
1141         return 0;
1142 fail:
1143         return -EFAULT;
1144 }
1145
1146 static int
1147 dpaa2_eventdev_destroy(const char *name)
1148 {
1149         struct rte_eventdev *eventdev;
1150         struct dpaa2_eventdev *priv;
1151         int i;
1152
1153         eventdev = rte_event_pmd_get_named_dev(name);
1154         if (eventdev == NULL) {
1155                 RTE_EDEV_LOG_ERR("eventdev with name %s not allocated", name);
1156                 return -1;
1157         }
1158
1159         /* For secondary processes, the primary has done all the work */
1160         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1161                 return 0;
1162
1163         priv = eventdev->data->dev_private;
1164         for (i = 0; i < priv->max_event_queues; i++) {
1165                 if (priv->evq_info[i].dpcon)
1166                         rte_dpaa2_free_dpcon_dev(priv->evq_info[i].dpcon);
1167
1168                 if (priv->evq_info[i].dpci)
1169                         rte_dpaa2_free_dpci_dev(priv->evq_info[i].dpci);
1170
1171         }
1172         priv->max_event_queues = 0;
1173
1174         RTE_LOG(INFO, PMD, "%s eventdev cleaned\n", name);
1175         return 0;
1176 }
1177
1178
1179 static int
1180 dpaa2_eventdev_probe(struct rte_vdev_device *vdev)
1181 {
1182         const char *name;
1183
1184         name = rte_vdev_device_name(vdev);
1185         DPAA2_EVENTDEV_INFO("Initializing %s", name);
1186         return dpaa2_eventdev_create(name);
1187 }
1188
1189 static int
1190 dpaa2_eventdev_remove(struct rte_vdev_device *vdev)
1191 {
1192         const char *name;
1193
1194         name = rte_vdev_device_name(vdev);
1195         DPAA2_EVENTDEV_INFO("Closing %s", name);
1196
1197         dpaa2_eventdev_destroy(name);
1198
1199         return rte_event_pmd_vdev_uninit(name);
1200 }
1201
1202 static struct rte_vdev_driver vdev_eventdev_dpaa2_pmd = {
1203         .probe = dpaa2_eventdev_probe,
1204         .remove = dpaa2_eventdev_remove
1205 };
1206
1207 RTE_PMD_REGISTER_VDEV(EVENTDEV_NAME_DPAA2_PMD, vdev_eventdev_dpaa2_pmd);
1208 RTE_LOG_REGISTER(dpaa2_logtype_event, pmd.event.dpaa2, NOTICE);