cryptodev: rename function to check device validity
[dpdk.git] / lib / eventdev / rte_event_crypto_adapter.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2018 Intel Corporation.
3  * All rights reserved.
4  */
5
6 #include <string.h>
7 #include <stdbool.h>
8 #include <rte_common.h>
9 #include <rte_dev.h>
10 #include <rte_errno.h>
11 #include <rte_cryptodev.h>
12 #include <rte_cryptodev_pmd.h>
13 #include <rte_log.h>
14 #include <rte_malloc.h>
15 #include <rte_service_component.h>
16
17 #include "rte_eventdev.h"
18 #include "eventdev_pmd.h"
19 #include "rte_eventdev_trace.h"
20 #include "rte_event_crypto_adapter.h"
21
22 #define BATCH_SIZE 32
23 #define DEFAULT_MAX_NB 128
24 #define CRYPTO_ADAPTER_NAME_LEN 32
25 #define CRYPTO_ADAPTER_MEM_NAME_LEN 32
26 #define CRYPTO_ADAPTER_MAX_EV_ENQ_RETRIES 100
27
28 /* Flush an instance's enqueue buffers every CRYPTO_ENQ_FLUSH_THRESHOLD
29  * iterations of eca_crypto_adapter_enq_run()
30  */
31 #define CRYPTO_ENQ_FLUSH_THRESHOLD 1024
32
33 struct rte_event_crypto_adapter {
34         /* Event device identifier */
35         uint8_t eventdev_id;
36         /* Event port identifier */
37         uint8_t event_port_id;
38         /* Store event device's implicit release capability */
39         uint8_t implicit_release_disabled;
40         /* Max crypto ops processed in any service function invocation */
41         uint32_t max_nb;
42         /* Lock to serialize config updates with service function */
43         rte_spinlock_t lock;
44         /* Next crypto device to be processed */
45         uint16_t next_cdev_id;
46         /* Per crypto device structure */
47         struct crypto_device_info *cdevs;
48         /* Loop counter to flush crypto ops */
49         uint16_t transmit_loop_count;
50         /* Per instance stats structure */
51         struct rte_event_crypto_adapter_stats crypto_stats;
52         /* Configuration callback for rte_service configuration */
53         rte_event_crypto_adapter_conf_cb conf_cb;
54         /* Configuration callback argument */
55         void *conf_arg;
56         /* Set if  default_cb is being used */
57         int default_cb_arg;
58         /* Service initialization state */
59         uint8_t service_inited;
60         /* Memory allocation name */
61         char mem_name[CRYPTO_ADAPTER_MEM_NAME_LEN];
62         /* Socket identifier cached from eventdev */
63         int socket_id;
64         /* Per adapter EAL service */
65         uint32_t service_id;
66         /* No. of queue pairs configured */
67         uint16_t nb_qps;
68         /* Adapter mode */
69         enum rte_event_crypto_adapter_mode mode;
70 } __rte_cache_aligned;
71
72 /* Per crypto device information */
73 struct crypto_device_info {
74         /* Pointer to cryptodev */
75         struct rte_cryptodev *dev;
76         /* Pointer to queue pair info */
77         struct crypto_queue_pair_info *qpairs;
78         /* Next queue pair to be processed */
79         uint16_t next_queue_pair_id;
80         /* Set to indicate cryptodev->eventdev packet
81          * transfer uses a hardware mechanism
82          */
83         uint8_t internal_event_port;
84         /* Set to indicate processing has been started */
85         uint8_t dev_started;
86         /* If num_qpairs > 0, the start callback will
87          * be invoked if not already invoked
88          */
89         uint16_t num_qpairs;
90 } __rte_cache_aligned;
91
92 /* Per queue pair information */
93 struct crypto_queue_pair_info {
94         /* Set to indicate queue pair is enabled */
95         bool qp_enabled;
96         /* Pointer to hold rte_crypto_ops for batching */
97         struct rte_crypto_op **op_buffer;
98         /* No of crypto ops accumulated */
99         uint8_t len;
100 } __rte_cache_aligned;
101
102 static struct rte_event_crypto_adapter **event_crypto_adapter;
103
104 /* Macros to check for valid adapter */
105 #define EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \
106         if (!eca_valid_id(id)) { \
107                 RTE_EDEV_LOG_ERR("Invalid crypto adapter id = %d\n", id); \
108                 return retval; \
109         } \
110 } while (0)
111
112 static inline int
113 eca_valid_id(uint8_t id)
114 {
115         return id < RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE;
116 }
117
118 static int
119 eca_init(void)
120 {
121         const char *name = "crypto_adapter_array";
122         const struct rte_memzone *mz;
123         unsigned int sz;
124
125         sz = sizeof(*event_crypto_adapter) *
126             RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE;
127         sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
128
129         mz = rte_memzone_lookup(name);
130         if (mz == NULL) {
131                 mz = rte_memzone_reserve_aligned(name, sz, rte_socket_id(), 0,
132                                                  RTE_CACHE_LINE_SIZE);
133                 if (mz == NULL) {
134                         RTE_EDEV_LOG_ERR("failed to reserve memzone err = %"
135                                         PRId32, rte_errno);
136                         return -rte_errno;
137                 }
138         }
139
140         event_crypto_adapter = mz->addr;
141         return 0;
142 }
143
144 static inline struct rte_event_crypto_adapter *
145 eca_id_to_adapter(uint8_t id)
146 {
147         return event_crypto_adapter ?
148                 event_crypto_adapter[id] : NULL;
149 }
150
151 static int
152 eca_default_config_cb(uint8_t id, uint8_t dev_id,
153                         struct rte_event_crypto_adapter_conf *conf, void *arg)
154 {
155         struct rte_event_dev_config dev_conf;
156         struct rte_eventdev *dev;
157         uint8_t port_id;
158         int started;
159         int ret;
160         struct rte_event_port_conf *port_conf = arg;
161         struct rte_event_crypto_adapter *adapter = eca_id_to_adapter(id);
162
163         if (adapter == NULL)
164                 return -EINVAL;
165
166         dev = &rte_eventdevs[adapter->eventdev_id];
167         dev_conf = dev->data->dev_conf;
168
169         started = dev->data->dev_started;
170         if (started)
171                 rte_event_dev_stop(dev_id);
172         port_id = dev_conf.nb_event_ports;
173         dev_conf.nb_event_ports += 1;
174         ret = rte_event_dev_configure(dev_id, &dev_conf);
175         if (ret) {
176                 RTE_EDEV_LOG_ERR("failed to configure event dev %u\n", dev_id);
177                 if (started) {
178                         if (rte_event_dev_start(dev_id))
179                                 return -EIO;
180                 }
181                 return ret;
182         }
183
184         ret = rte_event_port_setup(dev_id, port_id, port_conf);
185         if (ret) {
186                 RTE_EDEV_LOG_ERR("failed to setup event port %u\n", port_id);
187                 return ret;
188         }
189
190         conf->event_port_id = port_id;
191         conf->max_nb = DEFAULT_MAX_NB;
192         if (started)
193                 ret = rte_event_dev_start(dev_id);
194
195         adapter->default_cb_arg = 1;
196         return ret;
197 }
198
199 int
200 rte_event_crypto_adapter_create_ext(uint8_t id, uint8_t dev_id,
201                                 rte_event_crypto_adapter_conf_cb conf_cb,
202                                 enum rte_event_crypto_adapter_mode mode,
203                                 void *conf_arg)
204 {
205         struct rte_event_crypto_adapter *adapter;
206         char mem_name[CRYPTO_ADAPTER_NAME_LEN];
207         struct rte_event_dev_info dev_info;
208         int socket_id;
209         uint8_t i;
210         int ret;
211
212         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
213         RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
214         if (conf_cb == NULL)
215                 return -EINVAL;
216
217         if (event_crypto_adapter == NULL) {
218                 ret = eca_init();
219                 if (ret)
220                         return ret;
221         }
222
223         adapter = eca_id_to_adapter(id);
224         if (adapter != NULL) {
225                 RTE_EDEV_LOG_ERR("Crypto adapter id %u already exists!", id);
226                 return -EEXIST;
227         }
228
229         socket_id = rte_event_dev_socket_id(dev_id);
230         snprintf(mem_name, CRYPTO_ADAPTER_MEM_NAME_LEN,
231                  "rte_event_crypto_adapter_%d", id);
232
233         adapter = rte_zmalloc_socket(mem_name, sizeof(*adapter),
234                         RTE_CACHE_LINE_SIZE, socket_id);
235         if (adapter == NULL) {
236                 RTE_EDEV_LOG_ERR("Failed to get mem for event crypto adapter!");
237                 return -ENOMEM;
238         }
239
240         ret = rte_event_dev_info_get(dev_id, &dev_info);
241         if (ret < 0) {
242                 RTE_EDEV_LOG_ERR("Failed to get info for eventdev %d: %s!",
243                                  dev_id, dev_info.driver_name);
244                 rte_free(adapter);
245                 return ret;
246         }
247
248         adapter->implicit_release_disabled = (dev_info.event_dev_cap &
249                         RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
250         adapter->eventdev_id = dev_id;
251         adapter->socket_id = socket_id;
252         adapter->conf_cb = conf_cb;
253         adapter->conf_arg = conf_arg;
254         adapter->mode = mode;
255         strcpy(adapter->mem_name, mem_name);
256         adapter->cdevs = rte_zmalloc_socket(adapter->mem_name,
257                                         rte_cryptodev_count() *
258                                         sizeof(struct crypto_device_info), 0,
259                                         socket_id);
260         if (adapter->cdevs == NULL) {
261                 RTE_EDEV_LOG_ERR("Failed to get mem for crypto devices\n");
262                 rte_free(adapter);
263                 return -ENOMEM;
264         }
265
266         rte_spinlock_init(&adapter->lock);
267         for (i = 0; i < rte_cryptodev_count(); i++)
268                 adapter->cdevs[i].dev = rte_cryptodev_pmd_get_dev(i);
269
270         event_crypto_adapter[id] = adapter;
271
272         rte_eventdev_trace_crypto_adapter_create(id, dev_id, adapter, conf_arg,
273                 mode);
274         return 0;
275 }
276
277
278 int
279 rte_event_crypto_adapter_create(uint8_t id, uint8_t dev_id,
280                                 struct rte_event_port_conf *port_config,
281                                 enum rte_event_crypto_adapter_mode mode)
282 {
283         struct rte_event_port_conf *pc;
284         int ret;
285
286         if (port_config == NULL)
287                 return -EINVAL;
288         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
289
290         pc = rte_malloc(NULL, sizeof(*pc), 0);
291         if (pc == NULL)
292                 return -ENOMEM;
293         *pc = *port_config;
294         ret = rte_event_crypto_adapter_create_ext(id, dev_id,
295                                                   eca_default_config_cb,
296                                                   mode,
297                                                   pc);
298         if (ret)
299                 rte_free(pc);
300
301         return ret;
302 }
303
304 int
305 rte_event_crypto_adapter_free(uint8_t id)
306 {
307         struct rte_event_crypto_adapter *adapter;
308
309         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
310
311         adapter = eca_id_to_adapter(id);
312         if (adapter == NULL)
313                 return -EINVAL;
314
315         if (adapter->nb_qps) {
316                 RTE_EDEV_LOG_ERR("%" PRIu16 "Queue pairs not deleted",
317                                 adapter->nb_qps);
318                 return -EBUSY;
319         }
320
321         rte_eventdev_trace_crypto_adapter_free(id, adapter);
322         if (adapter->default_cb_arg)
323                 rte_free(adapter->conf_arg);
324         rte_free(adapter->cdevs);
325         rte_free(adapter);
326         event_crypto_adapter[id] = NULL;
327
328         return 0;
329 }
330
331 static inline unsigned int
332 eca_enq_to_cryptodev(struct rte_event_crypto_adapter *adapter,
333                  struct rte_event *ev, unsigned int cnt)
334 {
335         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
336         union rte_event_crypto_metadata *m_data = NULL;
337         struct crypto_queue_pair_info *qp_info = NULL;
338         struct rte_crypto_op *crypto_op;
339         unsigned int i, n;
340         uint16_t qp_id, len, ret;
341         uint8_t cdev_id;
342
343         len = 0;
344         ret = 0;
345         n = 0;
346         stats->event_deq_count += cnt;
347
348         for (i = 0; i < cnt; i++) {
349                 crypto_op = ev[i].event_ptr;
350                 if (crypto_op == NULL)
351                         continue;
352                 if (crypto_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
353                         m_data = rte_cryptodev_sym_session_get_user_data(
354                                         crypto_op->sym->session);
355                         if (m_data == NULL) {
356                                 rte_pktmbuf_free(crypto_op->sym->m_src);
357                                 rte_crypto_op_free(crypto_op);
358                                 continue;
359                         }
360
361                         cdev_id = m_data->request_info.cdev_id;
362                         qp_id = m_data->request_info.queue_pair_id;
363                         qp_info = &adapter->cdevs[cdev_id].qpairs[qp_id];
364                         if (!qp_info->qp_enabled) {
365                                 rte_pktmbuf_free(crypto_op->sym->m_src);
366                                 rte_crypto_op_free(crypto_op);
367                                 continue;
368                         }
369                         len = qp_info->len;
370                         qp_info->op_buffer[len] = crypto_op;
371                         len++;
372                 } else if (crypto_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&
373                                 crypto_op->private_data_offset) {
374                         m_data = (union rte_event_crypto_metadata *)
375                                  ((uint8_t *)crypto_op +
376                                         crypto_op->private_data_offset);
377                         cdev_id = m_data->request_info.cdev_id;
378                         qp_id = m_data->request_info.queue_pair_id;
379                         qp_info = &adapter->cdevs[cdev_id].qpairs[qp_id];
380                         if (!qp_info->qp_enabled) {
381                                 rte_pktmbuf_free(crypto_op->sym->m_src);
382                                 rte_crypto_op_free(crypto_op);
383                                 continue;
384                         }
385                         len = qp_info->len;
386                         qp_info->op_buffer[len] = crypto_op;
387                         len++;
388                 } else {
389                         rte_pktmbuf_free(crypto_op->sym->m_src);
390                         rte_crypto_op_free(crypto_op);
391                         continue;
392                 }
393
394                 if (len == BATCH_SIZE) {
395                         struct rte_crypto_op **op_buffer = qp_info->op_buffer;
396                         ret = rte_cryptodev_enqueue_burst(cdev_id,
397                                                           qp_id,
398                                                           op_buffer,
399                                                           BATCH_SIZE);
400
401                         stats->crypto_enq_count += ret;
402
403                         while (ret < len) {
404                                 struct rte_crypto_op *op;
405                                 op = op_buffer[ret++];
406                                 stats->crypto_enq_fail++;
407                                 rte_pktmbuf_free(op->sym->m_src);
408                                 rte_crypto_op_free(op);
409                         }
410
411                         len = 0;
412                 }
413
414                 if (qp_info)
415                         qp_info->len = len;
416                 n += ret;
417         }
418
419         return n;
420 }
421
422 static unsigned int
423 eca_crypto_enq_flush(struct rte_event_crypto_adapter *adapter)
424 {
425         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
426         struct crypto_device_info *curr_dev;
427         struct crypto_queue_pair_info *curr_queue;
428         struct rte_crypto_op **op_buffer;
429         struct rte_cryptodev *dev;
430         uint8_t cdev_id;
431         uint16_t qp;
432         uint16_t ret;
433         uint16_t num_cdev = rte_cryptodev_count();
434
435         ret = 0;
436         for (cdev_id = 0; cdev_id < num_cdev; cdev_id++) {
437                 curr_dev = &adapter->cdevs[cdev_id];
438                 dev = curr_dev->dev;
439                 if (dev == NULL)
440                         continue;
441                 for (qp = 0; qp < dev->data->nb_queue_pairs; qp++) {
442
443                         curr_queue = &curr_dev->qpairs[qp];
444                         if (!curr_queue->qp_enabled)
445                                 continue;
446
447                         op_buffer = curr_queue->op_buffer;
448                         ret = rte_cryptodev_enqueue_burst(cdev_id,
449                                                           qp,
450                                                           op_buffer,
451                                                           curr_queue->len);
452                         stats->crypto_enq_count += ret;
453
454                         while (ret < curr_queue->len) {
455                                 struct rte_crypto_op *op;
456                                 op = op_buffer[ret++];
457                                 stats->crypto_enq_fail++;
458                                 rte_pktmbuf_free(op->sym->m_src);
459                                 rte_crypto_op_free(op);
460                         }
461                         curr_queue->len = 0;
462                 }
463         }
464
465         return ret;
466 }
467
468 static int
469 eca_crypto_adapter_enq_run(struct rte_event_crypto_adapter *adapter,
470                         unsigned int max_enq)
471 {
472         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
473         struct rte_event ev[BATCH_SIZE];
474         unsigned int nb_enq, nb_enqueued;
475         uint16_t n;
476         uint8_t event_dev_id = adapter->eventdev_id;
477         uint8_t event_port_id = adapter->event_port_id;
478
479         nb_enqueued = 0;
480         if (adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)
481                 return 0;
482
483         for (nb_enq = 0; nb_enq < max_enq; nb_enq += n) {
484                 stats->event_poll_count++;
485                 n = rte_event_dequeue_burst(event_dev_id,
486                                             event_port_id, ev, BATCH_SIZE, 0);
487
488                 if (!n)
489                         break;
490
491                 nb_enqueued += eca_enq_to_cryptodev(adapter, ev, n);
492         }
493
494         if ((++adapter->transmit_loop_count &
495                 (CRYPTO_ENQ_FLUSH_THRESHOLD - 1)) == 0) {
496                 nb_enqueued += eca_crypto_enq_flush(adapter);
497         }
498
499         return nb_enqueued;
500 }
501
502 static inline void
503 eca_ops_enqueue_burst(struct rte_event_crypto_adapter *adapter,
504                   struct rte_crypto_op **ops, uint16_t num)
505 {
506         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
507         union rte_event_crypto_metadata *m_data = NULL;
508         uint8_t event_dev_id = adapter->eventdev_id;
509         uint8_t event_port_id = adapter->event_port_id;
510         struct rte_event events[BATCH_SIZE];
511         uint16_t nb_enqueued, nb_ev;
512         uint8_t retry;
513         uint8_t i;
514
515         nb_ev = 0;
516         retry = 0;
517         nb_enqueued = 0;
518         num = RTE_MIN(num, BATCH_SIZE);
519         for (i = 0; i < num; i++) {
520                 struct rte_event *ev = &events[nb_ev++];
521                 if (ops[i]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
522                         m_data = rte_cryptodev_sym_session_get_user_data(
523                                         ops[i]->sym->session);
524                 } else if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&
525                                 ops[i]->private_data_offset) {
526                         m_data = (union rte_event_crypto_metadata *)
527                                  ((uint8_t *)ops[i] +
528                                   ops[i]->private_data_offset);
529                 }
530
531                 if (unlikely(m_data == NULL)) {
532                         rte_pktmbuf_free(ops[i]->sym->m_src);
533                         rte_crypto_op_free(ops[i]);
534                         continue;
535                 }
536
537                 rte_memcpy(ev, &m_data->response_info, sizeof(*ev));
538                 ev->event_ptr = ops[i];
539                 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
540                 if (adapter->implicit_release_disabled)
541                         ev->op = RTE_EVENT_OP_FORWARD;
542                 else
543                         ev->op = RTE_EVENT_OP_NEW;
544         }
545
546         do {
547                 nb_enqueued += rte_event_enqueue_burst(event_dev_id,
548                                                   event_port_id,
549                                                   &events[nb_enqueued],
550                                                   nb_ev - nb_enqueued);
551         } while (retry++ < CRYPTO_ADAPTER_MAX_EV_ENQ_RETRIES &&
552                  nb_enqueued < nb_ev);
553
554         /* Free mbufs and rte_crypto_ops for failed events */
555         for (i = nb_enqueued; i < nb_ev; i++) {
556                 struct rte_crypto_op *op = events[i].event_ptr;
557                 rte_pktmbuf_free(op->sym->m_src);
558                 rte_crypto_op_free(op);
559         }
560
561         stats->event_enq_fail_count += nb_ev - nb_enqueued;
562         stats->event_enq_count += nb_enqueued;
563         stats->event_enq_retry_count += retry - 1;
564 }
565
566 static inline unsigned int
567 eca_crypto_adapter_deq_run(struct rte_event_crypto_adapter *adapter,
568                         unsigned int max_deq)
569 {
570         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
571         struct crypto_device_info *curr_dev;
572         struct crypto_queue_pair_info *curr_queue;
573         struct rte_crypto_op *ops[BATCH_SIZE];
574         uint16_t n, nb_deq;
575         struct rte_cryptodev *dev;
576         uint8_t cdev_id;
577         uint16_t qp, dev_qps;
578         bool done;
579         uint16_t num_cdev = rte_cryptodev_count();
580
581         nb_deq = 0;
582         do {
583                 uint16_t queues = 0;
584                 done = true;
585
586                 for (cdev_id = adapter->next_cdev_id;
587                         cdev_id < num_cdev; cdev_id++) {
588                         curr_dev = &adapter->cdevs[cdev_id];
589                         dev = curr_dev->dev;
590                         if (dev == NULL)
591                                 continue;
592                         dev_qps = dev->data->nb_queue_pairs;
593
594                         for (qp = curr_dev->next_queue_pair_id;
595                                 queues < dev_qps; qp = (qp + 1) % dev_qps,
596                                 queues++) {
597
598                                 curr_queue = &curr_dev->qpairs[qp];
599                                 if (!curr_queue->qp_enabled)
600                                         continue;
601
602                                 n = rte_cryptodev_dequeue_burst(cdev_id, qp,
603                                         ops, BATCH_SIZE);
604                                 if (!n)
605                                         continue;
606
607                                 done = false;
608                                 stats->crypto_deq_count += n;
609                                 eca_ops_enqueue_burst(adapter, ops, n);
610                                 nb_deq += n;
611
612                                 if (nb_deq > max_deq) {
613                                         if ((qp + 1) == dev_qps) {
614                                                 adapter->next_cdev_id =
615                                                         (cdev_id + 1)
616                                                         % num_cdev;
617                                         }
618                                         curr_dev->next_queue_pair_id = (qp + 1)
619                                                 % dev->data->nb_queue_pairs;
620
621                                         return nb_deq;
622                                 }
623                         }
624                 }
625         } while (done == false);
626         return nb_deq;
627 }
628
629 static void
630 eca_crypto_adapter_run(struct rte_event_crypto_adapter *adapter,
631                         unsigned int max_ops)
632 {
633         while (max_ops) {
634                 unsigned int e_cnt, d_cnt;
635
636                 e_cnt = eca_crypto_adapter_deq_run(adapter, max_ops);
637                 max_ops -= RTE_MIN(max_ops, e_cnt);
638
639                 d_cnt = eca_crypto_adapter_enq_run(adapter, max_ops);
640                 max_ops -= RTE_MIN(max_ops, d_cnt);
641
642                 if (e_cnt == 0 && d_cnt == 0)
643                         break;
644
645         }
646 }
647
648 static int
649 eca_service_func(void *args)
650 {
651         struct rte_event_crypto_adapter *adapter = args;
652
653         if (rte_spinlock_trylock(&adapter->lock) == 0)
654                 return 0;
655         eca_crypto_adapter_run(adapter, adapter->max_nb);
656         rte_spinlock_unlock(&adapter->lock);
657
658         return 0;
659 }
660
661 static int
662 eca_init_service(struct rte_event_crypto_adapter *adapter, uint8_t id)
663 {
664         struct rte_event_crypto_adapter_conf adapter_conf;
665         struct rte_service_spec service;
666         int ret;
667
668         if (adapter->service_inited)
669                 return 0;
670
671         memset(&service, 0, sizeof(service));
672         snprintf(service.name, CRYPTO_ADAPTER_NAME_LEN,
673                 "rte_event_crypto_adapter_%d", id);
674         service.socket_id = adapter->socket_id;
675         service.callback = eca_service_func;
676         service.callback_userdata = adapter;
677         /* Service function handles locking for queue add/del updates */
678         service.capabilities = RTE_SERVICE_CAP_MT_SAFE;
679         ret = rte_service_component_register(&service, &adapter->service_id);
680         if (ret) {
681                 RTE_EDEV_LOG_ERR("failed to register service %s err = %" PRId32,
682                         service.name, ret);
683                 return ret;
684         }
685
686         ret = adapter->conf_cb(id, adapter->eventdev_id,
687                 &adapter_conf, adapter->conf_arg);
688         if (ret) {
689                 RTE_EDEV_LOG_ERR("configuration callback failed err = %" PRId32,
690                         ret);
691                 return ret;
692         }
693
694         adapter->max_nb = adapter_conf.max_nb;
695         adapter->event_port_id = adapter_conf.event_port_id;
696         adapter->service_inited = 1;
697
698         return ret;
699 }
700
701 static void
702 eca_update_qp_info(struct rte_event_crypto_adapter *adapter,
703                         struct crypto_device_info *dev_info,
704                         int32_t queue_pair_id,
705                         uint8_t add)
706 {
707         struct crypto_queue_pair_info *qp_info;
708         int enabled;
709         uint16_t i;
710
711         if (dev_info->qpairs == NULL)
712                 return;
713
714         if (queue_pair_id == -1) {
715                 for (i = 0; i < dev_info->dev->data->nb_queue_pairs; i++)
716                         eca_update_qp_info(adapter, dev_info, i, add);
717         } else {
718                 qp_info = &dev_info->qpairs[queue_pair_id];
719                 enabled = qp_info->qp_enabled;
720                 if (add) {
721                         adapter->nb_qps += !enabled;
722                         dev_info->num_qpairs += !enabled;
723                 } else {
724                         adapter->nb_qps -= enabled;
725                         dev_info->num_qpairs -= enabled;
726                 }
727                 qp_info->qp_enabled = !!add;
728         }
729 }
730
731 static int
732 eca_add_queue_pair(struct rte_event_crypto_adapter *adapter,
733                 uint8_t cdev_id,
734                 int queue_pair_id)
735 {
736         struct crypto_device_info *dev_info = &adapter->cdevs[cdev_id];
737         struct crypto_queue_pair_info *qpairs;
738         uint32_t i;
739
740         if (dev_info->qpairs == NULL) {
741                 dev_info->qpairs =
742                     rte_zmalloc_socket(adapter->mem_name,
743                                         dev_info->dev->data->nb_queue_pairs *
744                                         sizeof(struct crypto_queue_pair_info),
745                                         0, adapter->socket_id);
746                 if (dev_info->qpairs == NULL)
747                         return -ENOMEM;
748
749                 qpairs = dev_info->qpairs;
750                 qpairs->op_buffer = rte_zmalloc_socket(adapter->mem_name,
751                                         BATCH_SIZE *
752                                         sizeof(struct rte_crypto_op *),
753                                         0, adapter->socket_id);
754                 if (!qpairs->op_buffer) {
755                         rte_free(qpairs);
756                         return -ENOMEM;
757                 }
758         }
759
760         if (queue_pair_id == -1) {
761                 for (i = 0; i < dev_info->dev->data->nb_queue_pairs; i++)
762                         eca_update_qp_info(adapter, dev_info, i, 1);
763         } else
764                 eca_update_qp_info(adapter, dev_info,
765                                         (uint16_t)queue_pair_id, 1);
766
767         return 0;
768 }
769
770 int
771 rte_event_crypto_adapter_queue_pair_add(uint8_t id,
772                         uint8_t cdev_id,
773                         int32_t queue_pair_id,
774                         const struct rte_event *event)
775 {
776         struct rte_event_crypto_adapter *adapter;
777         struct rte_eventdev *dev;
778         struct crypto_device_info *dev_info;
779         uint32_t cap;
780         int ret;
781
782         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
783
784         if (!rte_cryptodev_is_valid_dev(cdev_id)) {
785                 RTE_EDEV_LOG_ERR("Invalid dev_id=%" PRIu8, cdev_id);
786                 return -EINVAL;
787         }
788
789         adapter = eca_id_to_adapter(id);
790         if (adapter == NULL)
791                 return -EINVAL;
792
793         dev = &rte_eventdevs[adapter->eventdev_id];
794         ret = rte_event_crypto_adapter_caps_get(adapter->eventdev_id,
795                                                 cdev_id,
796                                                 &cap);
797         if (ret) {
798                 RTE_EDEV_LOG_ERR("Failed to get adapter caps dev %" PRIu8
799                         " cdev %" PRIu8, id, cdev_id);
800                 return ret;
801         }
802
803         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) &&
804             (event == NULL)) {
805                 RTE_EDEV_LOG_ERR("Conf value can not be NULL for dev_id=%u",
806                                   cdev_id);
807                 return -EINVAL;
808         }
809
810         dev_info = &adapter->cdevs[cdev_id];
811
812         if (queue_pair_id != -1 &&
813             (uint16_t)queue_pair_id >= dev_info->dev->data->nb_queue_pairs) {
814                 RTE_EDEV_LOG_ERR("Invalid queue_pair_id %" PRIu16,
815                                  (uint16_t)queue_pair_id);
816                 return -EINVAL;
817         }
818
819         /* In case HW cap is RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD,
820          * no need of service core as HW supports event forward capability.
821          */
822         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) ||
823             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND &&
824              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) ||
825             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
826              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)) {
827                 RTE_FUNC_PTR_OR_ERR_RET(
828                         *dev->dev_ops->crypto_adapter_queue_pair_add,
829                         -ENOTSUP);
830                 if (dev_info->qpairs == NULL) {
831                         dev_info->qpairs =
832                             rte_zmalloc_socket(adapter->mem_name,
833                                         dev_info->dev->data->nb_queue_pairs *
834                                         sizeof(struct crypto_queue_pair_info),
835                                         0, adapter->socket_id);
836                         if (dev_info->qpairs == NULL)
837                                 return -ENOMEM;
838                 }
839
840                 ret = (*dev->dev_ops->crypto_adapter_queue_pair_add)(dev,
841                                 dev_info->dev,
842                                 queue_pair_id,
843                                 event);
844                 if (ret)
845                         return ret;
846
847                 else
848                         eca_update_qp_info(adapter, &adapter->cdevs[cdev_id],
849                                            queue_pair_id, 1);
850         }
851
852         /* In case HW cap is RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW,
853          * or SW adapter, initiate services so the application can choose
854          * which ever way it wants to use the adapter.
855          * Case 1: RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW
856          *         Application may wants to use one of below two mode
857          *          a. OP_FORWARD mode -> HW Dequeue + SW enqueue
858          *          b. OP_NEW mode -> HW Dequeue
859          * Case 2: No HW caps, use SW adapter
860          *          a. OP_FORWARD mode -> SW enqueue & dequeue
861          *          b. OP_NEW mode -> SW Dequeue
862          */
863         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
864              !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) &&
865              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_FORWARD) ||
866              (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW) &&
867               !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) &&
868               !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) &&
869                (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA))) {
870                 rte_spinlock_lock(&adapter->lock);
871                 ret = eca_init_service(adapter, id);
872                 if (ret == 0)
873                         ret = eca_add_queue_pair(adapter, cdev_id,
874                                                  queue_pair_id);
875                 rte_spinlock_unlock(&adapter->lock);
876
877                 if (ret)
878                         return ret;
879
880                 rte_service_component_runstate_set(adapter->service_id, 1);
881         }
882
883         rte_eventdev_trace_crypto_adapter_queue_pair_add(id, cdev_id, event,
884                 queue_pair_id);
885         return 0;
886 }
887
888 int
889 rte_event_crypto_adapter_queue_pair_del(uint8_t id, uint8_t cdev_id,
890                                         int32_t queue_pair_id)
891 {
892         struct rte_event_crypto_adapter *adapter;
893         struct crypto_device_info *dev_info;
894         struct rte_eventdev *dev;
895         int ret;
896         uint32_t cap;
897         uint16_t i;
898
899         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
900
901         if (!rte_cryptodev_is_valid_dev(cdev_id)) {
902                 RTE_EDEV_LOG_ERR("Invalid dev_id=%" PRIu8, cdev_id);
903                 return -EINVAL;
904         }
905
906         adapter = eca_id_to_adapter(id);
907         if (adapter == NULL)
908                 return -EINVAL;
909
910         dev = &rte_eventdevs[adapter->eventdev_id];
911         ret = rte_event_crypto_adapter_caps_get(adapter->eventdev_id,
912                                                 cdev_id,
913                                                 &cap);
914         if (ret)
915                 return ret;
916
917         dev_info = &adapter->cdevs[cdev_id];
918
919         if (queue_pair_id != -1 &&
920             (uint16_t)queue_pair_id >= dev_info->dev->data->nb_queue_pairs) {
921                 RTE_EDEV_LOG_ERR("Invalid queue_pair_id %" PRIu16,
922                                  (uint16_t)queue_pair_id);
923                 return -EINVAL;
924         }
925
926         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) ||
927             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
928              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)) {
929                 RTE_FUNC_PTR_OR_ERR_RET(
930                         *dev->dev_ops->crypto_adapter_queue_pair_del,
931                         -ENOTSUP);
932                 ret = (*dev->dev_ops->crypto_adapter_queue_pair_del)(dev,
933                                                 dev_info->dev,
934                                                 queue_pair_id);
935                 if (ret == 0) {
936                         eca_update_qp_info(adapter,
937                                         &adapter->cdevs[cdev_id],
938                                         queue_pair_id,
939                                         0);
940                         if (dev_info->num_qpairs == 0) {
941                                 rte_free(dev_info->qpairs);
942                                 dev_info->qpairs = NULL;
943                         }
944                 }
945         } else {
946                 if (adapter->nb_qps == 0)
947                         return 0;
948
949                 rte_spinlock_lock(&adapter->lock);
950                 if (queue_pair_id == -1) {
951                         for (i = 0; i < dev_info->dev->data->nb_queue_pairs;
952                                 i++)
953                                 eca_update_qp_info(adapter, dev_info,
954                                                         queue_pair_id, 0);
955                 } else {
956                         eca_update_qp_info(adapter, dev_info,
957                                                 (uint16_t)queue_pair_id, 0);
958                 }
959
960                 if (dev_info->num_qpairs == 0) {
961                         rte_free(dev_info->qpairs);
962                         dev_info->qpairs = NULL;
963                 }
964
965                 rte_spinlock_unlock(&adapter->lock);
966                 rte_service_component_runstate_set(adapter->service_id,
967                                 adapter->nb_qps);
968         }
969
970         rte_eventdev_trace_crypto_adapter_queue_pair_del(id, cdev_id,
971                 queue_pair_id, ret);
972         return ret;
973 }
974
975 static int
976 eca_adapter_ctrl(uint8_t id, int start)
977 {
978         struct rte_event_crypto_adapter *adapter;
979         struct crypto_device_info *dev_info;
980         struct rte_eventdev *dev;
981         uint32_t i;
982         int use_service;
983         int stop = !start;
984
985         use_service = 0;
986         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
987         adapter = eca_id_to_adapter(id);
988         if (adapter == NULL)
989                 return -EINVAL;
990
991         dev = &rte_eventdevs[adapter->eventdev_id];
992
993         for (i = 0; i < rte_cryptodev_count(); i++) {
994                 dev_info = &adapter->cdevs[i];
995                 /* if start  check for num queue pairs */
996                 if (start && !dev_info->num_qpairs)
997                         continue;
998                 /* if stop check if dev has been started */
999                 if (stop && !dev_info->dev_started)
1000                         continue;
1001                 use_service |= !dev_info->internal_event_port;
1002                 dev_info->dev_started = start;
1003                 if (dev_info->internal_event_port == 0)
1004                         continue;
1005                 start ? (*dev->dev_ops->crypto_adapter_start)(dev,
1006                                                 &dev_info->dev[i]) :
1007                         (*dev->dev_ops->crypto_adapter_stop)(dev,
1008                                                 &dev_info->dev[i]);
1009         }
1010
1011         if (use_service)
1012                 rte_service_runstate_set(adapter->service_id, start);
1013
1014         return 0;
1015 }
1016
1017 int
1018 rte_event_crypto_adapter_start(uint8_t id)
1019 {
1020         struct rte_event_crypto_adapter *adapter;
1021
1022         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1023         adapter = eca_id_to_adapter(id);
1024         if (adapter == NULL)
1025                 return -EINVAL;
1026
1027         rte_eventdev_trace_crypto_adapter_start(id, adapter);
1028         return eca_adapter_ctrl(id, 1);
1029 }
1030
1031 int
1032 rte_event_crypto_adapter_stop(uint8_t id)
1033 {
1034         rte_eventdev_trace_crypto_adapter_stop(id);
1035         return eca_adapter_ctrl(id, 0);
1036 }
1037
1038 int
1039 rte_event_crypto_adapter_stats_get(uint8_t id,
1040                                 struct rte_event_crypto_adapter_stats *stats)
1041 {
1042         struct rte_event_crypto_adapter *adapter;
1043         struct rte_event_crypto_adapter_stats dev_stats_sum = { 0 };
1044         struct rte_event_crypto_adapter_stats dev_stats;
1045         struct rte_eventdev *dev;
1046         struct crypto_device_info *dev_info;
1047         uint32_t i;
1048         int ret;
1049
1050         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1051
1052         adapter = eca_id_to_adapter(id);
1053         if (adapter == NULL || stats == NULL)
1054                 return -EINVAL;
1055
1056         dev = &rte_eventdevs[adapter->eventdev_id];
1057         memset(stats, 0, sizeof(*stats));
1058         for (i = 0; i < rte_cryptodev_count(); i++) {
1059                 dev_info = &adapter->cdevs[i];
1060                 if (dev_info->internal_event_port == 0 ||
1061                         dev->dev_ops->crypto_adapter_stats_get == NULL)
1062                         continue;
1063                 ret = (*dev->dev_ops->crypto_adapter_stats_get)(dev,
1064                                                 dev_info->dev,
1065                                                 &dev_stats);
1066                 if (ret)
1067                         continue;
1068
1069                 dev_stats_sum.crypto_deq_count += dev_stats.crypto_deq_count;
1070                 dev_stats_sum.event_enq_count +=
1071                         dev_stats.event_enq_count;
1072         }
1073
1074         if (adapter->service_inited)
1075                 *stats = adapter->crypto_stats;
1076
1077         stats->crypto_deq_count += dev_stats_sum.crypto_deq_count;
1078         stats->event_enq_count += dev_stats_sum.event_enq_count;
1079
1080         return 0;
1081 }
1082
1083 int
1084 rte_event_crypto_adapter_stats_reset(uint8_t id)
1085 {
1086         struct rte_event_crypto_adapter *adapter;
1087         struct crypto_device_info *dev_info;
1088         struct rte_eventdev *dev;
1089         uint32_t i;
1090
1091         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1092
1093         adapter = eca_id_to_adapter(id);
1094         if (adapter == NULL)
1095                 return -EINVAL;
1096
1097         dev = &rte_eventdevs[adapter->eventdev_id];
1098         for (i = 0; i < rte_cryptodev_count(); i++) {
1099                 dev_info = &adapter->cdevs[i];
1100                 if (dev_info->internal_event_port == 0 ||
1101                         dev->dev_ops->crypto_adapter_stats_reset == NULL)
1102                         continue;
1103                 (*dev->dev_ops->crypto_adapter_stats_reset)(dev,
1104                                                 dev_info->dev);
1105         }
1106
1107         memset(&adapter->crypto_stats, 0, sizeof(adapter->crypto_stats));
1108         return 0;
1109 }
1110
1111 int
1112 rte_event_crypto_adapter_service_id_get(uint8_t id, uint32_t *service_id)
1113 {
1114         struct rte_event_crypto_adapter *adapter;
1115
1116         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1117
1118         adapter = eca_id_to_adapter(id);
1119         if (adapter == NULL || service_id == NULL)
1120                 return -EINVAL;
1121
1122         if (adapter->service_inited)
1123                 *service_id = adapter->service_id;
1124
1125         return adapter->service_inited ? 0 : -ESRCH;
1126 }
1127
1128 int
1129 rte_event_crypto_adapter_event_port_get(uint8_t id, uint8_t *event_port_id)
1130 {
1131         struct rte_event_crypto_adapter *adapter;
1132
1133         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1134
1135         adapter = eca_id_to_adapter(id);
1136         if (adapter == NULL || event_port_id == NULL)
1137                 return -EINVAL;
1138
1139         *event_port_id = adapter->event_port_id;
1140
1141         return 0;
1142 }