095e218161f7bec3c8cb138915451a64081d75ed
[dpdk.git] / drivers / crypto / dpaa_sec / dpaa_sec.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  *
3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
4  *   Copyright 2017-2018 NXP
5  *
6  */
7
8 #include <fcntl.h>
9 #include <unistd.h>
10 #include <sched.h>
11 #include <net/if.h>
12
13 #include <rte_byteorder.h>
14 #include <rte_common.h>
15 #include <rte_cryptodev_pmd.h>
16 #include <rte_crypto.h>
17 #include <rte_cryptodev.h>
18 #include <rte_security_driver.h>
19 #include <rte_cycles.h>
20 #include <rte_dev.h>
21 #include <rte_kvargs.h>
22 #include <rte_malloc.h>
23 #include <rte_mbuf.h>
24 #include <rte_memcpy.h>
25 #include <rte_string_fns.h>
26 #include <rte_spinlock.h>
27
28 #include <fsl_usd.h>
29 #include <fsl_qman.h>
30 #include <of.h>
31
32 /* RTA header files */
33 #include <hw/desc/common.h>
34 #include <hw/desc/algo.h>
35 #include <hw/desc/ipsec.h>
36
37 #include <rte_dpaa_bus.h>
38 #include <dpaa_sec.h>
39 #include <dpaa_sec_log.h>
40
41 enum rta_sec_era rta_sec_era;
42
43 int dpaa_logtype_sec;
44
45 static uint8_t cryptodev_driver_id;
46
47 static __thread struct rte_crypto_op **dpaa_sec_ops;
48 static __thread int dpaa_sec_op_nb;
49
50 static int
51 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess);
52
53 static inline void
54 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx)
55 {
56         if (!ctx->fd_status) {
57                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
58         } else {
59                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
60                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
61         }
62
63         /* report op status to sym->op and then free the ctx memeory  */
64         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
65 }
66
67 static inline struct dpaa_sec_op_ctx *
68 dpaa_sec_alloc_ctx(dpaa_sec_session *ses)
69 {
70         struct dpaa_sec_op_ctx *ctx;
71         int retval;
72
73         retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
74         if (!ctx || retval) {
75                 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
76                 return NULL;
77         }
78         /*
79          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
80          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
81          * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
82          * each packet, memset is costlier than dcbz_64().
83          */
84         dcbz_64(&ctx->job.sg[SG_CACHELINE_0]);
85         dcbz_64(&ctx->job.sg[SG_CACHELINE_1]);
86         dcbz_64(&ctx->job.sg[SG_CACHELINE_2]);
87         dcbz_64(&ctx->job.sg[SG_CACHELINE_3]);
88
89         ctx->ctx_pool = ses->ctx_pool;
90         ctx->vtop_offset = (size_t) ctx
91                                 - rte_mempool_virt2iova(ctx);
92
93         return ctx;
94 }
95
96 static inline rte_iova_t
97 dpaa_mem_vtop(void *vaddr)
98 {
99         const struct rte_memseg *ms;
100
101         ms = rte_mem_virt2memseg(vaddr, NULL);
102         if (ms)
103                 return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
104         return (size_t)NULL;
105 }
106
107 static inline void *
108 dpaa_mem_ptov(rte_iova_t paddr)
109 {
110         return rte_mem_iova2virt(paddr);
111 }
112
113 static void
114 ern_sec_fq_handler(struct qman_portal *qm __rte_unused,
115                    struct qman_fq *fq,
116                    const struct qm_mr_entry *msg)
117 {
118         DPAA_SEC_DP_ERR("sec fq %d error, RC = %x, seqnum = %x\n",
119                         fq->fqid, msg->ern.rc, msg->ern.seqnum);
120 }
121
122 /* initialize the queue with dest chan as caam chan so that
123  * all the packets in this queue could be dispatched into caam
124  */
125 static int
126 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc,
127                  uint32_t fqid_out)
128 {
129         struct qm_mcc_initfq fq_opts;
130         uint32_t flags;
131         int ret = -1;
132
133         /* Clear FQ options */
134         memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq));
135
136         flags = QMAN_INITFQ_FLAG_SCHED;
137         fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA |
138                           QM_INITFQ_WE_CONTEXTB;
139
140         qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc);
141         fq_opts.fqd.context_b = fqid_out;
142         fq_opts.fqd.dest.channel = qm_channel_caam;
143         fq_opts.fqd.dest.wq = 0;
144
145         fq_in->cb.ern  = ern_sec_fq_handler;
146
147         DPAA_SEC_DEBUG("in-%x out-%x", fq_in->fqid, fqid_out);
148
149         ret = qman_init_fq(fq_in, flags, &fq_opts);
150         if (unlikely(ret != 0))
151                 DPAA_SEC_ERR("qman_init_fq failed %d", ret);
152
153         return ret;
154 }
155
156 /* something is put into in_fq and caam put the crypto result into out_fq */
157 static enum qman_cb_dqrr_result
158 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused,
159                   struct qman_fq *fq __always_unused,
160                   const struct qm_dqrr_entry *dqrr)
161 {
162         const struct qm_fd *fd;
163         struct dpaa_sec_job *job;
164         struct dpaa_sec_op_ctx *ctx;
165
166         if (dpaa_sec_op_nb >= DPAA_SEC_BURST)
167                 return qman_cb_dqrr_defer;
168
169         if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID))
170                 return qman_cb_dqrr_consume;
171
172         fd = &dqrr->fd;
173         /* sg is embedded in an op ctx,
174          * sg[0] is for output
175          * sg[1] for input
176          */
177         job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
178
179         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
180         ctx->fd_status = fd->status;
181         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
182                 struct qm_sg_entry *sg_out;
183                 uint32_t len;
184
185                 sg_out = &job->sg[0];
186                 hw_sg_to_cpu(sg_out);
187                 len = sg_out->length;
188                 ctx->op->sym->m_src->pkt_len = len;
189                 ctx->op->sym->m_src->data_len = len;
190         }
191         dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op;
192         dpaa_sec_op_ending(ctx);
193
194         return qman_cb_dqrr_consume;
195 }
196
197 /* caam result is put into this queue */
198 static int
199 dpaa_sec_init_tx(struct qman_fq *fq)
200 {
201         int ret;
202         struct qm_mcc_initfq opts;
203         uint32_t flags;
204
205         flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED |
206                 QMAN_FQ_FLAG_DYNAMIC_FQID;
207
208         ret = qman_create_fq(0, flags, fq);
209         if (unlikely(ret)) {
210                 DPAA_SEC_ERR("qman_create_fq failed");
211                 return ret;
212         }
213
214         memset(&opts, 0, sizeof(opts));
215         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
216                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
217
218         /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */
219
220         fq->cb.dqrr = dqrr_out_fq_cb_rx;
221         fq->cb.ern  = ern_sec_fq_handler;
222
223         ret = qman_init_fq(fq, 0, &opts);
224         if (unlikely(ret)) {
225                 DPAA_SEC_ERR("unable to init caam source fq!");
226                 return ret;
227         }
228
229         return ret;
230 }
231
232 static inline int is_cipher_only(dpaa_sec_session *ses)
233 {
234         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
235                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
236 }
237
238 static inline int is_auth_only(dpaa_sec_session *ses)
239 {
240         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
241                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
242 }
243
244 static inline int is_aead(dpaa_sec_session *ses)
245 {
246         return ((ses->cipher_alg == 0) &&
247                 (ses->auth_alg == 0) &&
248                 (ses->aead_alg != 0));
249 }
250
251 static inline int is_auth_cipher(dpaa_sec_session *ses)
252 {
253         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
254                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
255                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
256 }
257
258 static inline int is_proto_ipsec(dpaa_sec_session *ses)
259 {
260         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
261 }
262
263 static inline int is_encode(dpaa_sec_session *ses)
264 {
265         return ses->dir == DIR_ENC;
266 }
267
268 static inline int is_decode(dpaa_sec_session *ses)
269 {
270         return ses->dir == DIR_DEC;
271 }
272
273 static inline void
274 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a)
275 {
276         switch (ses->auth_alg) {
277         case RTE_CRYPTO_AUTH_NULL:
278                 ses->digest_length = 0;
279                 break;
280         case RTE_CRYPTO_AUTH_MD5_HMAC:
281                 alginfo_a->algtype =
282                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
283                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
284                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
285                 break;
286         case RTE_CRYPTO_AUTH_SHA1_HMAC:
287                 alginfo_a->algtype =
288                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
289                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
290                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
291                 break;
292         case RTE_CRYPTO_AUTH_SHA224_HMAC:
293                 alginfo_a->algtype =
294                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
295                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
296                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
297                 break;
298         case RTE_CRYPTO_AUTH_SHA256_HMAC:
299                 alginfo_a->algtype =
300                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
301                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
302                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
303                 break;
304         case RTE_CRYPTO_AUTH_SHA384_HMAC:
305                 alginfo_a->algtype =
306                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
307                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
308                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
309                 break;
310         case RTE_CRYPTO_AUTH_SHA512_HMAC:
311                 alginfo_a->algtype =
312                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
313                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
314                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
315                 break;
316         default:
317                 DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg);
318         }
319 }
320
321 static inline void
322 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c)
323 {
324         switch (ses->cipher_alg) {
325         case RTE_CRYPTO_CIPHER_NULL:
326                 break;
327         case RTE_CRYPTO_CIPHER_AES_CBC:
328                 alginfo_c->algtype =
329                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
330                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
331                 alginfo_c->algmode = OP_ALG_AAI_CBC;
332                 break;
333         case RTE_CRYPTO_CIPHER_3DES_CBC:
334                 alginfo_c->algtype =
335                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
336                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
337                 alginfo_c->algmode = OP_ALG_AAI_CBC;
338                 break;
339         case RTE_CRYPTO_CIPHER_AES_CTR:
340                 alginfo_c->algtype =
341                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
342                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
343                 alginfo_c->algmode = OP_ALG_AAI_CTR;
344                 break;
345         default:
346                 DPAA_SEC_ERR("unsupported cipher alg %d", ses->cipher_alg);
347         }
348 }
349
350 static inline void
351 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo)
352 {
353         switch (ses->aead_alg) {
354         case RTE_CRYPTO_AEAD_AES_GCM:
355                 alginfo->algtype = OP_ALG_ALGSEL_AES;
356                 alginfo->algmode = OP_ALG_AAI_GCM;
357                 break;
358         default:
359                 DPAA_SEC_ERR("unsupported AEAD alg %d", ses->aead_alg);
360         }
361 }
362
363
364 /* prepare command block of the session */
365 static int
366 dpaa_sec_prep_cdb(dpaa_sec_session *ses)
367 {
368         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
369         int32_t shared_desc_len = 0;
370         struct sec_cdb *cdb = &ses->cdb;
371         int err;
372 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
373         int swap = false;
374 #else
375         int swap = true;
376 #endif
377
378         memset(cdb, 0, sizeof(struct sec_cdb));
379
380         if (is_cipher_only(ses)) {
381                 caam_cipher_alg(ses, &alginfo_c);
382                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
383                         DPAA_SEC_ERR("not supported cipher alg");
384                         return -ENOTSUP;
385                 }
386
387                 alginfo_c.key = (size_t)ses->cipher_key.data;
388                 alginfo_c.keylen = ses->cipher_key.length;
389                 alginfo_c.key_enc_flags = 0;
390                 alginfo_c.key_type = RTA_DATA_IMM;
391
392                 shared_desc_len = cnstr_shdsc_blkcipher(
393                                                 cdb->sh_desc, true,
394                                                 swap, &alginfo_c,
395                                                 NULL,
396                                                 ses->iv.length,
397                                                 ses->dir);
398         } else if (is_auth_only(ses)) {
399                 caam_auth_alg(ses, &alginfo_a);
400                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
401                         DPAA_SEC_ERR("not supported auth alg");
402                         return -ENOTSUP;
403                 }
404
405                 alginfo_a.key = (size_t)ses->auth_key.data;
406                 alginfo_a.keylen = ses->auth_key.length;
407                 alginfo_a.key_enc_flags = 0;
408                 alginfo_a.key_type = RTA_DATA_IMM;
409
410                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
411                                                    swap, &alginfo_a,
412                                                    !ses->dir,
413                                                    ses->digest_length);
414         } else if (is_aead(ses)) {
415                 caam_aead_alg(ses, &alginfo);
416                 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
417                         DPAA_SEC_ERR("not supported aead alg");
418                         return -ENOTSUP;
419                 }
420                 alginfo.key = (size_t)ses->aead_key.data;
421                 alginfo.keylen = ses->aead_key.length;
422                 alginfo.key_enc_flags = 0;
423                 alginfo.key_type = RTA_DATA_IMM;
424
425                 if (ses->dir == DIR_ENC)
426                         shared_desc_len = cnstr_shdsc_gcm_encap(
427                                         cdb->sh_desc, true, swap,
428                                         &alginfo,
429                                         ses->iv.length,
430                                         ses->digest_length);
431                 else
432                         shared_desc_len = cnstr_shdsc_gcm_decap(
433                                         cdb->sh_desc, true, swap,
434                                         &alginfo,
435                                         ses->iv.length,
436                                         ses->digest_length);
437         } else {
438                 caam_cipher_alg(ses, &alginfo_c);
439                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
440                         DPAA_SEC_ERR("not supported cipher alg");
441                         return -ENOTSUP;
442                 }
443
444                 alginfo_c.key = (size_t)ses->cipher_key.data;
445                 alginfo_c.keylen = ses->cipher_key.length;
446                 alginfo_c.key_enc_flags = 0;
447                 alginfo_c.key_type = RTA_DATA_IMM;
448
449                 caam_auth_alg(ses, &alginfo_a);
450                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
451                         DPAA_SEC_ERR("not supported auth alg");
452                         return -ENOTSUP;
453                 }
454
455                 alginfo_a.key = (size_t)ses->auth_key.data;
456                 alginfo_a.keylen = ses->auth_key.length;
457                 alginfo_a.key_enc_flags = 0;
458                 alginfo_a.key_type = RTA_DATA_IMM;
459
460                 cdb->sh_desc[0] = alginfo_c.keylen;
461                 cdb->sh_desc[1] = alginfo_a.keylen;
462                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
463                                        MIN_JOB_DESC_SIZE,
464                                        (unsigned int *)cdb->sh_desc,
465                                        &cdb->sh_desc[2], 2);
466
467                 if (err < 0) {
468                         DPAA_SEC_ERR("Crypto: Incorrect key lengths");
469                         return err;
470                 }
471                 if (cdb->sh_desc[2] & 1)
472                         alginfo_c.key_type = RTA_DATA_IMM;
473                 else {
474                         alginfo_c.key = (size_t)dpaa_mem_vtop(
475                                                 (void *)(size_t)alginfo_c.key);
476                         alginfo_c.key_type = RTA_DATA_PTR;
477                 }
478                 if (cdb->sh_desc[2] & (1<<1))
479                         alginfo_a.key_type = RTA_DATA_IMM;
480                 else {
481                         alginfo_a.key = (size_t)dpaa_mem_vtop(
482                                                 (void *)(size_t)alginfo_a.key);
483                         alginfo_a.key_type = RTA_DATA_PTR;
484                 }
485                 cdb->sh_desc[0] = 0;
486                 cdb->sh_desc[1] = 0;
487                 cdb->sh_desc[2] = 0;
488                 if (is_proto_ipsec(ses)) {
489                         if (ses->dir == DIR_ENC) {
490                                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
491                                                 cdb->sh_desc,
492                                                 true, swap, SHR_SERIAL,
493                                                 &ses->encap_pdb,
494                                                 (uint8_t *)&ses->ip4_hdr,
495                                                 &alginfo_c, &alginfo_a);
496                         } else if (ses->dir == DIR_DEC) {
497                                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
498                                                 cdb->sh_desc,
499                                                 true, swap, SHR_SERIAL,
500                                                 &ses->decap_pdb,
501                                                 &alginfo_c, &alginfo_a);
502                         }
503                 } else {
504                         /* Auth_only_len is set as 0 here and it will be
505                          * overwritten in fd for each packet.
506                          */
507                         shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
508                                         true, swap, &alginfo_c, &alginfo_a,
509                                         ses->iv.length, 0,
510                                         ses->digest_length, ses->dir);
511                 }
512         }
513
514         if (shared_desc_len < 0) {
515                 DPAA_SEC_ERR("error in preparing command block");
516                 return shared_desc_len;
517         }
518
519         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
520         cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word);
521         cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word);
522
523         return 0;
524 }
525
526 /* qp is lockless, should be accessed by only one thread */
527 static int
528 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
529 {
530         struct qman_fq *fq;
531         unsigned int pkts = 0;
532         int num_rx_bufs, ret;
533         struct qm_dqrr_entry *dq;
534         uint32_t vdqcr_flags = 0;
535
536         fq = &qp->outq;
537         /*
538          * Until request for four buffers, we provide exact number of buffers.
539          * Otherwise we do not set the QM_VDQCR_EXACT flag.
540          * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
541          * requested, so we request two less in this case.
542          */
543         if (nb_ops < 4) {
544                 vdqcr_flags = QM_VDQCR_EXACT;
545                 num_rx_bufs = nb_ops;
546         } else {
547                 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
548                         (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2);
549         }
550         ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
551         if (ret)
552                 return 0;
553
554         do {
555                 const struct qm_fd *fd;
556                 struct dpaa_sec_job *job;
557                 struct dpaa_sec_op_ctx *ctx;
558                 struct rte_crypto_op *op;
559
560                 dq = qman_dequeue(fq);
561                 if (!dq)
562                         continue;
563
564                 fd = &dq->fd;
565                 /* sg is embedded in an op ctx,
566                  * sg[0] is for output
567                  * sg[1] for input
568                  */
569                 job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
570
571                 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
572                 ctx->fd_status = fd->status;
573                 op = ctx->op;
574                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
575                         struct qm_sg_entry *sg_out;
576                         uint32_t len;
577
578                         sg_out = &job->sg[0];
579                         hw_sg_to_cpu(sg_out);
580                         len = sg_out->length;
581                         op->sym->m_src->pkt_len = len;
582                         op->sym->m_src->data_len = len;
583                 }
584                 if (!ctx->fd_status) {
585                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
586                 } else {
587                         DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status);
588                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
589                 }
590                 ops[pkts++] = op;
591
592                 /* report op status to sym->op and then free the ctx memeory */
593                 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
594
595                 qman_dqrr_consume(fq, dq);
596         } while (fq->flags & QMAN_FQ_STATE_VDQCR);
597
598         return pkts;
599 }
600
601 static inline struct dpaa_sec_job *
602 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
603 {
604         struct rte_crypto_sym_op *sym = op->sym;
605         struct rte_mbuf *mbuf = sym->m_src;
606         struct dpaa_sec_job *cf;
607         struct dpaa_sec_op_ctx *ctx;
608         struct qm_sg_entry *sg, *out_sg, *in_sg;
609         phys_addr_t start_addr;
610         uint8_t *old_digest, extra_segs;
611
612         if (is_decode(ses))
613                 extra_segs = 3;
614         else
615                 extra_segs = 2;
616
617         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
618                 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
619                                 MAX_SG_ENTRIES);
620                 return NULL;
621         }
622         ctx = dpaa_sec_alloc_ctx(ses);
623         if (!ctx)
624                 return NULL;
625
626         cf = &ctx->job;
627         ctx->op = op;
628         old_digest = ctx->digest;
629
630         /* output */
631         out_sg = &cf->sg[0];
632         qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr);
633         out_sg->length = ses->digest_length;
634         cpu_to_hw_sg(out_sg);
635
636         /* input */
637         in_sg = &cf->sg[1];
638         /* need to extend the input to a compound frame */
639         in_sg->extension = 1;
640         in_sg->final = 1;
641         in_sg->length = sym->auth.data.length;
642         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2]));
643
644         /* 1st seg */
645         sg = in_sg + 1;
646         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
647         sg->length = mbuf->data_len - sym->auth.data.offset;
648         sg->offset = sym->auth.data.offset;
649
650         /* Successive segs */
651         mbuf = mbuf->next;
652         while (mbuf) {
653                 cpu_to_hw_sg(sg);
654                 sg++;
655                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
656                 sg->length = mbuf->data_len;
657                 mbuf = mbuf->next;
658         }
659
660         if (is_decode(ses)) {
661                 /* Digest verification case */
662                 cpu_to_hw_sg(sg);
663                 sg++;
664                 rte_memcpy(old_digest, sym->auth.digest.data,
665                                 ses->digest_length);
666                 start_addr = dpaa_mem_vtop(old_digest);
667                 qm_sg_entry_set64(sg, start_addr);
668                 sg->length = ses->digest_length;
669                 in_sg->length += ses->digest_length;
670         } else {
671                 /* Digest calculation case */
672                 sg->length -= ses->digest_length;
673         }
674         sg->final = 1;
675         cpu_to_hw_sg(sg);
676         cpu_to_hw_sg(in_sg);
677
678         return cf;
679 }
680
681 /**
682  * packet looks like:
683  *              |<----data_len------->|
684  *    |ip_header|ah_header|icv|payload|
685  *              ^
686  *              |
687  *         mbuf->pkt.data
688  */
689 static inline struct dpaa_sec_job *
690 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
691 {
692         struct rte_crypto_sym_op *sym = op->sym;
693         struct rte_mbuf *mbuf = sym->m_src;
694         struct dpaa_sec_job *cf;
695         struct dpaa_sec_op_ctx *ctx;
696         struct qm_sg_entry *sg;
697         rte_iova_t start_addr;
698         uint8_t *old_digest;
699
700         ctx = dpaa_sec_alloc_ctx(ses);
701         if (!ctx)
702                 return NULL;
703
704         cf = &ctx->job;
705         ctx->op = op;
706         old_digest = ctx->digest;
707
708         start_addr = rte_pktmbuf_iova(mbuf);
709         /* output */
710         sg = &cf->sg[0];
711         qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
712         sg->length = ses->digest_length;
713         cpu_to_hw_sg(sg);
714
715         /* input */
716         sg = &cf->sg[1];
717         if (is_decode(ses)) {
718                 /* need to extend the input to a compound frame */
719                 sg->extension = 1;
720                 qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
721                 sg->length = sym->auth.data.length + ses->digest_length;
722                 sg->final = 1;
723                 cpu_to_hw_sg(sg);
724
725                 sg = &cf->sg[2];
726                 /* hash result or digest, save digest first */
727                 rte_memcpy(old_digest, sym->auth.digest.data,
728                            ses->digest_length);
729                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
730                 sg->length = sym->auth.data.length;
731                 cpu_to_hw_sg(sg);
732
733                 /* let's check digest by hw */
734                 start_addr = dpaa_mem_vtop(old_digest);
735                 sg++;
736                 qm_sg_entry_set64(sg, start_addr);
737                 sg->length = ses->digest_length;
738                 sg->final = 1;
739                 cpu_to_hw_sg(sg);
740         } else {
741                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
742                 sg->length = sym->auth.data.length;
743                 sg->final = 1;
744                 cpu_to_hw_sg(sg);
745         }
746
747         return cf;
748 }
749
750 static inline struct dpaa_sec_job *
751 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
752 {
753         struct rte_crypto_sym_op *sym = op->sym;
754         struct dpaa_sec_job *cf;
755         struct dpaa_sec_op_ctx *ctx;
756         struct qm_sg_entry *sg, *out_sg, *in_sg;
757         struct rte_mbuf *mbuf;
758         uint8_t req_segs;
759         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
760                         ses->iv.offset);
761
762         if (sym->m_dst) {
763                 mbuf = sym->m_dst;
764                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
765         } else {
766                 mbuf = sym->m_src;
767                 req_segs = mbuf->nb_segs * 2 + 3;
768         }
769
770         if (req_segs > MAX_SG_ENTRIES) {
771                 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
772                                 MAX_SG_ENTRIES);
773                 return NULL;
774         }
775
776         ctx = dpaa_sec_alloc_ctx(ses);
777         if (!ctx)
778                 return NULL;
779
780         cf = &ctx->job;
781         ctx->op = op;
782
783         /* output */
784         out_sg = &cf->sg[0];
785         out_sg->extension = 1;
786         out_sg->length = sym->cipher.data.length;
787         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2]));
788         cpu_to_hw_sg(out_sg);
789
790         /* 1st seg */
791         sg = &cf->sg[2];
792         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
793         sg->length = mbuf->data_len - sym->cipher.data.offset;
794         sg->offset = sym->cipher.data.offset;
795
796         /* Successive segs */
797         mbuf = mbuf->next;
798         while (mbuf) {
799                 cpu_to_hw_sg(sg);
800                 sg++;
801                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
802                 sg->length = mbuf->data_len;
803                 mbuf = mbuf->next;
804         }
805         sg->final = 1;
806         cpu_to_hw_sg(sg);
807
808         /* input */
809         mbuf = sym->m_src;
810         in_sg = &cf->sg[1];
811         in_sg->extension = 1;
812         in_sg->final = 1;
813         in_sg->length = sym->cipher.data.length + ses->iv.length;
814
815         sg++;
816         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
817         cpu_to_hw_sg(in_sg);
818
819         /* IV */
820         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
821         sg->length = ses->iv.length;
822         cpu_to_hw_sg(sg);
823
824         /* 1st seg */
825         sg++;
826         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
827         sg->length = mbuf->data_len - sym->cipher.data.offset;
828         sg->offset = sym->cipher.data.offset;
829
830         /* Successive segs */
831         mbuf = mbuf->next;
832         while (mbuf) {
833                 cpu_to_hw_sg(sg);
834                 sg++;
835                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
836                 sg->length = mbuf->data_len;
837                 mbuf = mbuf->next;
838         }
839         sg->final = 1;
840         cpu_to_hw_sg(sg);
841
842         return cf;
843 }
844
845 static inline struct dpaa_sec_job *
846 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
847 {
848         struct rte_crypto_sym_op *sym = op->sym;
849         struct dpaa_sec_job *cf;
850         struct dpaa_sec_op_ctx *ctx;
851         struct qm_sg_entry *sg;
852         rte_iova_t src_start_addr, dst_start_addr;
853         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
854                         ses->iv.offset);
855
856         ctx = dpaa_sec_alloc_ctx(ses);
857         if (!ctx)
858                 return NULL;
859
860         cf = &ctx->job;
861         ctx->op = op;
862
863         src_start_addr = rte_pktmbuf_iova(sym->m_src);
864
865         if (sym->m_dst)
866                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
867         else
868                 dst_start_addr = src_start_addr;
869
870         /* output */
871         sg = &cf->sg[0];
872         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
873         sg->length = sym->cipher.data.length + ses->iv.length;
874         cpu_to_hw_sg(sg);
875
876         /* input */
877         sg = &cf->sg[1];
878
879         /* need to extend the input to a compound frame */
880         sg->extension = 1;
881         sg->final = 1;
882         sg->length = sym->cipher.data.length + ses->iv.length;
883         qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
884         cpu_to_hw_sg(sg);
885
886         sg = &cf->sg[2];
887         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
888         sg->length = ses->iv.length;
889         cpu_to_hw_sg(sg);
890
891         sg++;
892         qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset);
893         sg->length = sym->cipher.data.length;
894         sg->final = 1;
895         cpu_to_hw_sg(sg);
896
897         return cf;
898 }
899
900 static inline struct dpaa_sec_job *
901 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
902 {
903         struct rte_crypto_sym_op *sym = op->sym;
904         struct dpaa_sec_job *cf;
905         struct dpaa_sec_op_ctx *ctx;
906         struct qm_sg_entry *sg, *out_sg, *in_sg;
907         struct rte_mbuf *mbuf;
908         uint8_t req_segs;
909         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
910                         ses->iv.offset);
911
912         if (sym->m_dst) {
913                 mbuf = sym->m_dst;
914                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
915         } else {
916                 mbuf = sym->m_src;
917                 req_segs = mbuf->nb_segs * 2 + 4;
918         }
919
920         if (ses->auth_only_len)
921                 req_segs++;
922
923         if (req_segs > MAX_SG_ENTRIES) {
924                 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
925                                 MAX_SG_ENTRIES);
926                 return NULL;
927         }
928
929         ctx = dpaa_sec_alloc_ctx(ses);
930         if (!ctx)
931                 return NULL;
932
933         cf = &ctx->job;
934         ctx->op = op;
935
936         rte_prefetch0(cf->sg);
937
938         /* output */
939         out_sg = &cf->sg[0];
940         out_sg->extension = 1;
941         if (is_encode(ses))
942                 out_sg->length = sym->aead.data.length + ses->auth_only_len
943                                                 + ses->digest_length;
944         else
945                 out_sg->length = sym->aead.data.length + ses->auth_only_len;
946
947         /* output sg entries */
948         sg = &cf->sg[2];
949         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
950         cpu_to_hw_sg(out_sg);
951
952         /* 1st seg */
953         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
954         sg->length = mbuf->data_len - sym->aead.data.offset +
955                                         ses->auth_only_len;
956         sg->offset = sym->aead.data.offset - ses->auth_only_len;
957
958         /* Successive segs */
959         mbuf = mbuf->next;
960         while (mbuf) {
961                 cpu_to_hw_sg(sg);
962                 sg++;
963                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
964                 sg->length = mbuf->data_len;
965                 mbuf = mbuf->next;
966         }
967         sg->length -= ses->digest_length;
968
969         if (is_encode(ses)) {
970                 cpu_to_hw_sg(sg);
971                 /* set auth output */
972                 sg++;
973                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
974                 sg->length = ses->digest_length;
975         }
976         sg->final = 1;
977         cpu_to_hw_sg(sg);
978
979         /* input */
980         mbuf = sym->m_src;
981         in_sg = &cf->sg[1];
982         in_sg->extension = 1;
983         in_sg->final = 1;
984         if (is_encode(ses))
985                 in_sg->length = ses->iv.length + sym->aead.data.length
986                                                         + ses->auth_only_len;
987         else
988                 in_sg->length = ses->iv.length + sym->aead.data.length
989                                 + ses->auth_only_len + ses->digest_length;
990
991         /* input sg entries */
992         sg++;
993         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
994         cpu_to_hw_sg(in_sg);
995
996         /* 1st seg IV */
997         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
998         sg->length = ses->iv.length;
999         cpu_to_hw_sg(sg);
1000
1001         /* 2nd seg auth only */
1002         if (ses->auth_only_len) {
1003                 sg++;
1004                 qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data));
1005                 sg->length = ses->auth_only_len;
1006                 cpu_to_hw_sg(sg);
1007         }
1008
1009         /* 3rd seg */
1010         sg++;
1011         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1012         sg->length = mbuf->data_len - sym->aead.data.offset;
1013         sg->offset = sym->aead.data.offset;
1014
1015         /* Successive segs */
1016         mbuf = mbuf->next;
1017         while (mbuf) {
1018                 cpu_to_hw_sg(sg);
1019                 sg++;
1020                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1021                 sg->length = mbuf->data_len;
1022                 mbuf = mbuf->next;
1023         }
1024
1025         if (is_decode(ses)) {
1026                 cpu_to_hw_sg(sg);
1027                 sg++;
1028                 memcpy(ctx->digest, sym->aead.digest.data,
1029                         ses->digest_length);
1030                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1031                 sg->length = ses->digest_length;
1032         }
1033         sg->final = 1;
1034         cpu_to_hw_sg(sg);
1035
1036         return cf;
1037 }
1038
1039 static inline struct dpaa_sec_job *
1040 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
1041 {
1042         struct rte_crypto_sym_op *sym = op->sym;
1043         struct dpaa_sec_job *cf;
1044         struct dpaa_sec_op_ctx *ctx;
1045         struct qm_sg_entry *sg;
1046         uint32_t length = 0;
1047         rte_iova_t src_start_addr, dst_start_addr;
1048         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1049                         ses->iv.offset);
1050
1051         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1052
1053         if (sym->m_dst)
1054                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1055         else
1056                 dst_start_addr = src_start_addr;
1057
1058         ctx = dpaa_sec_alloc_ctx(ses);
1059         if (!ctx)
1060                 return NULL;
1061
1062         cf = &ctx->job;
1063         ctx->op = op;
1064
1065         /* input */
1066         rte_prefetch0(cf->sg);
1067         sg = &cf->sg[2];
1068         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
1069         if (is_encode(ses)) {
1070                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1071                 sg->length = ses->iv.length;
1072                 length += sg->length;
1073                 cpu_to_hw_sg(sg);
1074
1075                 sg++;
1076                 if (ses->auth_only_len) {
1077                         qm_sg_entry_set64(sg,
1078                                           dpaa_mem_vtop(sym->aead.aad.data));
1079                         sg->length = ses->auth_only_len;
1080                         length += sg->length;
1081                         cpu_to_hw_sg(sg);
1082                         sg++;
1083                 }
1084                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1085                 sg->length = sym->aead.data.length;
1086                 length += sg->length;
1087                 sg->final = 1;
1088                 cpu_to_hw_sg(sg);
1089         } else {
1090                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1091                 sg->length = ses->iv.length;
1092                 length += sg->length;
1093                 cpu_to_hw_sg(sg);
1094
1095                 sg++;
1096                 if (ses->auth_only_len) {
1097                         qm_sg_entry_set64(sg,
1098                                           dpaa_mem_vtop(sym->aead.aad.data));
1099                         sg->length = ses->auth_only_len;
1100                         length += sg->length;
1101                         cpu_to_hw_sg(sg);
1102                         sg++;
1103                 }
1104                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1105                 sg->length = sym->aead.data.length;
1106                 length += sg->length;
1107                 cpu_to_hw_sg(sg);
1108
1109                 memcpy(ctx->digest, sym->aead.digest.data,
1110                        ses->digest_length);
1111                 sg++;
1112
1113                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1114                 sg->length = ses->digest_length;
1115                 length += sg->length;
1116                 sg->final = 1;
1117                 cpu_to_hw_sg(sg);
1118         }
1119         /* input compound frame */
1120         cf->sg[1].length = length;
1121         cf->sg[1].extension = 1;
1122         cf->sg[1].final = 1;
1123         cpu_to_hw_sg(&cf->sg[1]);
1124
1125         /* output */
1126         sg++;
1127         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
1128         qm_sg_entry_set64(sg,
1129                 dst_start_addr + sym->aead.data.offset - ses->auth_only_len);
1130         sg->length = sym->aead.data.length + ses->auth_only_len;
1131         length = sg->length;
1132         if (is_encode(ses)) {
1133                 cpu_to_hw_sg(sg);
1134                 /* set auth output */
1135                 sg++;
1136                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1137                 sg->length = ses->digest_length;
1138                 length += sg->length;
1139         }
1140         sg->final = 1;
1141         cpu_to_hw_sg(sg);
1142
1143         /* output compound frame */
1144         cf->sg[0].length = length;
1145         cf->sg[0].extension = 1;
1146         cpu_to_hw_sg(&cf->sg[0]);
1147
1148         return cf;
1149 }
1150
1151 static inline struct dpaa_sec_job *
1152 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1153 {
1154         struct rte_crypto_sym_op *sym = op->sym;
1155         struct dpaa_sec_job *cf;
1156         struct dpaa_sec_op_ctx *ctx;
1157         struct qm_sg_entry *sg, *out_sg, *in_sg;
1158         struct rte_mbuf *mbuf;
1159         uint8_t req_segs;
1160         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1161                         ses->iv.offset);
1162
1163         if (sym->m_dst) {
1164                 mbuf = sym->m_dst;
1165                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1166         } else {
1167                 mbuf = sym->m_src;
1168                 req_segs = mbuf->nb_segs * 2 + 4;
1169         }
1170
1171         if (req_segs > MAX_SG_ENTRIES) {
1172                 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1173                                 MAX_SG_ENTRIES);
1174                 return NULL;
1175         }
1176
1177         ctx = dpaa_sec_alloc_ctx(ses);
1178         if (!ctx)
1179                 return NULL;
1180
1181         cf = &ctx->job;
1182         ctx->op = op;
1183
1184         rte_prefetch0(cf->sg);
1185
1186         /* output */
1187         out_sg = &cf->sg[0];
1188         out_sg->extension = 1;
1189         if (is_encode(ses))
1190                 out_sg->length = sym->auth.data.length + ses->digest_length;
1191         else
1192                 out_sg->length = sym->auth.data.length;
1193
1194         /* output sg entries */
1195         sg = &cf->sg[2];
1196         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
1197         cpu_to_hw_sg(out_sg);
1198
1199         /* 1st seg */
1200         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1201         sg->length = mbuf->data_len - sym->auth.data.offset;
1202         sg->offset = sym->auth.data.offset;
1203
1204         /* Successive segs */
1205         mbuf = mbuf->next;
1206         while (mbuf) {
1207                 cpu_to_hw_sg(sg);
1208                 sg++;
1209                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1210                 sg->length = mbuf->data_len;
1211                 mbuf = mbuf->next;
1212         }
1213         sg->length -= ses->digest_length;
1214
1215         if (is_encode(ses)) {
1216                 cpu_to_hw_sg(sg);
1217                 /* set auth output */
1218                 sg++;
1219                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1220                 sg->length = ses->digest_length;
1221         }
1222         sg->final = 1;
1223         cpu_to_hw_sg(sg);
1224
1225         /* input */
1226         mbuf = sym->m_src;
1227         in_sg = &cf->sg[1];
1228         in_sg->extension = 1;
1229         in_sg->final = 1;
1230         if (is_encode(ses))
1231                 in_sg->length = ses->iv.length + sym->auth.data.length;
1232         else
1233                 in_sg->length = ses->iv.length + sym->auth.data.length
1234                                                 + ses->digest_length;
1235
1236         /* input sg entries */
1237         sg++;
1238         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
1239         cpu_to_hw_sg(in_sg);
1240
1241         /* 1st seg IV */
1242         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1243         sg->length = ses->iv.length;
1244         cpu_to_hw_sg(sg);
1245
1246         /* 2nd seg */
1247         sg++;
1248         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1249         sg->length = mbuf->data_len - sym->auth.data.offset;
1250         sg->offset = sym->auth.data.offset;
1251
1252         /* Successive segs */
1253         mbuf = mbuf->next;
1254         while (mbuf) {
1255                 cpu_to_hw_sg(sg);
1256                 sg++;
1257                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1258                 sg->length = mbuf->data_len;
1259                 mbuf = mbuf->next;
1260         }
1261
1262         sg->length -= ses->digest_length;
1263         if (is_decode(ses)) {
1264                 cpu_to_hw_sg(sg);
1265                 sg++;
1266                 memcpy(ctx->digest, sym->auth.digest.data,
1267                         ses->digest_length);
1268                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1269                 sg->length = ses->digest_length;
1270         }
1271         sg->final = 1;
1272         cpu_to_hw_sg(sg);
1273
1274         return cf;
1275 }
1276
1277 static inline struct dpaa_sec_job *
1278 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
1279 {
1280         struct rte_crypto_sym_op *sym = op->sym;
1281         struct dpaa_sec_job *cf;
1282         struct dpaa_sec_op_ctx *ctx;
1283         struct qm_sg_entry *sg;
1284         rte_iova_t src_start_addr, dst_start_addr;
1285         uint32_t length = 0;
1286         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1287                         ses->iv.offset);
1288
1289         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1290         if (sym->m_dst)
1291                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1292         else
1293                 dst_start_addr = src_start_addr;
1294
1295         ctx = dpaa_sec_alloc_ctx(ses);
1296         if (!ctx)
1297                 return NULL;
1298
1299         cf = &ctx->job;
1300         ctx->op = op;
1301
1302         /* input */
1303         rte_prefetch0(cf->sg);
1304         sg = &cf->sg[2];
1305         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
1306         if (is_encode(ses)) {
1307                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1308                 sg->length = ses->iv.length;
1309                 length += sg->length;
1310                 cpu_to_hw_sg(sg);
1311
1312                 sg++;
1313                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1314                 sg->length = sym->auth.data.length;
1315                 length += sg->length;
1316                 sg->final = 1;
1317                 cpu_to_hw_sg(sg);
1318         } else {
1319                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1320                 sg->length = ses->iv.length;
1321                 length += sg->length;
1322                 cpu_to_hw_sg(sg);
1323
1324                 sg++;
1325
1326                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1327                 sg->length = sym->auth.data.length;
1328                 length += sg->length;
1329                 cpu_to_hw_sg(sg);
1330
1331                 memcpy(ctx->digest, sym->auth.digest.data,
1332                        ses->digest_length);
1333                 sg++;
1334
1335                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1336                 sg->length = ses->digest_length;
1337                 length += sg->length;
1338                 sg->final = 1;
1339                 cpu_to_hw_sg(sg);
1340         }
1341         /* input compound frame */
1342         cf->sg[1].length = length;
1343         cf->sg[1].extension = 1;
1344         cf->sg[1].final = 1;
1345         cpu_to_hw_sg(&cf->sg[1]);
1346
1347         /* output */
1348         sg++;
1349         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
1350         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
1351         sg->length = sym->cipher.data.length;
1352         length = sg->length;
1353         if (is_encode(ses)) {
1354                 cpu_to_hw_sg(sg);
1355                 /* set auth output */
1356                 sg++;
1357                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1358                 sg->length = ses->digest_length;
1359                 length += sg->length;
1360         }
1361         sg->final = 1;
1362         cpu_to_hw_sg(sg);
1363
1364         /* output compound frame */
1365         cf->sg[0].length = length;
1366         cf->sg[0].extension = 1;
1367         cpu_to_hw_sg(&cf->sg[0]);
1368
1369         return cf;
1370 }
1371
1372 static inline struct dpaa_sec_job *
1373 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses)
1374 {
1375         struct rte_crypto_sym_op *sym = op->sym;
1376         struct dpaa_sec_job *cf;
1377         struct dpaa_sec_op_ctx *ctx;
1378         struct qm_sg_entry *sg;
1379         phys_addr_t src_start_addr, dst_start_addr;
1380
1381         ctx = dpaa_sec_alloc_ctx(ses);
1382         if (!ctx)
1383                 return NULL;
1384         cf = &ctx->job;
1385         ctx->op = op;
1386
1387         src_start_addr = rte_pktmbuf_mtophys(sym->m_src);
1388
1389         if (sym->m_dst)
1390                 dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst);
1391         else
1392                 dst_start_addr = src_start_addr;
1393
1394         /* input */
1395         sg = &cf->sg[1];
1396         qm_sg_entry_set64(sg, src_start_addr);
1397         sg->length = sym->m_src->pkt_len;
1398         sg->final = 1;
1399         cpu_to_hw_sg(sg);
1400
1401         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1402         /* output */
1403         sg = &cf->sg[0];
1404         qm_sg_entry_set64(sg, dst_start_addr);
1405         sg->length = sym->m_src->buf_len - sym->m_src->data_off;
1406         cpu_to_hw_sg(sg);
1407
1408         return cf;
1409 }
1410
1411 static uint16_t
1412 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1413                        uint16_t nb_ops)
1414 {
1415         /* Function to transmit the frames to given device and queuepair */
1416         uint32_t loop;
1417         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1418         uint16_t num_tx = 0;
1419         struct qm_fd fds[DPAA_SEC_BURST], *fd;
1420         uint32_t frames_to_send;
1421         struct rte_crypto_op *op;
1422         struct dpaa_sec_job *cf;
1423         dpaa_sec_session *ses;
1424         uint32_t auth_only_len;
1425         struct qman_fq *inq[DPAA_SEC_BURST];
1426
1427         while (nb_ops) {
1428                 frames_to_send = (nb_ops > DPAA_SEC_BURST) ?
1429                                 DPAA_SEC_BURST : nb_ops;
1430                 for (loop = 0; loop < frames_to_send; loop++) {
1431                         op = *(ops++);
1432                         switch (op->sess_type) {
1433                         case RTE_CRYPTO_OP_WITH_SESSION:
1434                                 ses = (dpaa_sec_session *)
1435                                         get_sym_session_private_data(
1436                                                         op->sym->session,
1437                                                         cryptodev_driver_id);
1438                                 break;
1439                         case RTE_CRYPTO_OP_SECURITY_SESSION:
1440                                 ses = (dpaa_sec_session *)
1441                                         get_sec_session_private_data(
1442                                                         op->sym->sec_session);
1443                                 break;
1444                         default:
1445                                 DPAA_SEC_DP_ERR(
1446                                         "sessionless crypto op not supported");
1447                                 frames_to_send = loop;
1448                                 nb_ops = loop;
1449                                 goto send_pkts;
1450                         }
1451                         if (unlikely(!ses->qp || ses->qp != qp)) {
1452                                 DPAA_SEC_DP_ERR("sess->qp - %p qp %p",
1453                                              ses->qp, qp);
1454                                 if (dpaa_sec_attach_sess_q(qp, ses)) {
1455                                         frames_to_send = loop;
1456                                         nb_ops = loop;
1457                                         goto send_pkts;
1458                                 }
1459                         }
1460
1461                         auth_only_len = op->sym->auth.data.length -
1462                                                 op->sym->cipher.data.length;
1463                         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1464                                 if (is_auth_only(ses)) {
1465                                         cf = build_auth_only(op, ses);
1466                                 } else if (is_cipher_only(ses)) {
1467                                         cf = build_cipher_only(op, ses);
1468                                 } else if (is_aead(ses)) {
1469                                         cf = build_cipher_auth_gcm(op, ses);
1470                                         auth_only_len = ses->auth_only_len;
1471                                 } else if (is_auth_cipher(ses)) {
1472                                         cf = build_cipher_auth(op, ses);
1473                                 } else if (is_proto_ipsec(ses)) {
1474                                         cf = build_proto(op, ses);
1475                                 } else {
1476                                         DPAA_SEC_DP_ERR("not supported ops");
1477                                         frames_to_send = loop;
1478                                         nb_ops = loop;
1479                                         goto send_pkts;
1480                                 }
1481                         } else {
1482                                 if (is_auth_only(ses)) {
1483                                         cf = build_auth_only_sg(op, ses);
1484                                 } else if (is_cipher_only(ses)) {
1485                                         cf = build_cipher_only_sg(op, ses);
1486                                 } else if (is_aead(ses)) {
1487                                         cf = build_cipher_auth_gcm_sg(op, ses);
1488                                         auth_only_len = ses->auth_only_len;
1489                                 } else if (is_auth_cipher(ses)) {
1490                                         cf = build_cipher_auth_sg(op, ses);
1491                                 } else {
1492                                         DPAA_SEC_DP_ERR("not supported ops");
1493                                         frames_to_send = loop;
1494                                         nb_ops = loop;
1495                                         goto send_pkts;
1496                                 }
1497                         }
1498                         if (unlikely(!cf)) {
1499                                 frames_to_send = loop;
1500                                 nb_ops = loop;
1501                                 goto send_pkts;
1502                         }
1503
1504                         fd = &fds[loop];
1505                         inq[loop] = ses->inq;
1506                         fd->opaque_addr = 0;
1507                         fd->cmd = 0;
1508                         qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg));
1509                         fd->_format1 = qm_fd_compound;
1510                         fd->length29 = 2 * sizeof(struct qm_sg_entry);
1511                         /* Auth_only_len is set as 0 in descriptor and it is
1512                          * overwritten here in the fd.cmd which will update
1513                          * the DPOVRD reg.
1514                          */
1515                         if (auth_only_len)
1516                                 fd->cmd = 0x80000000 | auth_only_len;
1517
1518                 }
1519 send_pkts:
1520                 loop = 0;
1521                 while (loop < frames_to_send) {
1522                         loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
1523                                         frames_to_send - loop);
1524                 }
1525                 nb_ops -= frames_to_send;
1526                 num_tx += frames_to_send;
1527         }
1528
1529         dpaa_qp->tx_pkts += num_tx;
1530         dpaa_qp->tx_errs += nb_ops - num_tx;
1531
1532         return num_tx;
1533 }
1534
1535 static uint16_t
1536 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1537                        uint16_t nb_ops)
1538 {
1539         uint16_t num_rx;
1540         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1541
1542         num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops);
1543
1544         dpaa_qp->rx_pkts += num_rx;
1545         dpaa_qp->rx_errs += nb_ops - num_rx;
1546
1547         DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
1548
1549         return num_rx;
1550 }
1551
1552 /** Release queue pair */
1553 static int
1554 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev,
1555                             uint16_t qp_id)
1556 {
1557         struct dpaa_sec_dev_private *internals;
1558         struct dpaa_sec_qp *qp = NULL;
1559
1560         PMD_INIT_FUNC_TRACE();
1561
1562         DPAA_SEC_DEBUG("dev =%p, queue =%d", dev, qp_id);
1563
1564         internals = dev->data->dev_private;
1565         if (qp_id >= internals->max_nb_queue_pairs) {
1566                 DPAA_SEC_ERR("Max supported qpid %d",
1567                              internals->max_nb_queue_pairs);
1568                 return -EINVAL;
1569         }
1570
1571         qp = &internals->qps[qp_id];
1572         qp->internals = NULL;
1573         dev->data->queue_pairs[qp_id] = NULL;
1574
1575         return 0;
1576 }
1577
1578 /** Setup a queue pair */
1579 static int
1580 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1581                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1582                 __rte_unused int socket_id,
1583                 __rte_unused struct rte_mempool *session_pool)
1584 {
1585         struct dpaa_sec_dev_private *internals;
1586         struct dpaa_sec_qp *qp = NULL;
1587
1588         DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1589
1590         internals = dev->data->dev_private;
1591         if (qp_id >= internals->max_nb_queue_pairs) {
1592                 DPAA_SEC_ERR("Max supported qpid %d",
1593                              internals->max_nb_queue_pairs);
1594                 return -EINVAL;
1595         }
1596
1597         qp = &internals->qps[qp_id];
1598         qp->internals = internals;
1599         dev->data->queue_pairs[qp_id] = qp;
1600
1601         return 0;
1602 }
1603
1604 /** Return the number of allocated queue pairs */
1605 static uint32_t
1606 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev)
1607 {
1608         PMD_INIT_FUNC_TRACE();
1609
1610         return dev->data->nb_queue_pairs;
1611 }
1612
1613 /** Returns the size of session structure */
1614 static unsigned int
1615 dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1616 {
1617         PMD_INIT_FUNC_TRACE();
1618
1619         return sizeof(dpaa_sec_session);
1620 }
1621
1622 static int
1623 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused,
1624                      struct rte_crypto_sym_xform *xform,
1625                      dpaa_sec_session *session)
1626 {
1627         session->cipher_alg = xform->cipher.algo;
1628         session->iv.length = xform->cipher.iv.length;
1629         session->iv.offset = xform->cipher.iv.offset;
1630         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1631                                                RTE_CACHE_LINE_SIZE);
1632         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1633                 DPAA_SEC_ERR("No Memory for cipher key");
1634                 return -ENOMEM;
1635         }
1636         session->cipher_key.length = xform->cipher.key.length;
1637
1638         memcpy(session->cipher_key.data, xform->cipher.key.data,
1639                xform->cipher.key.length);
1640         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1641                         DIR_ENC : DIR_DEC;
1642
1643         return 0;
1644 }
1645
1646 static int
1647 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused,
1648                    struct rte_crypto_sym_xform *xform,
1649                    dpaa_sec_session *session)
1650 {
1651         session->auth_alg = xform->auth.algo;
1652         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1653                                              RTE_CACHE_LINE_SIZE);
1654         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1655                 DPAA_SEC_ERR("No Memory for auth key");
1656                 return -ENOMEM;
1657         }
1658         session->auth_key.length = xform->auth.key.length;
1659         session->digest_length = xform->auth.digest_length;
1660
1661         memcpy(session->auth_key.data, xform->auth.key.data,
1662                xform->auth.key.length);
1663         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1664                         DIR_ENC : DIR_DEC;
1665
1666         return 0;
1667 }
1668
1669 static int
1670 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused,
1671                    struct rte_crypto_sym_xform *xform,
1672                    dpaa_sec_session *session)
1673 {
1674         session->aead_alg = xform->aead.algo;
1675         session->iv.length = xform->aead.iv.length;
1676         session->iv.offset = xform->aead.iv.offset;
1677         session->auth_only_len = xform->aead.aad_length;
1678         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1679                                              RTE_CACHE_LINE_SIZE);
1680         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1681                 DPAA_SEC_ERR("No Memory for aead key\n");
1682                 return -ENOMEM;
1683         }
1684         session->aead_key.length = xform->aead.key.length;
1685         session->digest_length = xform->aead.digest_length;
1686
1687         memcpy(session->aead_key.data, xform->aead.key.data,
1688                xform->aead.key.length);
1689         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1690                         DIR_ENC : DIR_DEC;
1691
1692         return 0;
1693 }
1694
1695 static struct qman_fq *
1696 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi)
1697 {
1698         unsigned int i;
1699
1700         for (i = 0; i < qi->max_nb_sessions; i++) {
1701                 if (qi->inq_attach[i] == 0) {
1702                         qi->inq_attach[i] = 1;
1703                         return &qi->inq[i];
1704                 }
1705         }
1706         DPAA_SEC_WARN("All ses session in use %x", qi->max_nb_sessions);
1707
1708         return NULL;
1709 }
1710
1711 static int
1712 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
1713 {
1714         unsigned int i;
1715
1716         for (i = 0; i < qi->max_nb_sessions; i++) {
1717                 if (&qi->inq[i] == fq) {
1718                         qman_retire_fq(fq, NULL);
1719                         qman_oos_fq(fq);
1720                         qi->inq_attach[i] = 0;
1721                         return 0;
1722                 }
1723         }
1724         return -1;
1725 }
1726
1727 static int
1728 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
1729 {
1730         int ret;
1731
1732         sess->qp = qp;
1733         ret = dpaa_sec_prep_cdb(sess);
1734         if (ret) {
1735                 DPAA_SEC_ERR("Unable to prepare sec cdb");
1736                 return -1;
1737         }
1738         if (unlikely(!RTE_PER_LCORE(dpaa_io))) {
1739                 ret = rte_dpaa_portal_init((void *)0);
1740                 if (ret) {
1741                         DPAA_SEC_ERR("Failure in affining portal");
1742                         return ret;
1743                 }
1744         }
1745         ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb),
1746                                qman_fq_fqid(&qp->outq));
1747         if (ret)
1748                 DPAA_SEC_ERR("Unable to init sec queue");
1749
1750         return ret;
1751 }
1752
1753 static int
1754 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
1755                             struct rte_crypto_sym_xform *xform, void *sess)
1756 {
1757         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1758         dpaa_sec_session *session = sess;
1759
1760         PMD_INIT_FUNC_TRACE();
1761
1762         if (unlikely(sess == NULL)) {
1763                 DPAA_SEC_ERR("invalid session struct");
1764                 return -EINVAL;
1765         }
1766         memset(session, 0, sizeof(dpaa_sec_session));
1767
1768         /* Default IV length = 0 */
1769         session->iv.length = 0;
1770
1771         /* Cipher Only */
1772         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1773                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1774                 dpaa_sec_cipher_init(dev, xform, session);
1775
1776         /* Authentication Only */
1777         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1778                    xform->next == NULL) {
1779                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1780                 dpaa_sec_auth_init(dev, xform, session);
1781
1782         /* Cipher then Authenticate */
1783         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1784                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1785                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1786                         dpaa_sec_cipher_init(dev, xform, session);
1787                         dpaa_sec_auth_init(dev, xform->next, session);
1788                 } else {
1789                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1790                         return -EINVAL;
1791                 }
1792
1793         /* Authenticate then Cipher */
1794         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1795                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1796                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1797                         dpaa_sec_auth_init(dev, xform, session);
1798                         dpaa_sec_cipher_init(dev, xform->next, session);
1799                 } else {
1800                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1801                         return -EINVAL;
1802                 }
1803
1804         /* AEAD operation for AES-GCM kind of Algorithms */
1805         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1806                    xform->next == NULL) {
1807                 dpaa_sec_aead_init(dev, xform, session);
1808
1809         } else {
1810                 DPAA_SEC_ERR("Invalid crypto type");
1811                 return -EINVAL;
1812         }
1813         session->ctx_pool = internals->ctx_pool;
1814         rte_spinlock_lock(&internals->lock);
1815         session->inq = dpaa_sec_attach_rxq(internals);
1816         rte_spinlock_unlock(&internals->lock);
1817         if (session->inq == NULL) {
1818                 DPAA_SEC_ERR("unable to attach sec queue");
1819                 goto err1;
1820         }
1821
1822         return 0;
1823
1824 err1:
1825         rte_free(session->cipher_key.data);
1826         rte_free(session->auth_key.data);
1827         memset(session, 0, sizeof(dpaa_sec_session));
1828
1829         return -EINVAL;
1830 }
1831
1832 static int
1833 dpaa_sec_sym_session_configure(struct rte_cryptodev *dev,
1834                 struct rte_crypto_sym_xform *xform,
1835                 struct rte_cryptodev_sym_session *sess,
1836                 struct rte_mempool *mempool)
1837 {
1838         void *sess_private_data;
1839         int ret;
1840
1841         PMD_INIT_FUNC_TRACE();
1842
1843         if (rte_mempool_get(mempool, &sess_private_data)) {
1844                 DPAA_SEC_ERR("Couldn't get object from session mempool");
1845                 return -ENOMEM;
1846         }
1847
1848         ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data);
1849         if (ret != 0) {
1850                 DPAA_SEC_ERR("failed to configure session parameters");
1851
1852                 /* Return session to mempool */
1853                 rte_mempool_put(mempool, sess_private_data);
1854                 return ret;
1855         }
1856
1857         set_sym_session_private_data(sess, dev->driver_id,
1858                         sess_private_data);
1859
1860
1861         return 0;
1862 }
1863
1864 /** Clear the memory of session so it doesn't leave key material behind */
1865 static void
1866 dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
1867                 struct rte_cryptodev_sym_session *sess)
1868 {
1869         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
1870         uint8_t index = dev->driver_id;
1871         void *sess_priv = get_sym_session_private_data(sess, index);
1872
1873         PMD_INIT_FUNC_TRACE();
1874
1875         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
1876
1877         if (sess_priv) {
1878                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1879
1880                 if (s->inq)
1881                         dpaa_sec_detach_rxq(qi, s->inq);
1882                 rte_free(s->cipher_key.data);
1883                 rte_free(s->auth_key.data);
1884                 memset(s, 0, sizeof(dpaa_sec_session));
1885                 set_sym_session_private_data(sess, index, NULL);
1886                 rte_mempool_put(sess_mp, sess_priv);
1887         }
1888 }
1889
1890 static int
1891 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1892                            struct rte_security_session_conf *conf,
1893                            void *sess)
1894 {
1895         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1896         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1897         struct rte_crypto_auth_xform *auth_xform;
1898         struct rte_crypto_cipher_xform *cipher_xform;
1899         dpaa_sec_session *session = (dpaa_sec_session *)sess;
1900
1901         PMD_INIT_FUNC_TRACE();
1902
1903         memset(session, 0, sizeof(dpaa_sec_session));
1904         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1905                 cipher_xform = &conf->crypto_xform->cipher;
1906                 auth_xform = &conf->crypto_xform->next->auth;
1907         } else {
1908                 auth_xform = &conf->crypto_xform->auth;
1909                 cipher_xform = &conf->crypto_xform->next->cipher;
1910         }
1911         session->proto_alg = conf->protocol;
1912         session->cipher_key.data = rte_zmalloc(NULL,
1913                                                cipher_xform->key.length,
1914                                                RTE_CACHE_LINE_SIZE);
1915         if (session->cipher_key.data == NULL &&
1916                         cipher_xform->key.length > 0) {
1917                 DPAA_SEC_ERR("No Memory for cipher key");
1918                 return -ENOMEM;
1919         }
1920
1921         session->cipher_key.length = cipher_xform->key.length;
1922         session->auth_key.data = rte_zmalloc(NULL,
1923                                         auth_xform->key.length,
1924                                         RTE_CACHE_LINE_SIZE);
1925         if (session->auth_key.data == NULL &&
1926                         auth_xform->key.length > 0) {
1927                 DPAA_SEC_ERR("No Memory for auth key");
1928                 rte_free(session->cipher_key.data);
1929                 return -ENOMEM;
1930         }
1931         session->auth_key.length = auth_xform->key.length;
1932         memcpy(session->cipher_key.data, cipher_xform->key.data,
1933                         cipher_xform->key.length);
1934         memcpy(session->auth_key.data, auth_xform->key.data,
1935                         auth_xform->key.length);
1936
1937         switch (auth_xform->algo) {
1938         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1939                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1940                 break;
1941         case RTE_CRYPTO_AUTH_MD5_HMAC:
1942                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1943                 break;
1944         case RTE_CRYPTO_AUTH_SHA256_HMAC:
1945                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1946                 break;
1947         case RTE_CRYPTO_AUTH_SHA384_HMAC:
1948                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1949                 break;
1950         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1951                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1952                 break;
1953         case RTE_CRYPTO_AUTH_AES_CMAC:
1954                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1955                 break;
1956         case RTE_CRYPTO_AUTH_NULL:
1957                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1958                 break;
1959         case RTE_CRYPTO_AUTH_SHA224_HMAC:
1960         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
1961         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
1962         case RTE_CRYPTO_AUTH_SHA1:
1963         case RTE_CRYPTO_AUTH_SHA256:
1964         case RTE_CRYPTO_AUTH_SHA512:
1965         case RTE_CRYPTO_AUTH_SHA224:
1966         case RTE_CRYPTO_AUTH_SHA384:
1967         case RTE_CRYPTO_AUTH_MD5:
1968         case RTE_CRYPTO_AUTH_AES_GMAC:
1969         case RTE_CRYPTO_AUTH_KASUMI_F9:
1970         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
1971         case RTE_CRYPTO_AUTH_ZUC_EIA3:
1972                 DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
1973                         auth_xform->algo);
1974                 goto out;
1975         default:
1976                 DPAA_SEC_ERR("Crypto: Undefined Auth specified %u",
1977                         auth_xform->algo);
1978                 goto out;
1979         }
1980
1981         switch (cipher_xform->algo) {
1982         case RTE_CRYPTO_CIPHER_AES_CBC:
1983                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1984                 break;
1985         case RTE_CRYPTO_CIPHER_3DES_CBC:
1986                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1987                 break;
1988         case RTE_CRYPTO_CIPHER_AES_CTR:
1989                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1990                 break;
1991         case RTE_CRYPTO_CIPHER_NULL:
1992         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1993         case RTE_CRYPTO_CIPHER_3DES_ECB:
1994         case RTE_CRYPTO_CIPHER_AES_ECB:
1995         case RTE_CRYPTO_CIPHER_KASUMI_F8:
1996                 DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
1997                         cipher_xform->algo);
1998                 goto out;
1999         default:
2000                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
2001                         cipher_xform->algo);
2002                 goto out;
2003         }
2004
2005         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
2006                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
2007                                 sizeof(session->ip4_hdr));
2008                 session->ip4_hdr.ip_v = IPVERSION;
2009                 session->ip4_hdr.ip_hl = 5;
2010                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
2011                                                 sizeof(session->ip4_hdr));
2012                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2013                 session->ip4_hdr.ip_id = 0;
2014                 session->ip4_hdr.ip_off = 0;
2015                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2016                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
2017                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
2018                                 : IPPROTO_AH;
2019                 session->ip4_hdr.ip_sum = 0;
2020                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
2021                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
2022                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
2023                                                 (void *)&session->ip4_hdr,
2024                                                 sizeof(struct ip));
2025
2026                 session->encap_pdb.options =
2027                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2028                         PDBOPTS_ESP_OIHI_PDB_INL |
2029                         PDBOPTS_ESP_IVSRC |
2030                         PDBHMO_ESP_ENCAP_DTTL;
2031                 session->encap_pdb.spi = ipsec_xform->spi;
2032                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
2033
2034                 session->dir = DIR_ENC;
2035         } else if (ipsec_xform->direction ==
2036                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2037                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
2038                 session->decap_pdb.options = sizeof(struct ip) << 16;
2039                 session->dir = DIR_DEC;
2040         } else
2041                 goto out;
2042         session->ctx_pool = internals->ctx_pool;
2043         rte_spinlock_lock(&internals->lock);
2044         session->inq = dpaa_sec_attach_rxq(internals);
2045         rte_spinlock_unlock(&internals->lock);
2046         if (session->inq == NULL) {
2047                 DPAA_SEC_ERR("unable to attach sec queue");
2048                 goto out;
2049         }
2050
2051
2052         return 0;
2053 out:
2054         rte_free(session->auth_key.data);
2055         rte_free(session->cipher_key.data);
2056         memset(session, 0, sizeof(dpaa_sec_session));
2057         return -1;
2058 }
2059
2060 static int
2061 dpaa_sec_security_session_create(void *dev,
2062                                  struct rte_security_session_conf *conf,
2063                                  struct rte_security_session *sess,
2064                                  struct rte_mempool *mempool)
2065 {
2066         void *sess_private_data;
2067         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
2068         int ret;
2069
2070         if (rte_mempool_get(mempool, &sess_private_data)) {
2071                 DPAA_SEC_ERR("Couldn't get object from session mempool");
2072                 return -ENOMEM;
2073         }
2074
2075         switch (conf->protocol) {
2076         case RTE_SECURITY_PROTOCOL_IPSEC:
2077                 ret = dpaa_sec_set_ipsec_session(cdev, conf,
2078                                 sess_private_data);
2079                 break;
2080         case RTE_SECURITY_PROTOCOL_MACSEC:
2081                 return -ENOTSUP;
2082         default:
2083                 return -EINVAL;
2084         }
2085         if (ret != 0) {
2086                 DPAA_SEC_ERR("failed to configure session parameters");
2087                 /* Return session to mempool */
2088                 rte_mempool_put(mempool, sess_private_data);
2089                 return ret;
2090         }
2091
2092         set_sec_session_private_data(sess, sess_private_data);
2093
2094         return ret;
2095 }
2096
2097 /** Clear the memory of session so it doesn't leave key material behind */
2098 static int
2099 dpaa_sec_security_session_destroy(void *dev __rte_unused,
2100                 struct rte_security_session *sess)
2101 {
2102         PMD_INIT_FUNC_TRACE();
2103         void *sess_priv = get_sec_session_private_data(sess);
2104
2105         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
2106
2107         if (sess_priv) {
2108                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
2109
2110                 rte_free(s->cipher_key.data);
2111                 rte_free(s->auth_key.data);
2112                 memset(sess, 0, sizeof(dpaa_sec_session));
2113                 set_sec_session_private_data(sess, NULL);
2114                 rte_mempool_put(sess_mp, sess_priv);
2115         }
2116         return 0;
2117 }
2118
2119
2120 static int
2121 dpaa_sec_dev_configure(struct rte_cryptodev *dev,
2122                        struct rte_cryptodev_config *config __rte_unused)
2123 {
2124
2125         char str[20];
2126         struct dpaa_sec_dev_private *internals;
2127
2128         PMD_INIT_FUNC_TRACE();
2129
2130         internals = dev->data->dev_private;
2131         sprintf(str, "ctx_pool_%d", dev->data->dev_id);
2132         if (!internals->ctx_pool) {
2133                 internals->ctx_pool = rte_mempool_create((const char *)str,
2134                                                         CTX_POOL_NUM_BUFS,
2135                                                         CTX_POOL_BUF_SIZE,
2136                                                         CTX_POOL_CACHE_SIZE, 0,
2137                                                         NULL, NULL, NULL, NULL,
2138                                                         SOCKET_ID_ANY, 0);
2139                 if (!internals->ctx_pool) {
2140                         DPAA_SEC_ERR("%s create failed\n", str);
2141                         return -ENOMEM;
2142                 }
2143         } else
2144                 DPAA_SEC_INFO("mempool already created for dev_id : %d",
2145                                 dev->data->dev_id);
2146
2147         return 0;
2148 }
2149
2150 static int
2151 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused)
2152 {
2153         PMD_INIT_FUNC_TRACE();
2154         return 0;
2155 }
2156
2157 static void
2158 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused)
2159 {
2160         PMD_INIT_FUNC_TRACE();
2161 }
2162
2163 static int
2164 dpaa_sec_dev_close(struct rte_cryptodev *dev)
2165 {
2166         struct dpaa_sec_dev_private *internals;
2167
2168         PMD_INIT_FUNC_TRACE();
2169
2170         if (dev == NULL)
2171                 return -ENOMEM;
2172
2173         internals = dev->data->dev_private;
2174         rte_mempool_free(internals->ctx_pool);
2175         internals->ctx_pool = NULL;
2176
2177         return 0;
2178 }
2179
2180 static void
2181 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev,
2182                        struct rte_cryptodev_info *info)
2183 {
2184         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2185
2186         PMD_INIT_FUNC_TRACE();
2187         if (info != NULL) {
2188                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2189                 info->feature_flags = dev->feature_flags;
2190                 info->capabilities = dpaa_sec_capabilities;
2191                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2192                 info->driver_id = cryptodev_driver_id;
2193         }
2194 }
2195
2196 static struct rte_cryptodev_ops crypto_ops = {
2197         .dev_configure        = dpaa_sec_dev_configure,
2198         .dev_start            = dpaa_sec_dev_start,
2199         .dev_stop             = dpaa_sec_dev_stop,
2200         .dev_close            = dpaa_sec_dev_close,
2201         .dev_infos_get        = dpaa_sec_dev_infos_get,
2202         .queue_pair_setup     = dpaa_sec_queue_pair_setup,
2203         .queue_pair_release   = dpaa_sec_queue_pair_release,
2204         .queue_pair_count     = dpaa_sec_queue_pair_count,
2205         .sym_session_get_size     = dpaa_sec_sym_session_get_size,
2206         .sym_session_configure    = dpaa_sec_sym_session_configure,
2207         .sym_session_clear        = dpaa_sec_sym_session_clear
2208 };
2209
2210 static const struct rte_security_capability *
2211 dpaa_sec_capabilities_get(void *device __rte_unused)
2212 {
2213         return dpaa_sec_security_cap;
2214 }
2215
2216 struct rte_security_ops dpaa_sec_security_ops = {
2217         .session_create = dpaa_sec_security_session_create,
2218         .session_update = NULL,
2219         .session_stats_get = NULL,
2220         .session_destroy = dpaa_sec_security_session_destroy,
2221         .set_pkt_metadata = NULL,
2222         .capabilities_get = dpaa_sec_capabilities_get
2223 };
2224
2225 static int
2226 dpaa_sec_uninit(struct rte_cryptodev *dev)
2227 {
2228         struct dpaa_sec_dev_private *internals;
2229
2230         if (dev == NULL)
2231                 return -ENODEV;
2232
2233         internals = dev->data->dev_private;
2234         rte_free(dev->security_ctx);
2235
2236         /* In case close has been called, internals->ctx_pool would be NULL */
2237         rte_mempool_free(internals->ctx_pool);
2238         rte_free(internals);
2239
2240         DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u",
2241                       dev->data->name, rte_socket_id());
2242
2243         return 0;
2244 }
2245
2246 static int
2247 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
2248 {
2249         struct dpaa_sec_dev_private *internals;
2250         struct rte_security_ctx *security_instance;
2251         struct dpaa_sec_qp *qp;
2252         uint32_t i, flags;
2253         int ret;
2254
2255         PMD_INIT_FUNC_TRACE();
2256
2257         cryptodev->driver_id = cryptodev_driver_id;
2258         cryptodev->dev_ops = &crypto_ops;
2259
2260         cryptodev->enqueue_burst = dpaa_sec_enqueue_burst;
2261         cryptodev->dequeue_burst = dpaa_sec_dequeue_burst;
2262         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2263                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2264                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2265                         RTE_CRYPTODEV_FF_SECURITY |
2266                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2267                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
2268                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2269                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
2270                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
2271
2272         internals = cryptodev->data->dev_private;
2273         internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
2274         internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS;
2275
2276         /*
2277          * For secondary processes, we don't initialise any further as primary
2278          * has already done this work. Only check we don't need a different
2279          * RX function
2280          */
2281         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2282                 DPAA_SEC_WARN("Device already init by primary process");
2283                 return 0;
2284         }
2285
2286         /* Initialize security_ctx only for primary process*/
2287         security_instance = rte_malloc("rte_security_instances_ops",
2288                                 sizeof(struct rte_security_ctx), 0);
2289         if (security_instance == NULL)
2290                 return -ENOMEM;
2291         security_instance->device = (void *)cryptodev;
2292         security_instance->ops = &dpaa_sec_security_ops;
2293         security_instance->sess_cnt = 0;
2294         cryptodev->security_ctx = security_instance;
2295
2296         rte_spinlock_init(&internals->lock);
2297         for (i = 0; i < internals->max_nb_queue_pairs; i++) {
2298                 /* init qman fq for queue pair */
2299                 qp = &internals->qps[i];
2300                 ret = dpaa_sec_init_tx(&qp->outq);
2301                 if (ret) {
2302                         DPAA_SEC_ERR("config tx of queue pair  %d", i);
2303                         goto init_error;
2304                 }
2305         }
2306
2307         flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
2308                 QMAN_FQ_FLAG_TO_DCPORTAL;
2309         for (i = 0; i < internals->max_nb_sessions; i++) {
2310                 /* create rx qman fq for sessions*/
2311                 ret = qman_create_fq(0, flags, &internals->inq[i]);
2312                 if (unlikely(ret != 0)) {
2313                         DPAA_SEC_ERR("sec qman_create_fq failed");
2314                         goto init_error;
2315                 }
2316         }
2317
2318         RTE_LOG(INFO, PMD, "%s cryptodev init\n", cryptodev->data->name);
2319         return 0;
2320
2321 init_error:
2322         DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name);
2323
2324         dpaa_sec_uninit(cryptodev);
2325         return -EFAULT;
2326 }
2327
2328 static int
2329 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused,
2330                                 struct rte_dpaa_device *dpaa_dev)
2331 {
2332         struct rte_cryptodev *cryptodev;
2333         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
2334
2335         int retval;
2336
2337         sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id);
2338
2339         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
2340         if (cryptodev == NULL)
2341                 return -ENOMEM;
2342
2343         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2344                 cryptodev->data->dev_private = rte_zmalloc_socket(
2345                                         "cryptodev private structure",
2346                                         sizeof(struct dpaa_sec_dev_private),
2347                                         RTE_CACHE_LINE_SIZE,
2348                                         rte_socket_id());
2349
2350                 if (cryptodev->data->dev_private == NULL)
2351                         rte_panic("Cannot allocate memzone for private "
2352                                         "device data");
2353         }
2354
2355         dpaa_dev->crypto_dev = cryptodev;
2356         cryptodev->device = &dpaa_dev->device;
2357
2358         /* init user callbacks */
2359         TAILQ_INIT(&(cryptodev->link_intr_cbs));
2360
2361         /* if sec device version is not configured */
2362         if (!rta_get_sec_era()) {
2363                 const struct device_node *caam_node;
2364
2365                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2366                         const uint32_t *prop = of_get_property(caam_node,
2367                                         "fsl,sec-era",
2368                                         NULL);
2369                         if (prop) {
2370                                 rta_set_sec_era(
2371                                         INTL_SEC_ERA(rte_cpu_to_be_32(*prop)));
2372                                 break;
2373                         }
2374                 }
2375         }
2376
2377         /* Invoke PMD device initialization function */
2378         retval = dpaa_sec_dev_init(cryptodev);
2379         if (retval == 0)
2380                 return 0;
2381
2382         /* In case of error, cleanup is done */
2383         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2384                 rte_free(cryptodev->data->dev_private);
2385
2386         rte_cryptodev_pmd_release_device(cryptodev);
2387
2388         return -ENXIO;
2389 }
2390
2391 static int
2392 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev)
2393 {
2394         struct rte_cryptodev *cryptodev;
2395         int ret;
2396
2397         cryptodev = dpaa_dev->crypto_dev;
2398         if (cryptodev == NULL)
2399                 return -ENODEV;
2400
2401         ret = dpaa_sec_uninit(cryptodev);
2402         if (ret)
2403                 return ret;
2404
2405         return rte_cryptodev_pmd_destroy(cryptodev);
2406 }
2407
2408 static struct rte_dpaa_driver rte_dpaa_sec_driver = {
2409         .drv_type = FSL_DPAA_CRYPTO,
2410         .driver = {
2411                 .name = "DPAA SEC PMD"
2412         },
2413         .probe = cryptodev_dpaa_sec_probe,
2414         .remove = cryptodev_dpaa_sec_remove,
2415 };
2416
2417 static struct cryptodev_driver dpaa_sec_crypto_drv;
2418
2419 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver);
2420 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver,
2421                 cryptodev_driver_id);
2422
2423 RTE_INIT(dpaa_sec_init_log)
2424 {
2425         dpaa_logtype_sec = rte_log_register("pmd.crypto.dpaa");
2426         if (dpaa_logtype_sec >= 0)
2427                 rte_log_set_level(dpaa_logtype_sec, RTE_LOG_NOTICE);
2428 }