f584d0d6f14958b9ff0e7b79a8c29b04b5a4d0db
[dpdk.git] / drivers / crypto / openssl / rte_openssl_pmd.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2016-2017 Intel Corporation
3  */
4
5 #include <rte_common.h>
6 #include <rte_hexdump.h>
7 #include <rte_cryptodev.h>
8 #include <rte_cryptodev_pmd.h>
9 #include <rte_bus_vdev.h>
10 #include <rte_malloc.h>
11 #include <rte_cpuflags.h>
12
13 #include <openssl/hmac.h>
14 #include <openssl/evp.h>
15
16 #include "rte_openssl_pmd_private.h"
17
18 #define DES_BLOCK_SIZE 8
19
20 static uint8_t cryptodev_driver_id;
21
22 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
23 static HMAC_CTX *HMAC_CTX_new(void)
24 {
25         HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
26
27         if (ctx != NULL)
28                 HMAC_CTX_init(ctx);
29         return ctx;
30 }
31
32 static void HMAC_CTX_free(HMAC_CTX *ctx)
33 {
34         if (ctx != NULL) {
35                 HMAC_CTX_cleanup(ctx);
36                 OPENSSL_free(ctx);
37         }
38 }
39 #endif
40
41 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
42
43 /*----------------------------------------------------------------------------*/
44
45 /**
46  * Increment counter by 1
47  * Counter is 64 bit array, big-endian
48  */
49 static void
50 ctr_inc(uint8_t *ctr)
51 {
52         uint64_t *ctr64 = (uint64_t *)ctr;
53
54         *ctr64 = __builtin_bswap64(*ctr64);
55         (*ctr64)++;
56         *ctr64 = __builtin_bswap64(*ctr64);
57 }
58
59 /*
60  *------------------------------------------------------------------------------
61  * Session Prepare
62  *------------------------------------------------------------------------------
63  */
64
65 /** Get xform chain order */
66 static enum openssl_chain_order
67 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
68 {
69         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
70
71         if (xform != NULL) {
72                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
73                         if (xform->next == NULL)
74                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
75                         else if (xform->next->type ==
76                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
77                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
78                 }
79                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
80                         if (xform->next == NULL)
81                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
82                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
83                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
84                 }
85                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
86                         res = OPENSSL_CHAIN_COMBINED;
87         }
88
89         return res;
90 }
91
92 /** Get session cipher key from input cipher key */
93 static void
94 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key)
95 {
96         memcpy(session_key, input_key, keylen);
97 }
98
99 /** Get key ede 24 bytes standard from input key */
100 static int
101 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede)
102 {
103         int res = 0;
104
105         /* Initialize keys - 24 bytes: [key1-key2-key3] */
106         switch (keylen) {
107         case 24:
108                 memcpy(key_ede, key, 24);
109                 break;
110         case 16:
111                 /* K3 = K1 */
112                 memcpy(key_ede, key, 16);
113                 memcpy(key_ede + 16, key, 8);
114                 break;
115         case 8:
116                 /* K1 = K2 = K3 (DES compatibility) */
117                 memcpy(key_ede, key, 8);
118                 memcpy(key_ede + 8, key, 8);
119                 memcpy(key_ede + 16, key, 8);
120                 break;
121         default:
122                 OPENSSL_LOG_ERR("Unsupported key size");
123                 res = -EINVAL;
124         }
125
126         return res;
127 }
128
129 /** Get adequate openssl function for input cipher algorithm */
130 static uint8_t
131 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
132                 const EVP_CIPHER **algo)
133 {
134         int res = 0;
135
136         if (algo != NULL) {
137                 switch (sess_algo) {
138                 case RTE_CRYPTO_CIPHER_3DES_CBC:
139                         switch (keylen) {
140                         case 16:
141                                 *algo = EVP_des_ede_cbc();
142                                 break;
143                         case 24:
144                                 *algo = EVP_des_ede3_cbc();
145                                 break;
146                         default:
147                                 res = -EINVAL;
148                         }
149                         break;
150                 case RTE_CRYPTO_CIPHER_3DES_CTR:
151                         break;
152                 case RTE_CRYPTO_CIPHER_AES_CBC:
153                         switch (keylen) {
154                         case 16:
155                                 *algo = EVP_aes_128_cbc();
156                                 break;
157                         case 24:
158                                 *algo = EVP_aes_192_cbc();
159                                 break;
160                         case 32:
161                                 *algo = EVP_aes_256_cbc();
162                                 break;
163                         default:
164                                 res = -EINVAL;
165                         }
166                         break;
167                 case RTE_CRYPTO_CIPHER_AES_CTR:
168                         switch (keylen) {
169                         case 16:
170                                 *algo = EVP_aes_128_ctr();
171                                 break;
172                         case 24:
173                                 *algo = EVP_aes_192_ctr();
174                                 break;
175                         case 32:
176                                 *algo = EVP_aes_256_ctr();
177                                 break;
178                         default:
179                                 res = -EINVAL;
180                         }
181                         break;
182                 default:
183                         res = -EINVAL;
184                         break;
185                 }
186         } else {
187                 res = -EINVAL;
188         }
189
190         return res;
191 }
192
193 /** Get adequate openssl function for input auth algorithm */
194 static uint8_t
195 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
196                 const EVP_MD **algo)
197 {
198         int res = 0;
199
200         if (algo != NULL) {
201                 switch (sessalgo) {
202                 case RTE_CRYPTO_AUTH_MD5:
203                 case RTE_CRYPTO_AUTH_MD5_HMAC:
204                         *algo = EVP_md5();
205                         break;
206                 case RTE_CRYPTO_AUTH_SHA1:
207                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
208                         *algo = EVP_sha1();
209                         break;
210                 case RTE_CRYPTO_AUTH_SHA224:
211                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
212                         *algo = EVP_sha224();
213                         break;
214                 case RTE_CRYPTO_AUTH_SHA256:
215                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
216                         *algo = EVP_sha256();
217                         break;
218                 case RTE_CRYPTO_AUTH_SHA384:
219                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
220                         *algo = EVP_sha384();
221                         break;
222                 case RTE_CRYPTO_AUTH_SHA512:
223                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
224                         *algo = EVP_sha512();
225                         break;
226                 default:
227                         res = -EINVAL;
228                         break;
229                 }
230         } else {
231                 res = -EINVAL;
232         }
233
234         return res;
235 }
236
237 /** Get adequate openssl function for input cipher algorithm */
238 static uint8_t
239 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
240                 const EVP_CIPHER **algo)
241 {
242         int res = 0;
243
244         if (algo != NULL) {
245                 switch (sess_algo) {
246                 case RTE_CRYPTO_AEAD_AES_GCM:
247                         switch (keylen) {
248                         case 16:
249                                 *algo = EVP_aes_128_gcm();
250                                 break;
251                         case 24:
252                                 *algo = EVP_aes_192_gcm();
253                                 break;
254                         case 32:
255                                 *algo = EVP_aes_256_gcm();
256                                 break;
257                         default:
258                                 res = -EINVAL;
259                         }
260                         break;
261                 case RTE_CRYPTO_AEAD_AES_CCM:
262                         switch (keylen) {
263                         case 16:
264                                 *algo = EVP_aes_128_ccm();
265                                 break;
266                         case 24:
267                                 *algo = EVP_aes_192_ccm();
268                                 break;
269                         case 32:
270                                 *algo = EVP_aes_256_ccm();
271                                 break;
272                         default:
273                                 res = -EINVAL;
274                         }
275                         break;
276                 default:
277                         res = -EINVAL;
278                         break;
279                 }
280         } else {
281                 res = -EINVAL;
282         }
283
284         return res;
285 }
286
287 /* Set session AEAD encryption parameters */
288 static int
289 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
290                 enum rte_crypto_aead_algorithm algo,
291                 uint8_t tag_len, uint8_t *key)
292 {
293         int iv_type = 0;
294         unsigned int do_ccm;
295
296         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
297         sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
298
299         /* Select AEAD algo */
300         switch (algo) {
301         case RTE_CRYPTO_AEAD_AES_GCM:
302                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
303                 if (tag_len != 16)
304                         return -EINVAL;
305                 do_ccm = 0;
306                 break;
307         case RTE_CRYPTO_AEAD_AES_CCM:
308                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
309                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
310                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
311                         return -EINVAL;
312                 do_ccm = 1;
313                 break;
314         default:
315                 return -ENOTSUP;
316         }
317
318         sess->cipher.mode = OPENSSL_CIPHER_LIB;
319         sess->cipher.ctx = EVP_CIPHER_CTX_new();
320
321         if (get_aead_algo(algo, sess->cipher.key.length,
322                         &sess->cipher.evp_algo) != 0)
323                 return -EINVAL;
324
325         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
326
327         sess->chain_order = OPENSSL_CHAIN_COMBINED;
328
329         if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
330                         NULL, NULL, NULL) <= 0)
331                 return -EINVAL;
332
333         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
334                         NULL) <= 0)
335                 return -EINVAL;
336
337         if (do_ccm)
338                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
339                                 tag_len, NULL);
340
341         if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
342                 return -EINVAL;
343
344         return 0;
345 }
346
347 /* Set session AEAD decryption parameters */
348 static int
349 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
350                 enum rte_crypto_aead_algorithm algo,
351                 uint8_t tag_len, uint8_t *key)
352 {
353         int iv_type = 0;
354         unsigned int do_ccm = 0;
355
356         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
357         sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
358
359         /* Select AEAD algo */
360         switch (algo) {
361         case RTE_CRYPTO_AEAD_AES_GCM:
362                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
363                 if (tag_len != 16)
364                         return -EINVAL;
365                 break;
366         case RTE_CRYPTO_AEAD_AES_CCM:
367                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
368                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
369                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
370                         return -EINVAL;
371                 do_ccm = 1;
372                 break;
373         default:
374                 return -ENOTSUP;
375         }
376
377         sess->cipher.mode = OPENSSL_CIPHER_LIB;
378         sess->cipher.ctx = EVP_CIPHER_CTX_new();
379
380         if (get_aead_algo(algo, sess->cipher.key.length,
381                         &sess->cipher.evp_algo) != 0)
382                 return -EINVAL;
383
384         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
385
386         sess->chain_order = OPENSSL_CHAIN_COMBINED;
387
388         if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
389                         NULL, NULL, NULL) <= 0)
390                 return -EINVAL;
391
392         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
393                         sess->iv.length, NULL) <= 0)
394                 return -EINVAL;
395
396         if (do_ccm)
397                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
398                                 tag_len, NULL);
399
400         if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
401                 return -EINVAL;
402
403         return 0;
404 }
405
406 /** Set session cipher parameters */
407 static int
408 openssl_set_session_cipher_parameters(struct openssl_session *sess,
409                 const struct rte_crypto_sym_xform *xform)
410 {
411         /* Select cipher direction */
412         sess->cipher.direction = xform->cipher.op;
413         /* Select cipher key */
414         sess->cipher.key.length = xform->cipher.key.length;
415
416         /* Set IV parameters */
417         sess->iv.offset = xform->cipher.iv.offset;
418         sess->iv.length = xform->cipher.iv.length;
419
420         /* Select cipher algo */
421         switch (xform->cipher.algo) {
422         case RTE_CRYPTO_CIPHER_3DES_CBC:
423         case RTE_CRYPTO_CIPHER_AES_CBC:
424         case RTE_CRYPTO_CIPHER_AES_CTR:
425                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
426                 sess->cipher.algo = xform->cipher.algo;
427                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
428
429                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
430                                 &sess->cipher.evp_algo) != 0)
431                         return -EINVAL;
432
433                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
434                         sess->cipher.key.data);
435                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
436                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
437                                         sess->cipher.evp_algo,
438                                         NULL, xform->cipher.key.data,
439                                         NULL) != 1) {
440                                 return -EINVAL;
441                         }
442                 } else if (sess->cipher.direction ==
443                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
444                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
445                                         sess->cipher.evp_algo,
446                                         NULL, xform->cipher.key.data,
447                                         NULL) != 1) {
448                                 return -EINVAL;
449                         }
450                 }
451
452                 break;
453
454         case RTE_CRYPTO_CIPHER_3DES_CTR:
455                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
456                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
457
458                 if (get_cipher_key_ede(xform->cipher.key.data,
459                                 sess->cipher.key.length,
460                                 sess->cipher.key.data) != 0)
461                         return -EINVAL;
462                 break;
463
464         case RTE_CRYPTO_CIPHER_DES_CBC:
465                 sess->cipher.algo = xform->cipher.algo;
466                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
467                 sess->cipher.evp_algo = EVP_des_cbc();
468
469                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
470                         sess->cipher.key.data);
471                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
472                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
473                                         sess->cipher.evp_algo,
474                                         NULL, xform->cipher.key.data,
475                                         NULL) != 1) {
476                                 return -EINVAL;
477                         }
478                 } else if (sess->cipher.direction ==
479                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
480                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
481                                         sess->cipher.evp_algo,
482                                         NULL, xform->cipher.key.data,
483                                         NULL) != 1) {
484                                 return -EINVAL;
485                         }
486                 }
487
488                 break;
489
490         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
491                 sess->cipher.algo = xform->cipher.algo;
492                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
493                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
494                 sess->cipher.evp_algo = EVP_des_cbc();
495
496                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
497                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
498                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
499                                 NULL, xform->cipher.key.data, 0) != 1)
500                         return -EINVAL;
501
502                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
503                         sess->cipher.key.data);
504                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
505                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
506                                         sess->cipher.evp_algo,
507                                         NULL, xform->cipher.key.data,
508                                         NULL) != 1) {
509                                 return -EINVAL;
510                         }
511                 } else if (sess->cipher.direction ==
512                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
513                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
514                                         sess->cipher.evp_algo,
515                                         NULL, xform->cipher.key.data,
516                                         NULL) != 1) {
517                                 return -EINVAL;
518                         }
519                 }
520
521                 break;
522         default:
523                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
524                 return -ENOTSUP;
525         }
526
527         return 0;
528 }
529
530 /* Set session auth parameters */
531 static int
532 openssl_set_session_auth_parameters(struct openssl_session *sess,
533                 const struct rte_crypto_sym_xform *xform)
534 {
535         /* Select auth generate/verify */
536         sess->auth.operation = xform->auth.op;
537         sess->auth.algo = xform->auth.algo;
538
539         sess->auth.digest_length = xform->auth.digest_length;
540
541         /* Select auth algo */
542         switch (xform->auth.algo) {
543         case RTE_CRYPTO_AUTH_AES_GMAC:
544                 /*
545                  * OpenSSL requires GMAC to be a GCM operation
546                  * with no cipher data length
547                  */
548                 sess->cipher.key.length = xform->auth.key.length;
549
550                 /* Set IV parameters */
551                 sess->iv.offset = xform->auth.iv.offset;
552                 sess->iv.length = xform->auth.iv.length;
553
554                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
555                         return openssl_set_sess_aead_enc_param(sess,
556                                                 RTE_CRYPTO_AEAD_AES_GCM,
557                                                 xform->auth.digest_length,
558                                                 xform->auth.key.data);
559                 else
560                         return openssl_set_sess_aead_dec_param(sess,
561                                                 RTE_CRYPTO_AEAD_AES_GCM,
562                                                 xform->auth.digest_length,
563                                                 xform->auth.key.data);
564                 break;
565
566         case RTE_CRYPTO_AUTH_MD5:
567         case RTE_CRYPTO_AUTH_SHA1:
568         case RTE_CRYPTO_AUTH_SHA224:
569         case RTE_CRYPTO_AUTH_SHA256:
570         case RTE_CRYPTO_AUTH_SHA384:
571         case RTE_CRYPTO_AUTH_SHA512:
572                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
573                 if (get_auth_algo(xform->auth.algo,
574                                 &sess->auth.auth.evp_algo) != 0)
575                         return -EINVAL;
576                 sess->auth.auth.ctx = EVP_MD_CTX_create();
577                 break;
578
579         case RTE_CRYPTO_AUTH_MD5_HMAC:
580         case RTE_CRYPTO_AUTH_SHA1_HMAC:
581         case RTE_CRYPTO_AUTH_SHA224_HMAC:
582         case RTE_CRYPTO_AUTH_SHA256_HMAC:
583         case RTE_CRYPTO_AUTH_SHA384_HMAC:
584         case RTE_CRYPTO_AUTH_SHA512_HMAC:
585                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
586                 sess->auth.hmac.ctx = HMAC_CTX_new();
587                 if (get_auth_algo(xform->auth.algo,
588                                 &sess->auth.hmac.evp_algo) != 0)
589                         return -EINVAL;
590
591                 if (HMAC_Init_ex(sess->auth.hmac.ctx,
592                                 xform->auth.key.data,
593                                 xform->auth.key.length,
594                                 sess->auth.hmac.evp_algo, NULL) != 1)
595                         return -EINVAL;
596                 break;
597
598         default:
599                 return -ENOTSUP;
600         }
601
602         return 0;
603 }
604
605 /* Set session AEAD parameters */
606 static int
607 openssl_set_session_aead_parameters(struct openssl_session *sess,
608                 const struct rte_crypto_sym_xform *xform)
609 {
610         /* Select cipher key */
611         sess->cipher.key.length = xform->aead.key.length;
612
613         /* Set IV parameters */
614         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
615                 /*
616                  * For AES-CCM, the actual IV is placed
617                  * one byte after the start of the IV field,
618                  * according to the API.
619                  */
620                 sess->iv.offset = xform->aead.iv.offset + 1;
621         else
622                 sess->iv.offset = xform->aead.iv.offset;
623
624         sess->iv.length = xform->aead.iv.length;
625
626         sess->auth.aad_length = xform->aead.aad_length;
627         sess->auth.digest_length = xform->aead.digest_length;
628
629         sess->aead_algo = xform->aead.algo;
630         /* Select cipher direction */
631         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
632                 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
633                                 xform->aead.digest_length, xform->aead.key.data);
634         else
635                 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
636                                 xform->aead.digest_length, xform->aead.key.data);
637 }
638
639 /** Parse crypto xform chain and set private session parameters */
640 int
641 openssl_set_session_parameters(struct openssl_session *sess,
642                 const struct rte_crypto_sym_xform *xform)
643 {
644         const struct rte_crypto_sym_xform *cipher_xform = NULL;
645         const struct rte_crypto_sym_xform *auth_xform = NULL;
646         const struct rte_crypto_sym_xform *aead_xform = NULL;
647         int ret;
648
649         sess->chain_order = openssl_get_chain_order(xform);
650         switch (sess->chain_order) {
651         case OPENSSL_CHAIN_ONLY_CIPHER:
652                 cipher_xform = xform;
653                 break;
654         case OPENSSL_CHAIN_ONLY_AUTH:
655                 auth_xform = xform;
656                 break;
657         case OPENSSL_CHAIN_CIPHER_AUTH:
658                 cipher_xform = xform;
659                 auth_xform = xform->next;
660                 break;
661         case OPENSSL_CHAIN_AUTH_CIPHER:
662                 auth_xform = xform;
663                 cipher_xform = xform->next;
664                 break;
665         case OPENSSL_CHAIN_COMBINED:
666                 aead_xform = xform;
667                 break;
668         default:
669                 return -EINVAL;
670         }
671
672         /* Default IV length = 0 */
673         sess->iv.length = 0;
674
675         /* cipher_xform must be check before auth_xform */
676         if (cipher_xform) {
677                 ret = openssl_set_session_cipher_parameters(
678                                 sess, cipher_xform);
679                 if (ret != 0) {
680                         OPENSSL_LOG_ERR(
681                                 "Invalid/unsupported cipher parameters");
682                         return ret;
683                 }
684         }
685
686         if (auth_xform) {
687                 ret = openssl_set_session_auth_parameters(sess, auth_xform);
688                 if (ret != 0) {
689                         OPENSSL_LOG_ERR(
690                                 "Invalid/unsupported auth parameters");
691                         return ret;
692                 }
693         }
694
695         if (aead_xform) {
696                 ret = openssl_set_session_aead_parameters(sess, aead_xform);
697                 if (ret != 0) {
698                         OPENSSL_LOG_ERR(
699                                 "Invalid/unsupported AEAD parameters");
700                         return ret;
701                 }
702         }
703
704         return 0;
705 }
706
707 /** Reset private session parameters */
708 void
709 openssl_reset_session(struct openssl_session *sess)
710 {
711         EVP_CIPHER_CTX_free(sess->cipher.ctx);
712
713         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
714                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
715
716         switch (sess->auth.mode) {
717         case OPENSSL_AUTH_AS_AUTH:
718                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
719                 break;
720         case OPENSSL_AUTH_AS_HMAC:
721                 EVP_PKEY_free(sess->auth.hmac.pkey);
722                 HMAC_CTX_free(sess->auth.hmac.ctx);
723                 break;
724         default:
725                 break;
726         }
727 }
728
729 /** Provide session for operation */
730 static struct openssl_session *
731 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
732 {
733         struct openssl_session *sess = NULL;
734
735         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
736                 /* get existing session */
737                 if (likely(op->sym->session != NULL))
738                         sess = (struct openssl_session *)
739                                         get_session_private_data(
740                                         op->sym->session,
741                                         cryptodev_driver_id);
742         } else {
743                 /* provide internal session */
744                 void *_sess = NULL;
745                 void *_sess_private_data = NULL;
746
747                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
748                         return NULL;
749
750                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
751                         return NULL;
752
753                 sess = (struct openssl_session *)_sess_private_data;
754
755                 if (unlikely(openssl_set_session_parameters(sess,
756                                 op->sym->xform) != 0)) {
757                         rte_mempool_put(qp->sess_mp, _sess);
758                         rte_mempool_put(qp->sess_mp, _sess_private_data);
759                         sess = NULL;
760                 }
761                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
762                 set_session_private_data(op->sym->session, cryptodev_driver_id,
763                         _sess_private_data);
764         }
765
766         if (sess == NULL)
767                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
768
769         return sess;
770 }
771
772 /*
773  *------------------------------------------------------------------------------
774  * Process Operations
775  *------------------------------------------------------------------------------
776  */
777 static inline int
778 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
779                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
780 {
781         struct rte_mbuf *m;
782         int dstlen;
783         int l, n = srclen;
784         uint8_t *src;
785
786         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
787                         m = m->next)
788                 offset -= rte_pktmbuf_data_len(m);
789
790         if (m == 0)
791                 return -1;
792
793         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
794
795         l = rte_pktmbuf_data_len(m) - offset;
796         if (srclen <= l) {
797                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
798                         return -1;
799                 *dst += l;
800                 return 0;
801         }
802
803         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
804                 return -1;
805
806         *dst += dstlen;
807         n -= l;
808
809         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
810                 src = rte_pktmbuf_mtod(m, uint8_t *);
811                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
812                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
813                         return -1;
814                 *dst += dstlen;
815                 n -= l;
816         }
817
818         return 0;
819 }
820
821 static inline int
822 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
823                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
824 {
825         struct rte_mbuf *m;
826         int dstlen;
827         int l, n = srclen;
828         uint8_t *src;
829
830         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
831                         m = m->next)
832                 offset -= rte_pktmbuf_data_len(m);
833
834         if (m == 0)
835                 return -1;
836
837         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
838
839         l = rte_pktmbuf_data_len(m) - offset;
840         if (srclen <= l) {
841                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
842                         return -1;
843                 *dst += l;
844                 return 0;
845         }
846
847         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
848                 return -1;
849
850         *dst += dstlen;
851         n -= l;
852
853         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
854                 src = rte_pktmbuf_mtod(m, uint8_t *);
855                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
856                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
857                         return -1;
858                 *dst += dstlen;
859                 n -= l;
860         }
861
862         return 0;
863 }
864
865 /** Process standard openssl cipher encryption */
866 static int
867 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
868                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
869 {
870         int totlen;
871
872         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
873                 goto process_cipher_encrypt_err;
874
875         EVP_CIPHER_CTX_set_padding(ctx, 0);
876
877         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
878                         srclen, ctx))
879                 goto process_cipher_encrypt_err;
880
881         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
882                 goto process_cipher_encrypt_err;
883
884         return 0;
885
886 process_cipher_encrypt_err:
887         OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
888         return -EINVAL;
889 }
890
891 /** Process standard openssl cipher encryption */
892 static int
893 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
894                 uint8_t *iv, int srclen,
895                 EVP_CIPHER_CTX *ctx)
896 {
897         uint8_t i;
898         uint8_t encrypted_iv[DES_BLOCK_SIZE];
899         int encrypted_ivlen;
900
901         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
902                         iv, DES_BLOCK_SIZE) <= 0)
903                 goto process_cipher_encrypt_err;
904
905         for (i = 0; i < srclen; i++)
906                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
907
908         return 0;
909
910 process_cipher_encrypt_err:
911         OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
912         return -EINVAL;
913 }
914 /** Process standard openssl cipher decryption */
915 static int
916 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
917                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
918 {
919         int totlen;
920
921         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
922                 goto process_cipher_decrypt_err;
923
924         EVP_CIPHER_CTX_set_padding(ctx, 0);
925
926         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
927                         srclen, ctx))
928                 goto process_cipher_decrypt_err;
929
930         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
931                 goto process_cipher_decrypt_err;
932         return 0;
933
934 process_cipher_decrypt_err:
935         OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
936         return -EINVAL;
937 }
938
939 /** Process cipher des 3 ctr encryption, decryption algorithm */
940 static int
941 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
942                 int offset, uint8_t *iv, uint8_t *key, int srclen,
943                 EVP_CIPHER_CTX *ctx)
944 {
945         uint8_t ebuf[8], ctr[8];
946         int unused, n;
947         struct rte_mbuf *m;
948         uint8_t *src;
949         int l;
950
951         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
952                         m = m->next)
953                 offset -= rte_pktmbuf_data_len(m);
954
955         if (m == 0)
956                 goto process_cipher_des3ctr_err;
957
958         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
959         l = rte_pktmbuf_data_len(m) - offset;
960
961         /* We use 3DES encryption also for decryption.
962          * IV is not important for 3DES ecb
963          */
964         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
965                 goto process_cipher_des3ctr_err;
966
967         memcpy(ctr, iv, 8);
968
969         for (n = 0; n < srclen; n++) {
970                 if (n % 8 == 0) {
971                         if (EVP_EncryptUpdate(ctx,
972                                         (unsigned char *)&ebuf, &unused,
973                                         (const unsigned char *)&ctr, 8) <= 0)
974                                 goto process_cipher_des3ctr_err;
975                         ctr_inc(ctr);
976                 }
977                 dst[n] = *(src++) ^ ebuf[n % 8];
978
979                 l--;
980                 if (!l) {
981                         m = m->next;
982                         if (m) {
983                                 src = rte_pktmbuf_mtod(m, uint8_t *);
984                                 l = rte_pktmbuf_data_len(m);
985                         }
986                 }
987         }
988
989         return 0;
990
991 process_cipher_des3ctr_err:
992         OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
993         return -EINVAL;
994 }
995
996 /** Process AES-GCM encrypt algorithm */
997 static int
998 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
999                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1000                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1001 {
1002         int len = 0, unused = 0;
1003         uint8_t empty[] = {};
1004
1005         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1006                 goto process_auth_encryption_gcm_err;
1007
1008         if (aadlen > 0)
1009                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1010                         goto process_auth_encryption_gcm_err;
1011
1012         if (srclen > 0)
1013                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1014                                 srclen, ctx))
1015                         goto process_auth_encryption_gcm_err;
1016
1017         /* Workaround open ssl bug in version less then 1.0.1f */
1018         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1019                 goto process_auth_encryption_gcm_err;
1020
1021         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1022                 goto process_auth_encryption_gcm_err;
1023
1024         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1025                 goto process_auth_encryption_gcm_err;
1026
1027         return 0;
1028
1029 process_auth_encryption_gcm_err:
1030         OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
1031         return -EINVAL;
1032 }
1033
1034 /** Process AES-CCM encrypt algorithm */
1035 static int
1036 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1037                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1038                 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1039 {
1040         int len = 0;
1041
1042         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1043                 goto process_auth_encryption_ccm_err;
1044
1045         if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1046                 goto process_auth_encryption_ccm_err;
1047
1048         if (aadlen > 0)
1049                 /*
1050                  * For AES-CCM, the actual AAD is placed
1051                  * 18 bytes after the start of the AAD field,
1052                  * according to the API.
1053                  */
1054                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1055                         goto process_auth_encryption_ccm_err;
1056
1057         if (srclen > 0)
1058                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1059                                 srclen, ctx))
1060                         goto process_auth_encryption_ccm_err;
1061
1062         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1063                 goto process_auth_encryption_ccm_err;
1064
1065         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1066                 goto process_auth_encryption_ccm_err;
1067
1068         return 0;
1069
1070 process_auth_encryption_ccm_err:
1071         OPENSSL_LOG_ERR("Process openssl auth encryption ccm failed");
1072         return -EINVAL;
1073 }
1074
1075 /** Process AES-GCM decrypt algorithm */
1076 static int
1077 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1078                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1079                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1080 {
1081         int len = 0, unused = 0;
1082         uint8_t empty[] = {};
1083
1084         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1085                 goto process_auth_decryption_gcm_err;
1086
1087         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1088                 goto process_auth_decryption_gcm_err;
1089
1090         if (aadlen > 0)
1091                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1092                         goto process_auth_decryption_gcm_err;
1093
1094         if (srclen > 0)
1095                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1096                                 srclen, ctx))
1097                         goto process_auth_decryption_gcm_err;
1098
1099         /* Workaround open ssl bug in version less then 1.0.1f */
1100         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1101                 goto process_auth_decryption_gcm_err;
1102
1103         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1104                 return -EFAULT;
1105
1106         return 0;
1107
1108 process_auth_decryption_gcm_err:
1109         OPENSSL_LOG_ERR("Process openssl auth decryption gcm failed");
1110         return -EINVAL;
1111 }
1112
1113 /** Process AES-CCM decrypt algorithm */
1114 static int
1115 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1116                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1117                 uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1118                 EVP_CIPHER_CTX *ctx)
1119 {
1120         int len = 0;
1121
1122         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1123                 goto process_auth_decryption_ccm_err;
1124
1125         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1126                 goto process_auth_decryption_ccm_err;
1127
1128         if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1129                 goto process_auth_decryption_ccm_err;
1130
1131         if (aadlen > 0)
1132                 /*
1133                  * For AES-CCM, the actual AAD is placed
1134                  * 18 bytes after the start of the AAD field,
1135                  * according to the API.
1136                  */
1137                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1138                         goto process_auth_decryption_ccm_err;
1139
1140         if (srclen > 0)
1141                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1142                                 srclen, ctx))
1143                         return -EFAULT;
1144
1145         return 0;
1146
1147 process_auth_decryption_ccm_err:
1148         OPENSSL_LOG_ERR("Process openssl auth decryption ccm failed");
1149         return -EINVAL;
1150 }
1151
1152 /** Process standard openssl auth algorithms */
1153 static int
1154 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1155                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1156                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1157 {
1158         size_t dstlen;
1159         struct rte_mbuf *m;
1160         int l, n = srclen;
1161         uint8_t *src;
1162
1163         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1164                         m = m->next)
1165                 offset -= rte_pktmbuf_data_len(m);
1166
1167         if (m == 0)
1168                 goto process_auth_err;
1169
1170         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1171                 goto process_auth_err;
1172
1173         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1174
1175         l = rte_pktmbuf_data_len(m) - offset;
1176         if (srclen <= l) {
1177                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1178                         goto process_auth_err;
1179                 goto process_auth_final;
1180         }
1181
1182         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1183                 goto process_auth_err;
1184
1185         n -= l;
1186
1187         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1188                 src = rte_pktmbuf_mtod(m, uint8_t *);
1189                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1190                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1191                         goto process_auth_err;
1192                 n -= l;
1193         }
1194
1195 process_auth_final:
1196         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1197                 goto process_auth_err;
1198         return 0;
1199
1200 process_auth_err:
1201         OPENSSL_LOG_ERR("Process openssl auth failed");
1202         return -EINVAL;
1203 }
1204
1205 /** Process standard openssl auth algorithms with hmac */
1206 static int
1207 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1208                 int srclen, HMAC_CTX *ctx)
1209 {
1210         unsigned int dstlen;
1211         struct rte_mbuf *m;
1212         int l, n = srclen;
1213         uint8_t *src;
1214
1215         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1216                         m = m->next)
1217                 offset -= rte_pktmbuf_data_len(m);
1218
1219         if (m == 0)
1220                 goto process_auth_err;
1221
1222         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1223
1224         l = rte_pktmbuf_data_len(m) - offset;
1225         if (srclen <= l) {
1226                 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1227                         goto process_auth_err;
1228                 goto process_auth_final;
1229         }
1230
1231         if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1232                 goto process_auth_err;
1233
1234         n -= l;
1235
1236         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1237                 src = rte_pktmbuf_mtod(m, uint8_t *);
1238                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1239                 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1240                         goto process_auth_err;
1241                 n -= l;
1242         }
1243
1244 process_auth_final:
1245         if (HMAC_Final(ctx, dst, &dstlen) != 1)
1246                 goto process_auth_err;
1247
1248         if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1249                 goto process_auth_err;
1250
1251         return 0;
1252
1253 process_auth_err:
1254         OPENSSL_LOG_ERR("Process openssl auth failed");
1255         return -EINVAL;
1256 }
1257
1258 /*----------------------------------------------------------------------------*/
1259
1260 /** Process auth/cipher combined operation */
1261 static void
1262 process_openssl_combined_op
1263                 (struct rte_crypto_op *op, struct openssl_session *sess,
1264                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1265 {
1266         /* cipher */
1267         uint8_t *dst = NULL, *iv, *tag, *aad;
1268         int srclen, aadlen, status = -1;
1269         uint32_t offset;
1270         uint8_t taglen;
1271
1272         /*
1273          * Segmented destination buffer is not supported for
1274          * encryption/decryption
1275          */
1276         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1277                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1278                 return;
1279         }
1280
1281         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1282                         sess->iv.offset);
1283         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1284                 srclen = 0;
1285                 offset = op->sym->auth.data.offset;
1286                 aadlen = op->sym->auth.data.length;
1287                 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1288                                 op->sym->auth.data.offset);
1289                 tag = op->sym->auth.digest.data;
1290                 if (tag == NULL)
1291                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1292                                 offset + aadlen);
1293         } else {
1294                 srclen = op->sym->aead.data.length;
1295                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1296                                 op->sym->aead.data.offset);
1297                 offset = op->sym->aead.data.offset;
1298                 aad = op->sym->aead.aad.data;
1299                 aadlen = sess->auth.aad_length;
1300                 tag = op->sym->aead.digest.data;
1301                 if (tag == NULL)
1302                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1303                                 offset + srclen);
1304         }
1305
1306         taglen = sess->auth.digest_length;
1307
1308         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1309                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1310                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1311                         status = process_openssl_auth_encryption_gcm(
1312                                         mbuf_src, offset, srclen,
1313                                         aad, aadlen, iv,
1314                                         dst, tag, sess->cipher.ctx);
1315                 else
1316                         status = process_openssl_auth_encryption_ccm(
1317                                         mbuf_src, offset, srclen,
1318                                         aad, aadlen, iv,
1319                                         dst, tag, taglen, sess->cipher.ctx);
1320
1321         } else {
1322                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1323                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1324                         status = process_openssl_auth_decryption_gcm(
1325                                         mbuf_src, offset, srclen,
1326                                         aad, aadlen, iv,
1327                                         dst, tag, sess->cipher.ctx);
1328                 else
1329                         status = process_openssl_auth_decryption_ccm(
1330                                         mbuf_src, offset, srclen,
1331                                         aad, aadlen, iv,
1332                                         dst, tag, taglen, sess->cipher.ctx);
1333         }
1334
1335         if (status != 0) {
1336                 if (status == (-EFAULT) &&
1337                                 sess->auth.operation ==
1338                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
1339                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1340                 else
1341                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1342         }
1343 }
1344
1345 /** Process cipher operation */
1346 static void
1347 process_openssl_cipher_op
1348                 (struct rte_crypto_op *op, struct openssl_session *sess,
1349                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1350 {
1351         uint8_t *dst, *iv;
1352         int srclen, status;
1353
1354         /*
1355          * Segmented destination buffer is not supported for
1356          * encryption/decryption
1357          */
1358         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1359                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1360                 return;
1361         }
1362
1363         srclen = op->sym->cipher.data.length;
1364         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1365                         op->sym->cipher.data.offset);
1366
1367         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1368                         sess->iv.offset);
1369
1370         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1371                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1372                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1373                                         op->sym->cipher.data.offset, iv,
1374                                         srclen, sess->cipher.ctx);
1375                 else
1376                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1377                                         op->sym->cipher.data.offset, iv,
1378                                         srclen, sess->cipher.ctx);
1379         else
1380                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1381                                 op->sym->cipher.data.offset, iv,
1382                                 sess->cipher.key.data, srclen,
1383                                 sess->cipher.ctx);
1384
1385         if (status != 0)
1386                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1387 }
1388
1389 /** Process cipher operation */
1390 static void
1391 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1392                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1393                 struct rte_mbuf *mbuf_dst)
1394 {
1395         uint8_t *src, *dst, *iv;
1396         uint8_t block_size, last_block_len;
1397         int srclen, status = 0;
1398
1399         srclen = op->sym->cipher.data.length;
1400         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1401                         op->sym->cipher.data.offset);
1402         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1403                         op->sym->cipher.data.offset);
1404
1405         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1406                         sess->iv.offset);
1407
1408         block_size = DES_BLOCK_SIZE;
1409
1410         last_block_len = srclen % block_size;
1411         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1412                 /* Encrypt only with ECB mode XOR IV */
1413                 if (srclen < block_size) {
1414                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1415                                         iv, srclen,
1416                                         sess->cipher.bpi_ctx);
1417                 } else {
1418                         srclen -= last_block_len;
1419                         /* Encrypt with the block aligned stream with CBC mode */
1420                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1421                                         op->sym->cipher.data.offset, iv,
1422                                         srclen, sess->cipher.ctx);
1423                         if (last_block_len) {
1424                                 /* Point at last block */
1425                                 dst += srclen;
1426                                 /*
1427                                  * IV is the last encrypted block from
1428                                  * the previous operation
1429                                  */
1430                                 iv = dst - block_size;
1431                                 src += srclen;
1432                                 srclen = last_block_len;
1433                                 /* Encrypt the last frame with ECB mode */
1434                                 status |= process_openssl_cipher_bpi_encrypt(src,
1435                                                 dst, iv,
1436                                                 srclen, sess->cipher.bpi_ctx);
1437                         }
1438                 }
1439         } else {
1440                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1441                 if (srclen < block_size) {
1442                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1443                                         iv,
1444                                         srclen,
1445                                         sess->cipher.bpi_ctx);
1446                 } else {
1447                         if (last_block_len) {
1448                                 /* Point at last block */
1449                                 dst += srclen - last_block_len;
1450                                 src += srclen - last_block_len;
1451                                 /*
1452                                  * IV is the last full block
1453                                  */
1454                                 iv = src - block_size;
1455                                 /*
1456                                  * Decrypt the last frame with ECB mode
1457                                  * (encrypt, as it is the same operation)
1458                                  */
1459                                 status = process_openssl_cipher_bpi_encrypt(src,
1460                                                 dst, iv,
1461                                                 last_block_len, sess->cipher.bpi_ctx);
1462                                 /* Prepare parameters for CBC mode op */
1463                                 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1464                                                 sess->iv.offset);
1465                                 dst += last_block_len - srclen;
1466                                 srclen -= last_block_len;
1467                         }
1468
1469                         /* Decrypt with CBC mode */
1470                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1471                                         op->sym->cipher.data.offset, iv,
1472                                         srclen, sess->cipher.ctx);
1473                 }
1474         }
1475
1476         if (status != 0)
1477                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1478 }
1479
1480 /** Process auth operation */
1481 static void
1482 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1483                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1484                 struct rte_mbuf *mbuf_dst)
1485 {
1486         uint8_t *dst;
1487         int srclen, status;
1488
1489         srclen = op->sym->auth.data.length;
1490
1491         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1492                 dst = qp->temp_digest;
1493         else {
1494                 dst = op->sym->auth.digest.data;
1495                 if (dst == NULL)
1496                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1497                                         op->sym->auth.data.offset +
1498                                         op->sym->auth.data.length);
1499         }
1500
1501         switch (sess->auth.mode) {
1502         case OPENSSL_AUTH_AS_AUTH:
1503                 status = process_openssl_auth(mbuf_src, dst,
1504                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1505                                 sess->auth.auth.ctx, sess->auth.auth.evp_algo);
1506                 break;
1507         case OPENSSL_AUTH_AS_HMAC:
1508                 status = process_openssl_auth_hmac(mbuf_src, dst,
1509                                 op->sym->auth.data.offset, srclen,
1510                                 sess->auth.hmac.ctx);
1511                 break;
1512         default:
1513                 status = -1;
1514                 break;
1515         }
1516
1517         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1518                 if (memcmp(dst, op->sym->auth.digest.data,
1519                                 sess->auth.digest_length) != 0) {
1520                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1521                 }
1522         }
1523
1524         if (status != 0)
1525                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1526 }
1527
1528 /** Process crypto operation for mbuf */
1529 static int
1530 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1531                 struct openssl_session *sess)
1532 {
1533         struct rte_mbuf *msrc, *mdst;
1534         int retval;
1535
1536         msrc = op->sym->m_src;
1537         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1538
1539         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1540
1541         switch (sess->chain_order) {
1542         case OPENSSL_CHAIN_ONLY_CIPHER:
1543                 process_openssl_cipher_op(op, sess, msrc, mdst);
1544                 break;
1545         case OPENSSL_CHAIN_ONLY_AUTH:
1546                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1547                 break;
1548         case OPENSSL_CHAIN_CIPHER_AUTH:
1549                 process_openssl_cipher_op(op, sess, msrc, mdst);
1550                 process_openssl_auth_op(qp, op, sess, mdst, mdst);
1551                 break;
1552         case OPENSSL_CHAIN_AUTH_CIPHER:
1553                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1554                 process_openssl_cipher_op(op, sess, msrc, mdst);
1555                 break;
1556         case OPENSSL_CHAIN_COMBINED:
1557                 process_openssl_combined_op(op, sess, msrc, mdst);
1558                 break;
1559         case OPENSSL_CHAIN_CIPHER_BPI:
1560                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
1561                 break;
1562         default:
1563                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1564                 break;
1565         }
1566
1567         /* Free session if a session-less crypto op */
1568         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1569                 openssl_reset_session(sess);
1570                 memset(sess, 0, sizeof(struct openssl_session));
1571                 memset(op->sym->session, 0,
1572                                 rte_cryptodev_get_header_session_size());
1573                 rte_mempool_put(qp->sess_mp, sess);
1574                 rte_mempool_put(qp->sess_mp, op->sym->session);
1575                 op->sym->session = NULL;
1576         }
1577
1578         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
1579                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1580
1581         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
1582                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
1583         else
1584                 retval = -1;
1585
1586         return retval;
1587 }
1588
1589 /*
1590  *------------------------------------------------------------------------------
1591  * PMD Framework
1592  *------------------------------------------------------------------------------
1593  */
1594
1595 /** Enqueue burst */
1596 static uint16_t
1597 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
1598                 uint16_t nb_ops)
1599 {
1600         struct openssl_session *sess;
1601         struct openssl_qp *qp = queue_pair;
1602         int i, retval;
1603
1604         for (i = 0; i < nb_ops; i++) {
1605                 sess = get_session(qp, ops[i]);
1606                 if (unlikely(sess == NULL))
1607                         goto enqueue_err;
1608
1609                 retval = process_op(qp, ops[i], sess);
1610                 if (unlikely(retval < 0))
1611                         goto enqueue_err;
1612         }
1613
1614         qp->stats.enqueued_count += i;
1615         return i;
1616
1617 enqueue_err:
1618         qp->stats.enqueue_err_count++;
1619         return i;
1620 }
1621
1622 /** Dequeue burst */
1623 static uint16_t
1624 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1625                 uint16_t nb_ops)
1626 {
1627         struct openssl_qp *qp = queue_pair;
1628
1629         unsigned int nb_dequeued = 0;
1630
1631         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
1632                         (void **)ops, nb_ops, NULL);
1633         qp->stats.dequeued_count += nb_dequeued;
1634
1635         return nb_dequeued;
1636 }
1637
1638 /** Create OPENSSL crypto device */
1639 static int
1640 cryptodev_openssl_create(const char *name,
1641                         struct rte_vdev_device *vdev,
1642                         struct rte_cryptodev_pmd_init_params *init_params)
1643 {
1644         struct rte_cryptodev *dev;
1645         struct openssl_private *internals;
1646
1647         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1648         if (dev == NULL) {
1649                 OPENSSL_LOG_ERR("failed to create cryptodev vdev");
1650                 goto init_error;
1651         }
1652
1653         dev->driver_id = cryptodev_driver_id;
1654         dev->dev_ops = rte_openssl_pmd_ops;
1655
1656         /* register rx/tx burst functions for data path */
1657         dev->dequeue_burst = openssl_pmd_dequeue_burst;
1658         dev->enqueue_burst = openssl_pmd_enqueue_burst;
1659
1660         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1661                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1662                         RTE_CRYPTODEV_FF_CPU_AESNI |
1663                         RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
1664
1665         /* Set vector instructions mode supported */
1666         internals = dev->data->dev_private;
1667
1668         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
1669         internals->max_nb_sessions = init_params->max_nb_sessions;
1670
1671         return 0;
1672
1673 init_error:
1674         OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
1675                         init_params->name);
1676
1677         cryptodev_openssl_remove(vdev);
1678         return -EFAULT;
1679 }
1680
1681 /** Initialise OPENSSL crypto device */
1682 static int
1683 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
1684 {
1685         struct rte_cryptodev_pmd_init_params init_params = {
1686                 "",
1687                 sizeof(struct openssl_private),
1688                 rte_socket_id(),
1689                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
1690                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
1691         };
1692         const char *name;
1693         const char *input_args;
1694
1695         name = rte_vdev_device_name(vdev);
1696         if (name == NULL)
1697                 return -EINVAL;
1698         input_args = rte_vdev_device_args(vdev);
1699
1700         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
1701
1702         return cryptodev_openssl_create(name, vdev, &init_params);
1703 }
1704
1705 /** Uninitialise OPENSSL crypto device */
1706 static int
1707 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
1708 {
1709         struct rte_cryptodev *cryptodev;
1710         const char *name;
1711
1712         name = rte_vdev_device_name(vdev);
1713         if (name == NULL)
1714                 return -EINVAL;
1715
1716         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
1717         if (cryptodev == NULL)
1718                 return -ENODEV;
1719
1720         return rte_cryptodev_pmd_destroy(cryptodev);
1721 }
1722
1723 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
1724         .probe = cryptodev_openssl_probe,
1725         .remove = cryptodev_openssl_remove
1726 };
1727
1728 static struct cryptodev_driver openssl_crypto_drv;
1729
1730 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
1731         cryptodev_openssl_pmd_drv);
1732 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
1733         "max_nb_queue_pairs=<int> "
1734         "max_nb_sessions=<int> "
1735         "socket_id=<int>");
1736 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv,
1737                 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id);