crypto/aesni_mb: support AES DOCSIS BPI
[dpdk.git] / drivers / crypto / aesni_mb / rte_aesni_mb_pmd_ops.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
5  *
6  *   Redistribution and use in source and binary forms, with or without
7  *   modification, are permitted provided that the following conditions
8  *   are met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above copyright
13  *       notice, this list of conditions and the following disclaimer in
14  *       the documentation and/or other materials provided with the
15  *       distribution.
16  *     * Neither the name of Intel Corporation nor the names of its
17  *       contributors may be used to endorse or promote products derived
18  *       from this software without specific prior written permission.
19  *
20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32
33 #include <string.h>
34
35 #include <rte_common.h>
36 #include <rte_malloc.h>
37 #include <rte_cryptodev_pmd.h>
38
39 #include "rte_aesni_mb_pmd_private.h"
40
41
42 static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = {
43         {       /* MD5 HMAC */
44                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
45                 {.sym = {
46                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
47                         {.auth = {
48                                 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
49                                 .block_size = 64,
50                                 .key_size = {
51                                         .min = 64,
52                                         .max = 64,
53                                         .increment = 0
54                                 },
55                                 .digest_size = {
56                                         .min = 12,
57                                         .max = 12,
58                                         .increment = 0
59                                 },
60                                 .aad_size = { 0 }
61                         }, }
62                 }, }
63         },
64         {       /* SHA1 HMAC */
65                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
66                 {.sym = {
67                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
68                         {.auth = {
69                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
70                                 .block_size = 64,
71                                 .key_size = {
72                                         .min = 64,
73                                         .max = 64,
74                                         .increment = 0
75                                 },
76                                 .digest_size = {
77                                         .min = 12,
78                                         .max = 12,
79                                         .increment = 0
80                                 },
81                                 .aad_size = { 0 }
82                         }, }
83                 }, }
84         },
85         {       /* SHA224 HMAC */
86                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
87                 {.sym = {
88                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
89                         {.auth = {
90                                 .algo = RTE_CRYPTO_AUTH_SHA224_HMAC,
91                                 .block_size = 64,
92                                 .key_size = {
93                                         .min = 64,
94                                         .max = 64,
95                                         .increment = 0
96                                 },
97                                 .digest_size = {
98                                         .min = 14,
99                                         .max = 14,
100                                         .increment = 0
101                                 },
102                                 .aad_size = { 0 }
103                         }, }
104                 }, }
105         },
106         {       /* SHA256 HMAC */
107                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
108                 {.sym = {
109                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
110                         {.auth = {
111                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
112                                 .block_size = 64,
113                                 .key_size = {
114                                         .min = 64,
115                                         .max = 64,
116                                         .increment = 0
117                                 },
118                                 .digest_size = {
119                                         .min = 16,
120                                         .max = 16,
121                                         .increment = 0
122                                 },
123                                 .aad_size = { 0 }
124                         }, }
125                 }, }
126         },
127         {       /* SHA384 HMAC */
128                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
129                 {.sym = {
130                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
131                         {.auth = {
132                                 .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
133                                 .block_size = 128,
134                                 .key_size = {
135                                         .min = 128,
136                                         .max = 128,
137                                         .increment = 0
138                                 },
139                                 .digest_size = {
140                                         .min = 24,
141                                         .max = 24,
142                                         .increment = 0
143                                 },
144                                 .aad_size = { 0 }
145                         }, }
146                 }, }
147         },
148         {       /* SHA512 HMAC */
149                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
150                 {.sym = {
151                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
152                         {.auth = {
153                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
154                                 .block_size = 128,
155                                 .key_size = {
156                                         .min = 128,
157                                         .max = 128,
158                                         .increment = 0
159                                 },
160                                 .digest_size = {
161                                         .min = 32,
162                                         .max = 32,
163                                         .increment = 0
164                                 },
165                                 .aad_size = { 0 }
166                         }, }
167                 }, }
168         },
169         {       /* AES XCBC HMAC */
170                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
171                 {.sym = {
172                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
173                         {.auth = {
174                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
175                                 .block_size = 16,
176                                 .key_size = {
177                                         .min = 16,
178                                         .max = 16,
179                                         .increment = 0
180                                 },
181                                 .digest_size = {
182                                         .min = 12,
183                                         .max = 12,
184                                         .increment = 0
185                                 },
186                                 .aad_size = { 0 }
187                         }, }
188                 }, }
189         },
190         {       /* AES CBC */
191                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
192                 {.sym = {
193                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
194                         {.cipher = {
195                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
196                                 .block_size = 16,
197                                 .key_size = {
198                                         .min = 16,
199                                         .max = 32,
200                                         .increment = 8
201                                 },
202                                 .iv_size = {
203                                         .min = 16,
204                                         .max = 16,
205                                         .increment = 0
206                                 }
207                         }, }
208                 }, }
209         },
210         {       /* AES CTR */
211                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
212                 {.sym = {
213                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
214                         {.cipher = {
215                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
216                                 .block_size = 16,
217                                 .key_size = {
218                                         .min = 16,
219                                         .max = 32,
220                                         .increment = 8
221                                 },
222                                 .iv_size = {
223                                         .min = 16,
224                                         .max = 16,
225                                         .increment = 0
226                                 }
227                         }, }
228                 }, }
229         },
230         {       /* AES DOCSIS BPI */
231                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
232                 {.sym = {
233                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
234                         {.cipher = {
235                                 .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI,
236                                 .block_size = 16,
237                                 .key_size = {
238                                         .min = 16,
239                                         .max = 16,
240                                         .increment = 0
241                                 },
242                                 .iv_size = {
243                                         .min = 16,
244                                         .max = 16,
245                                         .increment = 0
246                                 }
247                         }, }
248                 }, }
249         },
250
251         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
252 };
253
254
255 /** Configure device */
256 static int
257 aesni_mb_pmd_config(__rte_unused struct rte_cryptodev *dev,
258                 __rte_unused struct rte_cryptodev_config *config)
259 {
260         return 0;
261 }
262
263 /** Start device */
264 static int
265 aesni_mb_pmd_start(__rte_unused struct rte_cryptodev *dev)
266 {
267         return 0;
268 }
269
270 /** Stop device */
271 static void
272 aesni_mb_pmd_stop(__rte_unused struct rte_cryptodev *dev)
273 {
274 }
275
276 /** Close device */
277 static int
278 aesni_mb_pmd_close(__rte_unused struct rte_cryptodev *dev)
279 {
280         return 0;
281 }
282
283
284 /** Get device statistics */
285 static void
286 aesni_mb_pmd_stats_get(struct rte_cryptodev *dev,
287                 struct rte_cryptodev_stats *stats)
288 {
289         int qp_id;
290
291         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
292                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
293
294                 stats->enqueued_count += qp->stats.enqueued_count;
295                 stats->dequeued_count += qp->stats.dequeued_count;
296
297                 stats->enqueue_err_count += qp->stats.enqueue_err_count;
298                 stats->dequeue_err_count += qp->stats.dequeue_err_count;
299         }
300 }
301
302 /** Reset device statistics */
303 static void
304 aesni_mb_pmd_stats_reset(struct rte_cryptodev *dev)
305 {
306         int qp_id;
307
308         for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
309                 struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
310
311                 memset(&qp->stats, 0, sizeof(qp->stats));
312         }
313 }
314
315
316 /** Get device info */
317 static void
318 aesni_mb_pmd_info_get(struct rte_cryptodev *dev,
319                 struct rte_cryptodev_info *dev_info)
320 {
321         struct aesni_mb_private *internals = dev->data->dev_private;
322
323         if (dev_info != NULL) {
324                 dev_info->dev_type = dev->dev_type;
325                 dev_info->feature_flags = dev->feature_flags;
326                 dev_info->capabilities = aesni_mb_pmd_capabilities;
327                 dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
328                 dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
329         }
330 }
331
332 /** Release queue pair */
333 static int
334 aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
335 {
336         struct aesni_mb_qp *qp = dev->data->queue_pairs[qp_id];
337         struct rte_ring *r = NULL;
338
339         if (qp != NULL) {
340                 r = rte_ring_lookup(qp->name);
341                 if (r)
342                         rte_ring_free(r);
343                 rte_free(qp);
344                 dev->data->queue_pairs[qp_id] = NULL;
345         }
346         return 0;
347 }
348
349 /** set a unique name for the queue pair based on it's name, dev_id and qp_id */
350 static int
351 aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev,
352                 struct aesni_mb_qp *qp)
353 {
354         unsigned n = snprintf(qp->name, sizeof(qp->name),
355                         "aesni_mb_pmd_%u_qp_%u",
356                         dev->data->dev_id, qp->id);
357
358         if (n > sizeof(qp->name))
359                 return -1;
360
361         return 0;
362 }
363
364 /** Create a ring to place processed operations on */
365 static struct rte_ring *
366 aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp,
367                 const char *str, unsigned int ring_size, int socket_id)
368 {
369         struct rte_ring *r;
370         char ring_name[RTE_CRYPTODEV_NAME_LEN];
371
372         unsigned int n = snprintf(ring_name, sizeof(ring_name),
373                                 "%s_%s",
374                                 qp->name, str);
375
376         if (n > sizeof(ring_name))
377                 return NULL;
378
379         r = rte_ring_lookup(ring_name);
380         if (r) {
381                 if (rte_ring_get_size(r) >= ring_size) {
382                         MB_LOG_INFO("Reusing existing ring %s for processed ops",
383                         ring_name);
384                         return r;
385                 }
386
387                 MB_LOG_ERR("Unable to reuse existing ring %s for processed ops",
388                         ring_name);
389                 return NULL;
390         }
391
392         return rte_ring_create(ring_name, ring_size, socket_id,
393                         RING_F_SP_ENQ | RING_F_SC_DEQ);
394 }
395
396 /** Setup a queue pair */
397 static int
398 aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
399                 const struct rte_cryptodev_qp_conf *qp_conf,
400                  int socket_id)
401 {
402         struct aesni_mb_qp *qp = NULL;
403         struct aesni_mb_private *internals = dev->data->dev_private;
404
405         /* Free memory prior to re-allocation if needed. */
406         if (dev->data->queue_pairs[qp_id] != NULL)
407                 aesni_mb_pmd_qp_release(dev, qp_id);
408
409         /* Allocate the queue pair data structure. */
410         qp = rte_zmalloc_socket("AES-NI PMD Queue Pair", sizeof(*qp),
411                                         RTE_CACHE_LINE_SIZE, socket_id);
412         if (qp == NULL)
413                 return -ENOMEM;
414
415         qp->id = qp_id;
416         dev->data->queue_pairs[qp_id] = qp;
417
418         if (aesni_mb_pmd_qp_set_unique_name(dev, qp))
419                 goto qp_setup_cleanup;
420
421
422         qp->op_fns = &job_ops[internals->vector_mode];
423
424         qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp,
425                         "ingress", qp_conf->nb_descriptors, socket_id);
426         if (qp->ingress_queue == NULL)
427                 goto qp_setup_cleanup;
428
429         qp->sess_mp = dev->data->session_pool;
430
431         memset(&qp->stats, 0, sizeof(qp->stats));
432
433         /* Initialise multi-buffer manager */
434         (*qp->op_fns->job.init_mgr)(&qp->mb_mgr);
435         return 0;
436
437 qp_setup_cleanup:
438         if (qp)
439                 rte_free(qp);
440
441         return -1;
442 }
443
444 /** Start queue pair */
445 static int
446 aesni_mb_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
447                 __rte_unused uint16_t queue_pair_id)
448 {
449         return -ENOTSUP;
450 }
451
452 /** Stop queue pair */
453 static int
454 aesni_mb_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
455                 __rte_unused uint16_t queue_pair_id)
456 {
457         return -ENOTSUP;
458 }
459
460 /** Return the number of allocated queue pairs */
461 static uint32_t
462 aesni_mb_pmd_qp_count(struct rte_cryptodev *dev)
463 {
464         return dev->data->nb_queue_pairs;
465 }
466
467 /** Returns the size of the aesni multi-buffer session structure */
468 static unsigned
469 aesni_mb_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
470 {
471         return sizeof(struct aesni_mb_session);
472 }
473
474 /** Configure a aesni multi-buffer session from a crypto xform chain */
475 static void *
476 aesni_mb_pmd_session_configure(struct rte_cryptodev *dev,
477                 struct rte_crypto_sym_xform *xform,     void *sess)
478 {
479         struct aesni_mb_private *internals = dev->data->dev_private;
480
481         if (unlikely(sess == NULL)) {
482                 MB_LOG_ERR("invalid session struct");
483                 return NULL;
484         }
485
486         if (aesni_mb_set_session_parameters(&job_ops[internals->vector_mode],
487                         sess, xform) != 0) {
488                 MB_LOG_ERR("failed configure session parameters");
489                 return NULL;
490         }
491
492         return sess;
493 }
494
495 /** Clear the memory of session so it doesn't leave key material behind */
496 static void
497 aesni_mb_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess)
498 {
499         /*
500          * Current just resetting the whole data structure, need to investigate
501          * whether a more selective reset of key would be more performant
502          */
503         if (sess)
504                 memset(sess, 0, sizeof(struct aesni_mb_session));
505 }
506
507 struct rte_cryptodev_ops aesni_mb_pmd_ops = {
508                 .dev_configure          = aesni_mb_pmd_config,
509                 .dev_start              = aesni_mb_pmd_start,
510                 .dev_stop               = aesni_mb_pmd_stop,
511                 .dev_close              = aesni_mb_pmd_close,
512
513                 .stats_get              = aesni_mb_pmd_stats_get,
514                 .stats_reset            = aesni_mb_pmd_stats_reset,
515
516                 .dev_infos_get          = aesni_mb_pmd_info_get,
517
518                 .queue_pair_setup       = aesni_mb_pmd_qp_setup,
519                 .queue_pair_release     = aesni_mb_pmd_qp_release,
520                 .queue_pair_start       = aesni_mb_pmd_qp_start,
521                 .queue_pair_stop        = aesni_mb_pmd_qp_stop,
522                 .queue_pair_count       = aesni_mb_pmd_qp_count,
523
524                 .session_get_size       = aesni_mb_pmd_session_get_size,
525                 .session_configure      = aesni_mb_pmd_session_configure,
526                 .session_clear          = aesni_mb_pmd_session_clear
527 };
528
529 struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops;