4 * Copyright(c) 2017 Marvell International Ltd.
5 * Copyright(c) 2017 Semihalf.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * * Neither the name of the copyright holder nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 #include <rte_common.h>
38 #include <rte_malloc.h>
39 #include <rte_cryptodev_pmd.h>
41 #include "rte_mrvl_pmd_private.h"
44 * Capabilities list to be used in reporting to DPDK.
46 static const struct rte_cryptodev_capabilities
47 mrvl_crypto_pmd_capabilities[] = {
49 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
51 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
53 .algo = RTE_CRYPTO_AUTH_MD5_HMAC,
69 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
71 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
73 .algo = RTE_CRYPTO_AUTH_MD5,
89 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
91 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
93 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
109 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
111 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
113 .algo = RTE_CRYPTO_AUTH_SHA1,
129 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
131 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
133 .algo = RTE_CRYPTO_AUTH_SHA224,
149 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
151 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
153 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
169 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
171 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
173 .algo = RTE_CRYPTO_AUTH_SHA256,
189 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
191 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
193 .algo = RTE_CRYPTO_AUTH_SHA384_HMAC,
209 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
211 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
213 .algo = RTE_CRYPTO_AUTH_SHA384,
229 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
231 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
233 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
249 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
251 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
253 .algo = RTE_CRYPTO_AUTH_SHA512,
269 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
271 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
273 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
289 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
291 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
293 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
309 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
311 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
313 .algo = RTE_CRYPTO_AEAD_AES_GCM,
338 { /* AES GMAC (AUTH) */
339 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
341 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
343 .algo = RTE_CRYPTO_AUTH_AES_GMAC,
364 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
366 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
368 .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
384 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
386 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
388 .algo = RTE_CRYPTO_CIPHER_3DES_CTR,
404 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
409 * Configure device (PMD ops callback).
411 * @param dev Pointer to the device structure.
412 * @param config Pointer to configuration structure.
413 * @returns 0. Always.
416 mrvl_crypto_pmd_config(__rte_unused struct rte_cryptodev *dev,
417 __rte_unused struct rte_cryptodev_config *config)
423 * Start device (PMD ops callback).
425 * @param dev Pointer to the device structure.
426 * @returns 0. Always.
429 mrvl_crypto_pmd_start(__rte_unused struct rte_cryptodev *dev)
435 * Stop device (PMD ops callback).
437 * @param dev Pointer to the device structure.
438 * @returns 0. Always.
441 mrvl_crypto_pmd_stop(__rte_unused struct rte_cryptodev *dev)
446 * Get device statistics (PMD ops callback).
448 * @param dev Pointer to the device structure.
449 * @param stats Pointer to statistics structure [out].
452 mrvl_crypto_pmd_stats_get(struct rte_cryptodev *dev,
453 struct rte_cryptodev_stats *stats)
457 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
458 struct mrvl_crypto_qp *qp = dev->data->queue_pairs[qp_id];
460 stats->enqueued_count += qp->stats.enqueued_count;
461 stats->dequeued_count += qp->stats.dequeued_count;
463 stats->enqueue_err_count += qp->stats.enqueue_err_count;
464 stats->dequeue_err_count += qp->stats.dequeue_err_count;
469 * Reset device statistics (PMD ops callback).
471 * @param dev Pointer to the device structure.
474 mrvl_crypto_pmd_stats_reset(struct rte_cryptodev *dev)
478 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++) {
479 struct mrvl_crypto_qp *qp = dev->data->queue_pairs[qp_id];
481 memset(&qp->stats, 0, sizeof(qp->stats));
486 * Get device info (PMD ops callback).
488 * @param dev Pointer to the device structure.
489 * @param dev_info Pointer to the device info structure [out].
492 mrvl_crypto_pmd_info_get(struct rte_cryptodev *dev,
493 struct rte_cryptodev_info *dev_info)
495 struct mrvl_crypto_private *internals = dev->data->dev_private;
497 if (dev_info != NULL) {
498 dev_info->driver_id = dev->driver_id;
499 dev_info->feature_flags = dev->feature_flags;
500 dev_info->capabilities = mrvl_crypto_pmd_capabilities;
501 dev_info->max_nb_queue_pairs = internals->max_nb_qpairs;
502 dev_info->sym.max_nb_sessions = internals->max_nb_sessions;
507 * Release queue pair (PMD ops callback).
509 * @param dev Pointer to the device structure.
510 * @param qp_id ID of Queue Pair to release.
511 * @returns 0. Always.
514 mrvl_crypto_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
516 struct mrvl_crypto_qp *qp =
517 (struct mrvl_crypto_qp *)dev->data->queue_pairs[qp_id];
519 if (dev->data->queue_pairs[qp_id] != NULL) {
520 sam_cio_flush(qp->cio);
521 sam_cio_deinit(qp->cio);
522 rte_free(dev->data->queue_pairs[qp_id]);
523 dev->data->queue_pairs[qp_id] = NULL;
530 * Close device (PMD ops callback).
532 * @param dev Pointer to the device structure.
533 * @returns 0. Always.
536 mrvl_crypto_pmd_close(struct rte_cryptodev *dev)
540 for (qp_id = 0; qp_id < dev->data->nb_queue_pairs; qp_id++)
541 mrvl_crypto_pmd_qp_release(dev, qp_id);
547 * Setup a queue pair (PMD ops callback).
549 * @param dev Pointer to the device structure.
550 * @param qp_id ID of the Queue Pair.
551 * @param qp_conf Queue pair configuration (nb of descriptors).
552 * @param socket_id NUMA socket to allocate memory on.
553 * @returns 0 upon success, negative value otherwise.
556 mrvl_crypto_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
557 const struct rte_cryptodev_qp_conf *qp_conf,
558 int socket_id, struct rte_mempool *session_pool)
560 struct mrvl_crypto_qp *qp = NULL;
561 char match[RTE_CRYPTODEV_NAME_MAX_LEN];
564 /* Allocate the queue pair data structure. */
565 qp = rte_zmalloc_socket("MRVL Crypto PMD Queue Pair", sizeof(*qp),
566 RTE_CACHE_LINE_SIZE, socket_id);
570 /* Free old qp prior setup if needed. */
571 if (dev->data->queue_pairs[qp_id] != NULL)
572 mrvl_crypto_pmd_qp_release(dev, qp_id);
574 do { /* Error handling block */
577 * This extra check is necessary due to a bug in
580 int num = sam_get_num_inst();
582 MRVL_CRYPTO_LOG_ERR("No crypto engines detected.\n");
587 * In case two crypto engines are enabled qps will
588 * be evenly spread among them. Even and odd qps will
589 * be handled by cio-0 and cio-1 respectively. qp-cio mapping
590 * will look as follows:
593 * cio-x:y: cio-0:0, cio-1:0, cio-0:1, cio-1:1
596 * cio-x:y: cio-0:2, cio-1:2, cio-0:3, cio-1:3
598 * In case just one engine is enabled mapping will look as
601 * cio-x:y: cio-0:0, cio-0:1, cio-0:2, cio-0:3
603 n = snprintf(match, sizeof(match), "cio-%u:%u",
604 qp_id % num, qp_id / num);
606 if (n >= sizeof(match))
609 qp->cio_params.match = match;
610 qp->cio_params.size = qp_conf->nb_descriptors;
612 if (sam_cio_init(&qp->cio_params, &qp->cio) < 0)
615 qp->sess_mp = session_pool;
617 memset(&qp->stats, 0, sizeof(qp->stats));
618 dev->data->queue_pairs[qp_id] = qp;
626 /** Start queue pair (PMD ops callback) - not supported.
628 * @param dev Pointer to the device structure.
629 * @param qp_id ID of the Queue Pair.
630 * @returns -ENOTSUP. Always.
633 mrvl_crypto_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
634 __rte_unused uint16_t queue_pair_id)
639 /** Stop queue pair (PMD ops callback) - not supported.
641 * @param dev Pointer to the device structure.
642 * @param qp_id ID of the Queue Pair.
643 * @returns -ENOTSUP. Always.
646 mrvl_crypto_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
647 __rte_unused uint16_t queue_pair_id)
652 /** Return the number of allocated queue pairs (PMD ops callback).
654 * @param dev Pointer to the device structure.
655 * @returns Number of allocated queue pairs.
658 mrvl_crypto_pmd_qp_count(struct rte_cryptodev *dev)
660 return dev->data->nb_queue_pairs;
663 /** Returns the size of the session structure (PMD ops callback).
665 * @param dev Pointer to the device structure [Unused].
666 * @returns Size of Marvell crypto session.
669 mrvl_crypto_pmd_session_get_size(__rte_unused struct rte_cryptodev *dev)
671 return sizeof(struct mrvl_crypto_session);
674 /** Configure the session from a crypto xform chain (PMD ops callback).
676 * @param dev Pointer to the device structure.
677 * @param xform Pointer to the crytpo configuration structure.
678 * @param sess Pointer to the empty session structure.
679 * @returns 0 upon success, negative value otherwise.
682 mrvl_crypto_pmd_session_configure(__rte_unused struct rte_cryptodev *dev,
683 struct rte_crypto_sym_xform *xform,
684 struct rte_cryptodev_sym_session *sess,
685 struct rte_mempool *mp)
687 struct mrvl_crypto_session *mrvl_sess;
688 void *sess_private_data;
692 MRVL_CRYPTO_LOG_ERR("Invalid session struct.");
696 if (rte_mempool_get(mp, &sess_private_data)) {
697 CDEV_LOG_ERR("Couldn't get object from session mempool.");
701 ret = mrvl_crypto_set_session_parameters(sess_private_data, xform);
703 MRVL_CRYPTO_LOG_ERR("Failed to configure session parameters.");
705 /* Return session to mempool */
706 rte_mempool_put(mp, sess_private_data);
710 set_session_private_data(sess, dev->driver_id, sess_private_data);
712 mrvl_sess = (struct mrvl_crypto_session *)sess_private_data;
713 if (sam_session_create(&mrvl_sess->sam_sess_params,
714 &mrvl_sess->sam_sess) < 0) {
715 MRVL_CRYPTO_LOG_DBG("Failed to create session!");
723 * Clear the memory of session so it doesn't leave key material behind.
725 * @param dev Pointer to the device structure.
726 * @returns 0. Always.
729 mrvl_crypto_pmd_session_clear(struct rte_cryptodev *dev,
730 struct rte_cryptodev_sym_session *sess)
733 uint8_t index = dev->driver_id;
734 void *sess_priv = get_session_private_data(sess, index);
736 /* Zero out the whole structure */
738 struct mrvl_crypto_session *mrvl_sess =
739 (struct mrvl_crypto_session *)sess_priv;
741 if (mrvl_sess->sam_sess &&
742 sam_session_destroy(mrvl_sess->sam_sess) < 0) {
743 MRVL_CRYPTO_LOG_INFO("Error while destroying session!");
746 memset(sess, 0, sizeof(struct mrvl_crypto_session));
747 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
748 set_session_private_data(sess, index, NULL);
749 rte_mempool_put(sess_mp, sess_priv);
754 * PMD handlers for crypto ops.
756 static struct rte_cryptodev_ops mrvl_crypto_pmd_ops = {
757 .dev_configure = mrvl_crypto_pmd_config,
758 .dev_start = mrvl_crypto_pmd_start,
759 .dev_stop = mrvl_crypto_pmd_stop,
760 .dev_close = mrvl_crypto_pmd_close,
762 .dev_infos_get = mrvl_crypto_pmd_info_get,
764 .stats_get = mrvl_crypto_pmd_stats_get,
765 .stats_reset = mrvl_crypto_pmd_stats_reset,
767 .queue_pair_setup = mrvl_crypto_pmd_qp_setup,
768 .queue_pair_release = mrvl_crypto_pmd_qp_release,
769 .queue_pair_start = mrvl_crypto_pmd_qp_start,
770 .queue_pair_stop = mrvl_crypto_pmd_qp_stop,
771 .queue_pair_count = mrvl_crypto_pmd_qp_count,
773 .session_get_size = mrvl_crypto_pmd_session_get_size,
774 .session_configure = mrvl_crypto_pmd_session_configure,
775 .session_clear = mrvl_crypto_pmd_session_clear
778 struct rte_cryptodev_ops *rte_mrvl_crypto_pmd_ops = &mrvl_crypto_pmd_ops;