From: Slawomir Mrozowicz Date: Wed, 5 Jul 2017 05:26:17 +0000 (+0100) Subject: cryptodev: support device independent sessions X-Git-Tag: spdx-start~2649 X-Git-Url: http://git.droids-corp.org/?p=dpdk.git;a=commitdiff_plain;h=b3bbd9e5f265937164ed8c9c61d12f7543a745ea cryptodev: support device independent sessions Change crypto device's session management to make it device independent and simplify architecture when session is intended to be used on more than one device. Sessions private data is agnostic to underlying device by adding an indirection in the sessions private data using the crypto driver identifier. A single session can contain indirections to multiple device types. New function rte_cryptodev_sym_session_init has been created, to initialize the driver private session data per driver to be used on a same session, and rte_cryptodev_sym_session_clear to clear this data before calling rte_cryptodev_sym_session_free. Signed-off-by: Slawomir Mrozowicz Signed-off-by: Pablo de Lara Acked-by: Declan Doherty Acked-by: Akhil Goyal --- diff --git a/app/test-crypto-perf/cperf.h b/app/test-crypto-perf/cperf.h index 293ba9406e..c9f7f81749 100644 --- a/app/test-crypto-perf/cperf.h +++ b/app/test-crypto-perf/cperf.h @@ -41,7 +41,10 @@ struct cperf_options; struct cperf_test_vector; struct cperf_op_fns; -typedef void *(*cperf_constructor_t)(uint8_t dev_id, uint16_t qp_id, +typedef void *(*cperf_constructor_t)( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *t_vec, const struct cperf_op_fns *op_fns); diff --git a/app/test-crypto-perf/cperf_ops.c b/app/test-crypto-perf/cperf_ops.c index d7182784a3..a5bf09be94 100644 --- a/app/test-crypto-perf/cperf_ops.c +++ b/app/test-crypto-perf/cperf_ops.c @@ -367,7 +367,8 @@ cperf_set_ops_aead(struct rte_crypto_op **ops, } static struct rte_cryptodev_sym_session * -cperf_create_session(uint8_t dev_id, +cperf_create_session(struct rte_mempool *sess_mp, + uint8_t dev_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, uint16_t iv_offset) @@ -377,6 +378,7 @@ cperf_create_session(uint8_t dev_id, struct rte_crypto_sym_xform aead_xform; struct rte_cryptodev_sym_session *sess = NULL; + sess = rte_cryptodev_sym_session_create(sess_mp); /* * cipher only */ @@ -401,7 +403,8 @@ cperf_create_session(uint8_t dev_id, cipher_xform.cipher.iv.length = 0; } /* create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + rte_cryptodev_sym_session_init(dev_id, sess, &cipher_xform, + sess_mp); /* * auth only */ @@ -427,7 +430,8 @@ cperf_create_session(uint8_t dev_id, auth_xform.auth.iv.length = 0; } /* create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, &auth_xform); + rte_cryptodev_sym_session_init(dev_id, sess, &auth_xform, + sess_mp); /* * cipher and auth */ @@ -483,13 +487,13 @@ cperf_create_session(uint8_t dev_id, if (options->op_type == CPERF_CIPHER_THEN_AUTH) { cipher_xform.next = &auth_xform; /* create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, - &cipher_xform); + rte_cryptodev_sym_session_init(dev_id, + sess, &cipher_xform, sess_mp); } else { /* auth then cipher */ auth_xform.next = &cipher_xform; /* create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, - &auth_xform); + rte_cryptodev_sym_session_init(dev_id, + sess, &auth_xform, sess_mp); } } else { /* options->op_type == CPERF_AEAD */ aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD; @@ -509,7 +513,8 @@ cperf_create_session(uint8_t dev_id, options->aead_aad_sz; /* Create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, &aead_xform); + rte_cryptodev_sym_session_init(dev_id, + sess, &aead_xform, sess_mp); } return sess; diff --git a/app/test-crypto-perf/cperf_ops.h b/app/test-crypto-perf/cperf_ops.h index bb83cd5b26..1f8fa9370c 100644 --- a/app/test-crypto-perf/cperf_ops.h +++ b/app/test-crypto-perf/cperf_ops.h @@ -41,6 +41,7 @@ typedef struct rte_cryptodev_sym_session *(*cperf_sessions_create_t)( + struct rte_mempool *sess_mp, uint8_t dev_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, uint16_t iv_offset); diff --git a/app/test-crypto-perf/cperf_test_latency.c b/app/test-crypto-perf/cperf_test_latency.c index 8841d396a0..1a42602fe7 100644 --- a/app/test-crypto-perf/cperf_test_latency.c +++ b/app/test-crypto-perf/cperf_test_latency.c @@ -79,8 +79,10 @@ cperf_latency_test_free(struct cperf_latency_ctx *ctx, uint32_t mbuf_nb) uint32_t i; if (ctx) { - if (ctx->sess) - rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } if (ctx->mbufs_in) { for (i = 0; i < mbuf_nb; i++) @@ -191,7 +193,8 @@ error: } void * -cperf_latency_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_latency_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *op_fns) @@ -216,7 +219,8 @@ cperf_latency_test_constructor(uint8_t dev_id, uint16_t qp_id, sizeof(struct rte_crypto_sym_op) + sizeof(struct cperf_op_result *); - ctx->sess = op_fns->sess_create(dev_id, options, test_vector, iv_offset); + ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, test_vector, + iv_offset); if (ctx->sess == NULL) goto err; diff --git a/app/test-crypto-perf/cperf_test_latency.h b/app/test-crypto-perf/cperf_test_latency.h index 6a2cf610da..1bbedb4ea0 100644 --- a/app/test-crypto-perf/cperf_test_latency.h +++ b/app/test-crypto-perf/cperf_test_latency.h @@ -43,7 +43,10 @@ #include "cperf_test_vectors.h" void * -cperf_latency_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_latency_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *ops_fn); diff --git a/app/test-crypto-perf/cperf_test_throughput.c b/app/test-crypto-perf/cperf_test_throughput.c index 87fac0fb70..30250446cd 100644 --- a/app/test-crypto-perf/cperf_test_throughput.c +++ b/app/test-crypto-perf/cperf_test_throughput.c @@ -64,8 +64,10 @@ cperf_throughput_test_free(struct cperf_throughput_ctx *ctx, uint32_t mbuf_nb) uint32_t i; if (ctx) { - if (ctx->sess) - rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } if (ctx->mbufs_in) { for (i = 0; i < mbuf_nb; i++) @@ -175,7 +177,8 @@ error: } void * -cperf_throughput_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_throughput_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *op_fns) @@ -199,7 +202,8 @@ cperf_throughput_test_constructor(uint8_t dev_id, uint16_t qp_id, uint16_t iv_offset = sizeof(struct rte_crypto_op) + sizeof(struct rte_crypto_sym_op); - ctx->sess = op_fns->sess_create(dev_id, options, test_vector, iv_offset); + ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, test_vector, + iv_offset); if (ctx->sess == NULL) goto err; diff --git a/app/test-crypto-perf/cperf_test_throughput.h b/app/test-crypto-perf/cperf_test_throughput.h index f1b5766cd3..987d0c31e9 100644 --- a/app/test-crypto-perf/cperf_test_throughput.h +++ b/app/test-crypto-perf/cperf_test_throughput.h @@ -44,7 +44,10 @@ void * -cperf_throughput_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_throughput_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *ops_fn); diff --git a/app/test-crypto-perf/cperf_test_verify.c b/app/test-crypto-perf/cperf_test_verify.c index 3ee056009d..9c37954f74 100644 --- a/app/test-crypto-perf/cperf_test_verify.c +++ b/app/test-crypto-perf/cperf_test_verify.c @@ -68,8 +68,10 @@ cperf_verify_test_free(struct cperf_verify_ctx *ctx, uint32_t mbuf_nb) uint32_t i; if (ctx) { - if (ctx->sess) - rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } if (ctx->mbufs_in) { for (i = 0; i < mbuf_nb; i++) @@ -179,7 +181,8 @@ error: } void * -cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_verify_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *op_fns) @@ -203,7 +206,8 @@ cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id, uint16_t iv_offset = sizeof(struct rte_crypto_op) + sizeof(struct rte_crypto_sym_op); - ctx->sess = op_fns->sess_create(dev_id, options, test_vector, iv_offset); + ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, test_vector, + iv_offset); if (ctx->sess == NULL) goto err; diff --git a/app/test-crypto-perf/cperf_test_verify.h b/app/test-crypto-perf/cperf_test_verify.h index 3fa78ee6ae..e67b48d34d 100644 --- a/app/test-crypto-perf/cperf_test_verify.h +++ b/app/test-crypto-perf/cperf_test_verify.h @@ -44,7 +44,10 @@ void * -cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_verify_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *ops_fn); diff --git a/app/test-crypto-perf/main.c b/app/test-crypto-perf/main.c index cdd3cc83f0..e625c7418d 100644 --- a/app/test-crypto-perf/main.c +++ b/app/test-crypto-perf/main.c @@ -107,13 +107,11 @@ cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs, uint32_t max_sess_size = 0, sess_size; for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { - sess_size = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(cdev_id); + sess_size = rte_cryptodev_get_private_session_size(cdev_id); if (sess_size > max_sess_size) max_sess_size = sess_size; } - for (i = 0; i < enabled_cdev_count && i < RTE_CRYPTO_MAX_DEVS; i++) { cdev_id = enabled_cdevs[i]; @@ -475,7 +473,10 @@ main(int argc, char **argv) cdev_id = enabled_cdevs[i]; - ctx[cdev_id] = cperf_testmap[opts.test].constructor(cdev_id, 0, + uint8_t socket_id = rte_cryptodev_socket_id(cdev_id); + + ctx[cdev_id] = cperf_testmap[opts.test].constructor( + session_pool_socket[socket_id], cdev_id, 0, &opts, t_vec, &op_fns); if (ctx[cdev_id] == NULL) { RTE_LOG(ERR, USER1, "Test run constructor failed\n"); diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst index d8dca78ba0..68a1ccfd11 100644 --- a/doc/guides/rel_notes/deprecation.rst +++ b/doc/guides/rel_notes/deprecation.rst @@ -54,17 +54,6 @@ Deprecation Notices Target release for removal of the legacy API will be defined once most PMDs have switched to rte_flow. -* cryptodev: API changes are planned for 17.08 for the sessions management - to make it agnostic to the underlying devices, removing coupling with - crypto PMDs, so a single session can be used on multiple devices. - - An API of followed functions will be changed to allow operate on multiple - devices with one session: - - - ``rte_cryptodev_sym_session_create`` - - ``rte_cryptodev_sym_session_free`` - - ``rte_cryptodev_sym_session_pool_create`` - * librte_table: The ``key_mask`` parameter will be added to all the hash tables that currently do not have it, as well as to the hash compute function prototype. The non-"do-sig" versions of the hash tables will be removed diff --git a/doc/guides/rel_notes/release_17_08.rst b/doc/guides/rel_notes/release_17_08.rst index 0842edd920..de72b1c957 100644 --- a/doc/guides/rel_notes/release_17_08.rst +++ b/doc/guides/rel_notes/release_17_08.rst @@ -116,6 +116,8 @@ New Features * ``dev_id`` field has been removed. * ``driver_id`` field has been removed. * Mempool pointer ``mp`` has been removed. + * Replaced ``private`` marker with array of pointers to private data sessions + ``sess_private_data``. * **Updated cryptodev library.** @@ -123,6 +125,10 @@ New Features necessary to use a combination of cipher and authentication structures anymore. * Added helper functions for crypto device driver identification. + * Added support for multi-device sessions, so a single session can be + used in multiple drivers. + * Added functions to initialize and free individual driver private data + with a same session. * **Updated dpaa2_sec crypto PMD.** @@ -231,6 +237,9 @@ API Changes * ``rte_cryptodev_queue_pair_attach_sym_session()`` and ``rte_cryptodev_queue_pair_dettach_sym_session()`` functions require the new parameter ``device id``. + * Modified parameters of ``rte_cryptodev_sym_session_create()``, to accept + ``mempool``, instead of ``device id`` and ``rte_crypto_sym_xform``. + * Remove ``device id`` parameter from ``rte_cryptodev_sym_session_free()``. ABI Changes diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c index d226c2348a..91c3801c2b 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c @@ -155,22 +155,37 @@ aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_op *op) struct rte_crypto_sym_op *sym_op = op->sym; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - sess = (struct aesni_gcm_session *)sym_op->session->_private; + if (likely(sym_op->session != NULL)) + sess = (struct aesni_gcm_session *) + get_session_private_data( + sym_op->session, + cryptodev_driver_id); } else { void *_sess; + void *_sess_private_data = NULL; - if (rte_mempool_get(qp->sess_mp, &_sess)) - return sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - sess = (struct aesni_gcm_session *) - ((struct rte_cryptodev_sym_session *)_sess)->_private; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; - if (unlikely(aesni_gcm_set_session_parameters(qp->ops, sess, - sym_op->xform) != 0)) { + sess = (struct aesni_gcm_session *)_sess_private_data; + + if (unlikely(aesni_gcm_set_session_parameters(qp->ops, + sess, sym_op->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } + + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -370,13 +385,11 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, * - Returns NULL on invalid job */ static void -post_process_gcm_crypto_op(struct rte_crypto_op *op) +post_process_gcm_crypto_op(struct rte_crypto_op *op, + struct aesni_gcm_session *session) { struct rte_mbuf *m = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; - struct aesni_gcm_session *session = - (struct aesni_gcm_session *)op->sym->session->_private; - op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Verify digest if required */ @@ -411,6 +424,7 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) * Process a completed GCM request * * @param qp Queue Pair to process + * @param op Crypto operation * @param job JOB_AES_HMAC job * * @return @@ -418,12 +432,17 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) */ static void handle_completed_gcm_crypto_op(struct aesni_gcm_qp *qp, - struct rte_crypto_op *op) + struct rte_crypto_op *op, + struct aesni_gcm_session *sess) { - post_process_gcm_crypto_op(op); + post_process_gcm_crypto_op(op, sess); /* Free session if a session-less crypto op */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_gcm_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -458,7 +477,7 @@ aesni_gcm_pmd_dequeue_burst(void *queue_pair, break; } - handle_completed_gcm_crypto_op(qp, ops[i]); + handle_completed_gcm_crypto_op(qp, ops[i], sess); } qp->qp_stats.dequeued_count += i; diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c index 85c7485aac..21052cda40 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * Copyright(c) 2016 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -302,32 +302,55 @@ aesni_gcm_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni gcm session from a crypto xform chain */ -static void * -aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) +static int +aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; struct aesni_gcm_private *internals = dev->data->dev_private; if (unlikely(sess == NULL)) { GCM_LOG_ERR("invalid session struct"); - return NULL; + return -1; } + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; + } if (aesni_gcm_set_session_parameters(gcm_ops[internals->vector_mode], - sess, xform) != 0) { + sess_private_data, xform) != 0) { GCM_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct aesni_gcm_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_gcm_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_gcm_pmd_ops = { diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 699779a3ab..3aaa070e2c 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -354,25 +354,38 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) { struct aesni_mb_session *sess = NULL; - if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) - sess = (struct aesni_mb_session *)op->sym->session->_private; - else { + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct aesni_mb_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { void *_sess = NULL; + void *_sess_private_data = NULL; if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct aesni_mb_session *) - ((struct rte_cryptodev_sym_session *)_sess)->_private; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct aesni_mb_session *)_sess_private_data; if (unlikely(aesni_mb_set_session_parameters(qp->op_fns, sess, op->sym->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -512,19 +525,21 @@ verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op) { /** * Process a completed job and return rte_mbuf which job processed * + * @param qp Queue Pair to process * @param job JOB_AES_HMAC job to process * * @return - * - Returns processed mbuf which is trimmed of output digest used in - * verification of supplied digest in the case of a HASH_CIPHER operation + * - Returns processed crypto operation which mbuf is trimmed of output digest + * used in verification of supplied digest. * - Returns NULL on invalid job */ static inline struct rte_crypto_op * post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) { struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data; - - struct aesni_mb_session *sess; + struct aesni_mb_session *sess = get_session_private_data( + op->sym->session, + cryptodev_driver_id); if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ENQUEUED)) { switch (job->status) { @@ -532,9 +547,6 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; if (job->hash_alg != NULL_HASH) { - sess = (struct aesni_mb_session *) - op->sym->session->_private; - if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) verify_digest(job, op); @@ -547,6 +559,10 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) /* Free session if a session-less crypto op */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_mb_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index b5abb6ce2e..ae74ae394b 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -479,36 +479,56 @@ aesni_mb_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni multi-buffer session from a crypto xform chain */ -static void * +static int aesni_mb_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; struct aesni_mb_private *internals = dev->data->dev_private; if (unlikely(sess == NULL)) { MB_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } if (aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], - sess, xform) != 0) { + sess_private_data, xform) != 0) { MB_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_mb_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_mb_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct aesni_mb_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_mb_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_mb_pmd_ops = { diff --git a/drivers/crypto/armv8/rte_armv8_pmd.c b/drivers/crypto/armv8/rte_armv8_pmd.c index 590803a26c..eeeaab7a79 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd.c +++ b/drivers/crypto/armv8/rte_armv8_pmd.c @@ -557,24 +557,32 @@ get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) /* get existing session */ if (likely(op->sym->session != NULL)) { sess = (struct armv8_crypto_session *) - op->sym->session->_private; + get_session_private_data( + op->sym->session, + cryptodev_driver_id); } } else { /* provide internal session */ void *_sess = NULL; + void *_sess_private_data = NULL; - if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { - sess = (struct armv8_crypto_session *) - ((struct rte_cryptodev_sym_session *)_sess) - ->_private; - - if (unlikely(armv8_crypto_set_session_parameters( - sess, op->sym->xform) != 0)) { - rte_mempool_put(qp->sess_mp, _sess); - sess = NULL; - } else - op->sym->session = _sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct armv8_crypto_session *)_sess_private_data; + + if (unlikely(armv8_crypto_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } if (unlikely(sess == NULL)) @@ -704,6 +712,9 @@ process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op, /* Free session if a session-less crypto op */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { memset(sess, 0, sizeof(struct armv8_crypto_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/armv8/rte_armv8_pmd_ops.c b/drivers/crypto/armv8/rte_armv8_pmd_ops.c index 1bce2ea2b7..e14b68c955 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd_ops.c +++ b/drivers/crypto/armv8/rte_armv8_pmd_ops.c @@ -318,33 +318,54 @@ armv8_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure the session from a crypto xform chain */ -static void * -armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) +static int +armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { ARMV8_CRYPTO_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - if (armv8_crypto_set_session_parameters( - sess, xform) != 0) { + if (armv8_crypto_set_session_parameters(sess_private_data, xform) != 0) { ARMV8_CRYPTO_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, - void *sess) +armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); /* Zero out the whole structure */ - if (sess) - memset(sess, 0, sizeof(struct armv8_crypto_session)); + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct armv8_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops armv8_crypto_pmd_ops = { diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c index 6bfa812506..b8ac0f1a9b 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -619,7 +619,9 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, /*Clear the unused FD fields before sending*/ memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); sess = (dpaa2_sec_session *) - (*ops)->sym->session->_private; + get_session_private_data( + (*ops)->sym->session, + cryptodev_driver_id); mb_pool = (*ops)->sym->m_src->pool; bpid = mempool_to_bpid(mb_pool); ret = build_sec_fd(sess, *ops, &fd_arr[loop], bpid); @@ -1498,8 +1500,8 @@ error_out: return -1; } -static void * -dpaa2_sec_session_configure(struct rte_cryptodev *dev, +static int +dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *sess) { dpaa2_sec_session *session = sess; @@ -1508,7 +1510,7 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev, if (unlikely(sess == NULL)) { RTE_LOG(ERR, PMD, "invalid session struct"); - return NULL; + return -1; } /* Default IV length = 0 */ @@ -1544,24 +1546,59 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev, } else { RTE_LOG(ERR, PMD, "Invalid crypto type"); - return NULL; + return -1; + } + + return 0; +} + +static int +dpaa2_sec_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - return session; + if (dpaa2_sec_set_session_parameters(dev, xform, sess_private_data) != 0) { + PMD_DRV_LOG(ERR, "DPAA2 PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -dpaa2_sec_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +dpaa2_sec_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { PMD_INIT_FUNC_TRACE(); - dpaa2_sec_session *s = (dpaa2_sec_session *)sess; + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; - if (s) { + if (sess_priv) { rte_free(s->ctxt); rte_free(s->cipher_key.data); rte_free(s->auth_key.data); memset(sess, 0, sizeof(dpaa2_sec_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); } } diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd.c b/drivers/crypto/kasumi/rte_kasumi_pmd.c index 35afa99c2f..cff40fb55c 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd.c @@ -163,23 +163,40 @@ kasumi_set_session_parameters(struct kasumi_session *sess, static struct kasumi_session * kasumi_get_session(struct kasumi_qp *qp, struct rte_crypto_op *op) { - struct kasumi_session *sess; + struct kasumi_session *sess = NULL; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - sess = (struct kasumi_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_sym_session *c_sess = NULL; + if (likely(op->sym->session != NULL)) + sess = (struct kasumi_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct kasumi_session *)c_sess->_private; + sess = (struct kasumi_session *)_sess_private_data; if (unlikely(kasumi_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -354,6 +371,10 @@ process_ops(struct rte_crypto_op **ops, struct kasumi_session *session, ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct kasumi_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -406,7 +427,8 @@ process_op_bit(struct rte_crypto_op *op, struct kasumi_session *session, /* Free session if a session-less crypto op. */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + memset(op->sym->session, 0, sizeof(struct kasumi_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c index 5cb00247f1..e7bbc2941d 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c @@ -292,33 +292,54 @@ kasumi_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a KASUMI session from a crypto xform chain */ -static void * +static int kasumi_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { KASUMI_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - if (kasumi_set_session_parameters(sess, xform) != 0) { + if (kasumi_set_session_parameters(sess_private_data, xform) != 0) { KASUMI_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -kasumi_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +kasumi_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct kasumi_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct kasumi_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops kasumi_pmd_ops = { diff --git a/drivers/crypto/null/null_crypto_pmd.c b/drivers/crypto/null/null_crypto_pmd.c index 27cb9a2f9c..7f7dee6425 100644 --- a/drivers/crypto/null/null_crypto_pmd.c +++ b/drivers/crypto/null/null_crypto_pmd.c @@ -84,6 +84,14 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, /* set status as successful by default */ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + /* Free session if a session-less crypto op. */ + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, + sizeof(struct null_crypto_session)); + rte_cryptodev_sym_session_free(op->sym->session); + op->sym->session = NULL; + } + /* * if crypto session and operation are valid just enqueue the packet * in the processed ring @@ -94,24 +102,35 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, static struct null_crypto_session * get_session(struct null_crypto_qp *qp, struct rte_crypto_op *op) { - struct null_crypto_session *sess; + struct null_crypto_session *sess = NULL; struct rte_crypto_sym_op *sym_op = op->sym; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - if (unlikely(sym_op->session == NULL)) + if (likely(sym_op->session != NULL)) + sess = (struct null_crypto_session *) + get_session_private_data( + sym_op->session, cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct null_crypto_session *)sym_op->session->_private; - } else { - struct rte_cryptodev_sym_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct null_crypto_session *)c_sess->_private; - - if (null_crypto_set_session_parameters(sess, sym_op->xform) != 0) - return NULL; + sess = (struct null_crypto_session *)_sess_private_data; + + if (unlikely(null_crypto_set_session_parameters(sess, + sym_op->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } return sess; diff --git a/drivers/crypto/null/null_crypto_pmd_ops.c b/drivers/crypto/null/null_crypto_pmd_ops.c index a3f2e4c949..d57644d46b 100644 --- a/drivers/crypto/null/null_crypto_pmd_ops.c +++ b/drivers/crypto/null/null_crypto_pmd_ops.c @@ -299,33 +299,54 @@ null_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a null crypto session from a crypto xform chain */ -static void * +static int null_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mp) { - int retval; + void *sess_private_data; if (unlikely(sess == NULL)) { NULL_CRYPTO_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mp, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - retval = null_crypto_set_session_parameters( - (struct null_crypto_session *)sess, xform); - if (retval != 0) { + + if (null_crypto_set_session_parameters(sess_private_data, xform) != 0) { NULL_CRYPTO_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mp, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -null_crypto_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, - void *sess) +null_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct null_crypto_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct null_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops pmd_ops = { diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c index 6f5937d26b..d943d72bbc 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd.c +++ b/drivers/crypto/openssl/rte_openssl_pmd.c @@ -560,23 +560,31 @@ get_session(struct openssl_qp *qp, struct rte_crypto_op *op) /* get existing session */ if (likely(op->sym->session != NULL)) sess = (struct openssl_session *) - op->sym->session->_private; - } else { + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { /* provide internal session */ void *_sess = NULL; + void *_sess_private_data = NULL; - if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { - sess = (struct openssl_session *) - ((struct rte_cryptodev_sym_session *)_sess) - ->_private; - - if (unlikely(openssl_set_session_parameters( - sess, op->sym->xform) != 0)) { - rte_mempool_put(qp->sess_mp, _sess); - sess = NULL; - } else - op->sym->session = _sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct openssl_session *)_sess_private_data; + + if (unlikely(openssl_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } if (sess == NULL) @@ -1318,6 +1326,9 @@ process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { openssl_reset_session(sess); memset(sess, 0, sizeof(struct openssl_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/drivers/crypto/openssl/rte_openssl_pmd_ops.c index 6093b615ed..1f8a011f67 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd_ops.c +++ b/drivers/crypto/openssl/rte_openssl_pmd_ops.c @@ -668,36 +668,56 @@ openssl_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure the session from a crypto xform chain */ -static void * +static int openssl_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { OPENSSL_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } if (openssl_set_session_parameters( - sess, xform) != 0) { + sess_private_data, xform) != 0) { OPENSSL_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -openssl_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +openssl_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) { - openssl_reset_session(sess); - memset(sess, 0, sizeof(struct openssl_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + openssl_reset_session(sess_priv); + memset(sess_priv, 0, sizeof(struct openssl_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); } } diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index d0638a253d..3e621c5c41 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -216,23 +216,23 @@ static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, struct qat_crypto_op_cookie *qat_op_cookie); -void qat_crypto_sym_clear_session(struct rte_cryptodev *dev, - void *session) +void +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - struct qat_session *sess = session; - phys_addr_t cd_paddr; - PMD_INIT_FUNC_TRACE(); - if (sess) { - if (sess->bpi_ctx) { - bpi_cipher_ctx_free(sess->bpi_ctx); - sess->bpi_ctx = NULL; - } - cd_paddr = sess->cd_paddr; - memset(sess, 0, qat_crypto_sym_get_session_private_size(dev)); - sess->cd_paddr = cd_paddr; - } else - PMD_DRV_LOG(ERR, "NULL session"); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + struct qat_session *s = (struct qat_session *)sess_priv; + + if (sess_priv) { + if (s->bpi_ctx) + bpi_cipher_ctx_free(s->bpi_ctx); + memset(s, 0, qat_crypto_sym_get_session_private_size(dev)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } static int @@ -450,9 +450,37 @@ error_out: return NULL; } - -void * +int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; + } + + if (qat_crypto_set_session_parameters(dev, xform, sess_private_data) != 0) { + PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { struct qat_session *session = session_private; @@ -460,6 +488,10 @@ qat_crypto_sym_configure_session(struct rte_cryptodev *dev, int qat_cmd_id; PMD_INIT_FUNC_TRACE(); + /* Set context descriptor physical address */ + session->cd_paddr = rte_mempool_virt2phy(NULL, session) + + offsetof(struct qat_session, cd); + /* Get requested QAT command id */ qat_cmd_id = qat_get_cmd_id(xform); if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) { @@ -514,10 +546,10 @@ qat_crypto_sym_configure_session(struct rte_cryptodev *dev, goto error_out; } - return session; + return 0; error_out: - return NULL; + return -1; } struct qat_session * @@ -946,7 +978,10 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } else { struct qat_session *sess = (struct qat_session *) - (rx_op->sym->session->_private); + get_session_private_data( + rx_op->sym->session, + cryptodev_qat_driver_id); + if (sess->bpi_ctx) qat_bpicipher_postprocess(sess, rx_op); rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; @@ -1072,7 +1107,14 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, return -EINVAL; } - ctx = (struct qat_session *)op->sym->session->_private; + ctx = (struct qat_session *)get_session_private_data( + op->sym->session, cryptodev_qat_driver_id); + + if (unlikely(ctx == NULL)) { + PMD_DRV_LOG(ERR, "Session was not created for this device"); + return -EINVAL; + } + qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; @@ -1371,17 +1413,6 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t shift) return data - mult; } -void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess) -{ - struct rte_cryptodev_sym_session *sess = sym_sess; - struct qat_session *s = (void *)sess->_private; - - PMD_INIT_FUNC_TRACE(); - s->cd_paddr = rte_mempool_virt2phy(mp, sess) + - offsetof(struct qat_session, cd) + - offsetof(struct rte_cryptodev_sym_session, _private); -} - int qat_dev_config(__rte_unused struct rte_cryptodev *dev, __rte_unused struct rte_cryptodev_config *config) { diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h index 1258b6d452..d9d8887337 100644 --- a/drivers/crypto/qat/qat_crypto.h +++ b/drivers/crypto/qat/qat_crypto.h @@ -114,11 +114,15 @@ qat_pmd_session_mempool_create(struct rte_cryptodev *dev, extern unsigned qat_crypto_sym_get_session_private_size(struct rte_cryptodev *dev); -extern void -qat_crypto_sym_session_init(struct rte_mempool *mempool, void *priv_sess); - -extern void * +extern int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool); + + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private); struct qat_session * @@ -136,8 +140,8 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, extern void -qat_crypto_sym_clear_session(struct rte_cryptodev *dev, void *session); - +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *session); extern uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, diff --git a/drivers/crypto/qat/rte_qat_cryptodev.c b/drivers/crypto/qat/rte_qat_cryptodev.c index 1c5ff77541..9a710e6006 100644 --- a/drivers/crypto/qat/rte_qat_cryptodev.c +++ b/drivers/crypto/qat/rte_qat_cryptodev.c @@ -73,7 +73,6 @@ static struct rte_cryptodev_ops crypto_qat_ops = { /* Crypto related operations */ .session_get_size = qat_crypto_sym_get_session_private_size, .session_configure = qat_crypto_sym_configure_session, - .session_initialize = qat_crypto_sym_session_init, .session_clear = qat_crypto_sym_clear_session }; diff --git a/drivers/crypto/scheduler/scheduler_failover.c b/drivers/crypto/scheduler/scheduler_failover.c index 162a29bb67..2aa13f8e77 100644 --- a/drivers/crypto/scheduler/scheduler_failover.c +++ b/drivers/crypto/scheduler/scheduler_failover.c @@ -49,57 +49,18 @@ struct fo_scheduler_qp_ctx { }; static __rte_always_inline uint16_t -failover_slave_enqueue(struct scheduler_slave *slave, uint8_t slave_idx, +failover_slave_enqueue(struct scheduler_slave *slave, struct rte_crypto_op **ops, uint16_t nb_ops) { uint16_t i, processed_ops; - struct rte_cryptodev_sym_session *sessions[nb_ops]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; for (i = 0; i < nb_ops && i < 4; i++) rte_prefetch0(ops[i]->sym->session); - for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { - rte_prefetch0(ops[i + 4]->sym->session); - rte_prefetch0(ops[i + 5]->sym->session); - rte_prefetch0(ops[i + 6]->sym->session); - rte_prefetch0(ops[i + 7]->sym->session); - - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - - sessions[i] = ops[i]->sym->session; - sessions[i + 1] = ops[i + 1]->sym->session; - sessions[i + 2] = ops[i + 2]->sym->session; - sessions[i + 3] = ops[i + 3]->sym->session; - - ops[i]->sym->session = sess0->sessions[slave_idx]; - ops[i + 1]->sym->session = sess1->sessions[slave_idx]; - ops[i + 2]->sym->session = sess2->sessions[slave_idx]; - ops[i + 3]->sym->session = sess3->sessions[slave_idx]; - } - - for (; i < nb_ops; i++) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sessions[i] = ops[i]->sym->session; - ops[i]->sym->session = sess0->sessions[slave_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, ops, nb_ops); slave->nb_inflight_cops += processed_ops; - if (unlikely(processed_ops < nb_ops)) - for (i = processed_ops; i < nb_ops; i++) - ops[i]->sym->session = sessions[i]; - return processed_ops; } @@ -114,11 +75,11 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) return 0; enqueued_ops = failover_slave_enqueue(&qp_ctx->primary_slave, - PRIMARY_SLAVE_IDX, ops, nb_ops); + ops, nb_ops); if (enqueued_ops < nb_ops) enqueued_ops += failover_slave_enqueue(&qp_ctx->secondary_slave, - SECONDARY_SLAVE_IDX, &ops[enqueued_ops], + &ops[enqueued_ops], nb_ops - enqueued_ops); return enqueued_ops; diff --git a/drivers/crypto/scheduler/scheduler_multicore.c b/drivers/crypto/scheduler/scheduler_multicore.c index 8757552f7c..a568be4f11 100644 --- a/drivers/crypto/scheduler/scheduler_multicore.c +++ b/drivers/crypto/scheduler/scheduler_multicore.c @@ -168,7 +168,6 @@ mc_scheduler_worker(struct rte_cryptodev *dev) struct scheduler_slave *slave; struct rte_crypto_op *enq_ops[MC_SCHED_BUFFER_SIZE]; struct rte_crypto_op *deq_ops[MC_SCHED_BUFFER_SIZE]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; uint16_t processed_ops; uint16_t left_op = 0; uint16_t left_op_idx = 0; @@ -202,44 +201,6 @@ mc_scheduler_worker(struct rte_cryptodev *dev) uint16_t nb_deq_ops = rte_ring_dequeue_burst(enq_ring, (void *)enq_ops, MC_SCHED_BUFFER_SIZE, NULL); if (nb_deq_ops) { - uint16_t i; - - for (i = 0; i < nb_deq_ops && i < 4; i++) - rte_prefetch0(enq_ops[i]->sym->session); - - for (i = 0; (i < (nb_deq_ops - 8)) - && (nb_deq_ops > 8); i += 4) { - sess0 = (struct scheduler_session *) - enq_ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - enq_ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - enq_ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - enq_ops[i+3]->sym->session->_private; - - enq_ops[i]->sym->session = - sess0->sessions[worker_idx]; - enq_ops[i + 1]->sym->session = - sess1->sessions[worker_idx]; - enq_ops[i + 2]->sym->session = - sess2->sessions[worker_idx]; - enq_ops[i + 3]->sym->session = - sess3->sessions[worker_idx]; - - rte_prefetch0(enq_ops[i + 4]->sym->session); - rte_prefetch0(enq_ops[i + 5]->sym->session); - rte_prefetch0(enq_ops[i + 6]->sym->session); - rte_prefetch0(enq_ops[i + 7]->sym->session); - } - - for (; i < nb_deq_ops; i++) { - sess0 = (struct scheduler_session *) - enq_ops[i]->sym->session->_private; - enq_ops[i]->sym->session = - sess0->sessions[worker_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, enq_ops, nb_deq_ops); diff --git a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c index 6b628dfa60..1dd1bc3219 100644 --- a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c +++ b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c @@ -67,7 +67,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) struct scheduler_qp_ctx *qp_ctx = qp; struct psd_scheduler_qp_ctx *psd_qp_ctx = qp_ctx->private_qp_ctx; struct rte_crypto_op *sched_ops[NB_PKT_SIZE_SLAVES][nb_ops]; - struct scheduler_session *sess; uint32_t in_flight_ops[NB_PKT_SIZE_SLAVES] = { psd_qp_ctx->primary_slave.nb_inflight_cops, psd_qp_ctx->secondary_slave.nb_inflight_cops @@ -97,8 +96,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) rte_prefetch0(ops[i + 7]->sym); rte_prefetch0(ops[i + 7]->sym->session); - sess = (struct scheduler_session *) - ops[i]->sym->session->_private; /* job_len is initialized as cipher data length, once * it is 0, equals to auth data length */ @@ -118,11 +115,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; - ops[i]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+1]->sym->session->_private; job_len = ops[i+1]->sym->cipher.data.length; job_len += (ops[i+1]->sym->cipher.data.length == 0) * ops[i+1]->sym->auth.data.length; @@ -135,11 +129,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+1]; - ops[i+1]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+2]->sym->session->_private; job_len = ops[i+2]->sym->cipher.data.length; job_len += (ops[i+2]->sym->cipher.data.length == 0) * ops[i+2]->sym->auth.data.length; @@ -152,12 +143,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+2]; - ops[i+2]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - job_len = ops[i+3]->sym->cipher.data.length; job_len += (ops[i+3]->sym->cipher.data.length == 0) * ops[i+3]->sym->auth.data.length; @@ -170,14 +157,10 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+3]; - ops[i+3]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; } for (; i < nb_ops; i++) { - sess = (struct scheduler_session *) - ops[i]->sym->session->_private; - job_len = ops[i]->sym->cipher.data.length; job_len += (ops[i]->sym->cipher.data.length == 0) * ops[i]->sym->auth.data.length; @@ -190,7 +173,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; - ops[i]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; } diff --git a/drivers/crypto/scheduler/scheduler_pmd_ops.c b/drivers/crypto/scheduler/scheduler_pmd_ops.c index b9d89737c3..c450f6a0a2 100644 --- a/drivers/crypto/scheduler/scheduler_pmd_ops.c +++ b/drivers/crypto/scheduler/scheduler_pmd_ops.c @@ -85,8 +85,10 @@ scheduler_attach_init_slave(struct rte_cryptodev *dev) /** Configure device */ static int scheduler_pmd_config(struct rte_cryptodev *dev, - struct rte_cryptodev_config *config __rte_unused) + struct rte_cryptodev_config *config) { + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; int ret; /* although scheduler_attach_init_slave presents multiple times, @@ -96,6 +98,15 @@ scheduler_pmd_config(struct rte_cryptodev *dev, if (ret < 0) return ret; + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + + ret = rte_cryptodev_configure(slave_dev_id, config, + dev->data->session_pool); + if (ret < 0) + break; + } + return ret; } @@ -474,37 +485,39 @@ scheduler_pmd_qp_count(struct rte_cryptodev *dev) static uint32_t scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) { - return sizeof(struct scheduler_session); + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint8_t i = 0; + uint32_t max_priv_sess_size = 0; + + /* Check what is the maximum private session size for all slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *dev = &rte_cryptodevs[slave_dev_id]; + uint32_t priv_sess_size = (*dev->dev_ops->session_get_size)(dev); + + if (max_priv_sess_size < priv_sess_size) + max_priv_sess_size = priv_sess_size; + } + + return max_priv_sess_size; } static int -config_slave_sess(struct scheduler_ctx *sched_ctx, - struct rte_crypto_sym_xform *xform, - struct scheduler_session *sess, - uint32_t create) +scheduler_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + struct scheduler_ctx *sched_ctx = dev->data->dev_private; uint32_t i; for (i = 0; i < sched_ctx->nb_slaves; i++) { struct scheduler_slave *slave = &sched_ctx->slaves[i]; - if (sess->sessions[i]) { - if (create) - continue; - /* !create */ - sess->sessions[i] = rte_cryptodev_sym_session_free( - slave->dev_id, sess->sessions[i]); - } else { - if (!create) - continue; - /* create */ - sess->sessions[i] = - rte_cryptodev_sym_session_create( - slave->dev_id, xform); - if (!sess->sessions[i]) { - config_slave_sess(sched_ctx, NULL, sess, 0); - return -1; - } + if (rte_cryptodev_sym_session_init(slave->dev_id, sess, + xform, mempool) < 0) { + CS_LOG_ERR("unabled to config sym session"); + return -1; } } @@ -514,27 +527,17 @@ config_slave_sess(struct scheduler_ctx *sched_ctx, /** Clear the memory of session so it doesn't leave key material behind */ static void scheduler_pmd_session_clear(struct rte_cryptodev *dev, - void *sess) + struct rte_cryptodev_sym_session *sess) { struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; - config_slave_sess(sched_ctx, NULL, sess, 0); - - memset(sess, 0, sizeof(struct scheduler_session)); -} - -static void * -scheduler_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) -{ - struct scheduler_ctx *sched_ctx = dev->data->dev_private; + /* Clear private data of slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + struct scheduler_slave *slave = &sched_ctx->slaves[i]; - if (config_slave_sess(sched_ctx, xform, sess, 1) < 0) { - CS_LOG_ERR("unabled to config sym session"); - return NULL; + rte_cryptodev_sym_session_clear(slave->dev_id, sess); } - - return sess; } struct rte_cryptodev_ops scheduler_pmd_ops = { diff --git a/drivers/crypto/scheduler/scheduler_pmd_private.h b/drivers/crypto/scheduler/scheduler_pmd_private.h index b38567cd17..a786d3a4d9 100644 --- a/drivers/crypto/scheduler/scheduler_pmd_private.h +++ b/drivers/crypto/scheduler/scheduler_pmd_private.h @@ -105,10 +105,6 @@ struct scheduler_qp_ctx { uint32_t seqn; } __rte_cache_aligned; -struct scheduler_session { - struct rte_cryptodev_sym_session *sessions[ - RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; -}; extern uint8_t cryptodev_driver_id; diff --git a/drivers/crypto/scheduler/scheduler_roundrobin.c b/drivers/crypto/scheduler/scheduler_roundrobin.c index 0116276462..4a847281ac 100644 --- a/drivers/crypto/scheduler/scheduler_roundrobin.c +++ b/drivers/crypto/scheduler/scheduler_roundrobin.c @@ -52,8 +52,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) uint32_t slave_idx = rr_qp_ctx->last_enq_slave_idx; struct scheduler_slave *slave = &rr_qp_ctx->slaves[slave_idx]; uint16_t i, processed_ops; - struct rte_cryptodev_sym_session *sessions[nb_ops]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; if (unlikely(nb_ops == 0)) return 0; @@ -61,39 +59,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) for (i = 0; i < nb_ops && i < 4; i++) rte_prefetch0(ops[i]->sym->session); - for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - - sessions[i] = ops[i]->sym->session; - sessions[i + 1] = ops[i + 1]->sym->session; - sessions[i + 2] = ops[i + 2]->sym->session; - sessions[i + 3] = ops[i + 3]->sym->session; - - ops[i]->sym->session = sess0->sessions[slave_idx]; - ops[i + 1]->sym->session = sess1->sessions[slave_idx]; - ops[i + 2]->sym->session = sess2->sessions[slave_idx]; - ops[i + 3]->sym->session = sess3->sessions[slave_idx]; - - rte_prefetch0(ops[i + 4]->sym->session); - rte_prefetch0(ops[i + 5]->sym->session); - rte_prefetch0(ops[i + 6]->sym->session); - rte_prefetch0(ops[i + 7]->sym->session); - } - - for (; i < nb_ops; i++) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sessions[i] = ops[i]->sym->session; - ops[i]->sym->session = sess0->sessions[slave_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, ops, nb_ops); @@ -102,12 +67,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) rr_qp_ctx->last_enq_slave_idx += 1; rr_qp_ctx->last_enq_slave_idx %= rr_qp_ctx->nb_slaves; - /* recover session if enqueue is failed */ - if (unlikely(processed_ops < nb_ops)) { - for (i = processed_ops; i < nb_ops; i++) - ops[i]->sym->session = sessions[i]; - } - return processed_ops; } diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd.c b/drivers/crypto/snow3g/rte_snow3g_pmd.c index f28b8d6817..107e1b4247 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd.c @@ -163,23 +163,41 @@ snow3g_set_session_parameters(struct snow3g_session *sess, static struct snow3g_session * snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op) { - struct snow3g_session *sess; + struct snow3g_session *sess = NULL; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - sess = (struct snow3g_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_sym_session *c_sess = NULL; + if (likely(op->sym->session != NULL)) + sess = (struct snow3g_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct snow3g_session *)c_sess->_private; + sess = (struct snow3g_session *)_sess_private_data; if (unlikely(snow3g_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; } @@ -355,6 +373,10 @@ process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct snow3g_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -407,7 +429,8 @@ process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, /* Free session if a session-less crypto op. */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + memset(op->sym->session, 0, sizeof(struct snow3g_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c index e3fa0fa764..3accba5c35 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c @@ -290,33 +290,54 @@ snow3g_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a SNOW 3G session from a crypto xform chain */ -static void * +static int snow3g_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { SNOW3G_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - if (snow3g_set_session_parameters(sess, xform) != 0) { + if (snow3g_set_session_parameters(sess_private_data, xform) != 0) { SNOW3G_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -snow3g_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +snow3g_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct snow3g_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct snow3g_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops snow3g_pmd_ops = { diff --git a/drivers/crypto/zuc/rte_zuc_pmd.c b/drivers/crypto/zuc/rte_zuc_pmd.c index 951f1f455a..664c58ff6d 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd.c +++ b/drivers/crypto/zuc/rte_zuc_pmd.c @@ -162,23 +162,40 @@ zuc_set_session_parameters(struct zuc_session *sess, static struct zuc_session * zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) { - struct zuc_session *sess; + struct zuc_session *sess = NULL; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - sess = (struct zuc_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_sym_session *c_sess = NULL; + if (likely(op->sym->session != NULL)) + sess = (struct zuc_session *)get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct zuc_session *)c_sess->_private; + sess = (struct zuc_session *)_sess_private_data; if (unlikely(zuc_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; } @@ -337,6 +354,10 @@ process_ops(struct rte_crypto_op **ops, struct zuc_session *session, ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct zuc_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } diff --git a/drivers/crypto/zuc/rte_zuc_pmd_ops.c b/drivers/crypto/zuc/rte_zuc_pmd_ops.c index abfa0e26d9..adef3432b8 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd_ops.c +++ b/drivers/crypto/zuc/rte_zuc_pmd_ops.c @@ -290,33 +290,54 @@ zuc_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a ZUC session from a crypto xform chain */ -static void * +static int zuc_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { ZUC_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - if (zuc_set_session_parameters(sess, xform) != 0) { + if (zuc_set_session_parameters(sess_private_data, xform) != 0) { ZUC_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -zuc_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +zuc_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct zuc_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct zuc_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops zuc_pmd_ops = { diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c index 4d6c7ceb67..708eadd199 100644 --- a/examples/ipsec-secgw/ipsec-secgw.c +++ b/examples/ipsec-secgw/ipsec-secgw.c @@ -1242,8 +1242,7 @@ cryptodevs_init(void) uint32_t max_sess_sz = 0, sess_sz; for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { - sess_sz = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(cdev_id); + sess_sz = rte_cryptodev_get_private_session_size(cdev_id); if (sess_sz > max_sess_sz) max_sess_sz = sess_sz; } diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c index f09dce91bf..0afb9d6764 100644 --- a/examples/ipsec-secgw/ipsec.c +++ b/examples/ipsec-secgw/ipsec.c @@ -45,7 +45,7 @@ #include "esp.h" static inline int -create_session(struct ipsec_ctx *ipsec_ctx __rte_unused, struct ipsec_sa *sa) +create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) { struct rte_cryptodev_info cdev_info; unsigned long cdev_id_qp = 0; @@ -72,7 +72,10 @@ create_session(struct ipsec_ctx *ipsec_ctx __rte_unused, struct ipsec_sa *sa) ipsec_ctx->tbl[cdev_id_qp].qp); sa->crypto_session = rte_cryptodev_sym_session_create( - ipsec_ctx->tbl[cdev_id_qp].id, sa->xforms); + ipsec_ctx->session_pool); + rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id, + sa->crypto_session, sa->xforms, + ipsec_ctx->session_pool); rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id, &cdev_info); if (cdev_info.sym.max_nb_sessions_per_qp > 0) { diff --git a/examples/l2fwd-crypto/main.c b/examples/l2fwd-crypto/main.c index 9b24e51aaf..cba29ce19b 100644 --- a/examples/l2fwd-crypto/main.c +++ b/examples/l2fwd-crypto/main.c @@ -649,6 +649,9 @@ static struct rte_cryptodev_sym_session * initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) { struct rte_crypto_sym_xform *first_xform; + struct rte_cryptodev_sym_session *session; + uint8_t socket_id = rte_cryptodev_socket_id(cdev_id); + struct rte_mempool *sess_mp = session_pool_socket[socket_id]; if (options->xform_chain == L2FWD_CRYPTO_AEAD) { first_xform = &options->aead_xform; @@ -664,7 +667,16 @@ initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) first_xform = &options->auth_xform; } - return rte_cryptodev_sym_session_create(cdev_id, first_xform); + session = rte_cryptodev_sym_session_create(sess_mp); + + if (session == NULL) + return NULL; + + if (rte_cryptodev_sym_session_init(cdev_id, session, + first_xform, sess_mp) < 0) + return NULL; + + return session; } static void @@ -1935,8 +1947,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, } for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { - sess_sz = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(cdev_id); + sess_sz = rte_cryptodev_get_private_session_size(cdev_id); if (sess_sz > max_sess_sz) max_sess_sz = sess_sz; } diff --git a/lib/librte_cryptodev/rte_cryptodev.c b/lib/librte_cryptodev/rte_cryptodev.c index dfced857ab..373c05baed 100644 --- a/lib/librte_cryptodev/rte_cryptodev.c +++ b/lib/librte_cryptodev/rte_cryptodev.c @@ -69,6 +69,8 @@ #include "rte_cryptodev.h" #include "rte_cryptodev_pmd.h" +static uint8_t nb_drivers; + struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS]; struct rte_cryptodev *rte_cryptodevs = &rte_crypto_devices[0]; @@ -1080,53 +1082,47 @@ rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev, } -static void -rte_cryptodev_sym_session_init(struct rte_mempool *mp, - const struct rte_cryptodev *dev, - struct rte_cryptodev_sym_session *sess) +int +rte_cryptodev_sym_session_init(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, + struct rte_crypto_sym_xform *xforms, + struct rte_mempool *mp) { - memset(sess, 0, mp->elt_size); + struct rte_cryptodev *dev; + uint8_t index; - if (dev->dev_ops->session_initialize) - (*dev->dev_ops->session_initialize)(mp, sess); -} + dev = rte_cryptodev_pmd_get_dev(dev_id); + if (sess == NULL || xforms == NULL || dev == NULL) + return -1; -struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_create(uint8_t dev_id, - struct rte_crypto_sym_xform *xform) -{ - struct rte_cryptodev *dev; - struct rte_cryptodev_sym_session *sess; - void *_sess; + index = dev->driver_id; - if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) { - CDEV_LOG_ERR("Invalid dev_id=%d", dev_id); - return NULL; + if (sess->sess_private_data[index] == NULL) { + if (dev->dev_ops->session_configure(dev, xforms, sess, mp) < 0) { + CDEV_LOG_ERR( + "dev_id %d failed to configure session details", + dev_id); + return -1; + } } - dev = &rte_crypto_devices[dev_id]; + return 0; +} + +struct rte_cryptodev_sym_session * +rte_cryptodev_sym_session_create(struct rte_mempool *mp) +{ + struct rte_cryptodev_sym_session *sess; /* Allocate a session structure from the session pool */ - if (rte_mempool_get(dev->data->session_pool, &_sess)) { - CDEV_LOG_ERR("Couldn't get object from session mempool"); + if (rte_mempool_get(mp, (void *)&sess)) { + CDEV_LOG_ERR("couldn't get object from session mempool"); return NULL; } - sess = _sess; - - rte_cryptodev_sym_session_init(dev->data->session_pool, dev, - sess); - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_configure, NULL); - if (dev->dev_ops->session_configure(dev, xform, sess->_private) == - NULL) { - CDEV_LOG_ERR("dev_id %d failed to configure session details", - dev_id); - - /* Return session to mempool */ - rte_mempool_put(dev->data->session_pool, _sess); - return NULL; - } + /* Clear device session pointer */ + memset(sess, 0, (sizeof(void *) * nb_drivers)); return sess; } @@ -1146,7 +1142,10 @@ rte_cryptodev_queue_pair_attach_sym_session(uint8_t dev_id, uint16_t qp_id, /* The API is optional, not returning error if driver do not suuport */ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->qp_attach_session, 0); - if (dev->dev_ops->qp_attach_session(dev, qp_id, sess->_private)) { + + void *sess_priv = get_session_private_data(sess, dev->driver_id); + + if (dev->dev_ops->qp_attach_session(dev, qp_id, sess_priv)) { CDEV_LOG_ERR("dev_id %d failed to attach qp: %d with session", dev_id, qp_id); return -EPERM; @@ -1170,7 +1169,10 @@ rte_cryptodev_queue_pair_detach_sym_session(uint8_t dev_id, uint16_t qp_id, /* The API is optional, not returning error if driver do not suuport */ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->qp_detach_session, 0); - if (dev->dev_ops->qp_detach_session(dev, qp_id, sess->_private)) { + + void *sess_priv = get_session_private_data(sess, dev->driver_id); + + if (dev->dev_ops->qp_detach_session(dev, qp_id, sess_priv)) { CDEV_LOG_ERR("dev_id %d failed to detach qp: %d from session", dev_id, qp_id); return -EPERM; @@ -1178,34 +1180,62 @@ rte_cryptodev_queue_pair_detach_sym_session(uint8_t dev_id, uint16_t qp_id, return 0; } -struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_free(uint8_t dev_id, + +int +rte_cryptodev_sym_session_clear(uint8_t dev_id, struct rte_cryptodev_sym_session *sess) { struct rte_cryptodev *dev; - if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) { - CDEV_LOG_ERR("Invalid dev_id=%d", dev_id); - return sess; - } + dev = rte_cryptodev_pmd_get_dev(dev_id); - dev = &rte_crypto_devices[dev_id]; + if (dev == NULL || sess == NULL) + return -EINVAL; + + dev->dev_ops->session_clear(dev, sess); - /* Let device implementation clear session material */ - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_clear, sess); - dev->dev_ops->session_clear(dev, (void *)sess->_private); + return 0; +} + +int +rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess) +{ + uint8_t i; + void *sess_priv; + struct rte_mempool *sess_mp; + + if (sess == NULL) + return -EINVAL; + + /* Check that all device private data has been freed */ + for (i = 0; i < nb_drivers; i++) { + sess_priv = get_session_private_data(sess, i); + if (sess_priv != NULL) + return -EBUSY; + } /* Return session to mempool */ - struct rte_mempool *mp = rte_mempool_from_obj(sess); - rte_mempool_put(mp, (void *)sess); + sess_mp = rte_mempool_from_obj(sess); + rte_mempool_put(sess_mp, sess); - return NULL; + return 0; +} + +unsigned int +rte_cryptodev_get_header_session_size(void) +{ + /* + * Header contains pointers to the private data + * of all registered drivers + */ + return (sizeof(void *) * nb_drivers); } unsigned int rte_cryptodev_get_private_session_size(uint8_t dev_id) { struct rte_cryptodev *dev; + unsigned int header_size = sizeof(void *) * nb_drivers; unsigned int priv_sess_size; if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) @@ -1218,6 +1248,14 @@ rte_cryptodev_get_private_session_size(uint8_t dev_id) priv_sess_size = (*dev->dev_ops->session_get_size)(dev); + /* + * If size is less than session header size, + * return the latter, as this guarantees that + * sessionless operations will work + */ + if (priv_sess_size < header_size) + return header_size; + return priv_sess_size; } @@ -1333,8 +1371,6 @@ struct cryptodev_driver { uint8_t id; }; -static uint8_t nb_drivers; - int rte_cryptodev_driver_id_get(const char *name) { diff --git a/lib/librte_cryptodev/rte_cryptodev.h b/lib/librte_cryptodev/rte_cryptodev.h index 508c672e0b..3ba3efb7e8 100644 --- a/lib/librte_cryptodev/rte_cryptodev.h +++ b/lib/librte_cryptodev/rte_cryptodev.h @@ -883,50 +883,80 @@ rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id, /** Cryptodev symmetric crypto session */ struct rte_cryptodev_sym_session { RTE_STD_C11 - __extension__ char _private[0]; + __extension__ void *sess_private_data[0]; /**< Private session material */ }; /** - * Initialise a session for symmetric cryptographic operations. + * Create symmetric crypto session header (generic with no private data) * - * This function is used by the client to initialize immutable - * parameters of symmetric cryptographic operation. - * To perform the operation the rte_cryptodev_enqueue_burst function is - * used. Each mbuf should contain a reference to the session - * pointer returned from this function contained within it's crypto_op if a - * session-based operation is being provisioned. Memory to contain the session - * information is allocated from within mempool managed by the cryptodev. + * @param mempool Symmetric session mempool to allocate session + * objects from + * @return + * - On success return pointer to sym-session + * - On failure returns NULL + */ +struct rte_cryptodev_sym_session * +rte_cryptodev_sym_session_create(struct rte_mempool *mempool); + +/** + * Frees symmetric crypto session header, after checking that all + * the device private data has been freed, returning it + * to its original mempool. * - * The rte_cryptodev_session_free must be called to free allocated - * memory when the session is no longer required. + * @param sess Session header to be freed. * - * @param dev_id The device identifier. - * @param xform Crypto transform chain. + * @return + * - 0 if successful. + * - -EINVAL if session is NULL. + * - -EBUSY if not all device private data has been freed. + */ +int +rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess); +/** + * Fill out private data for the device id, based on its device type. + * + * @param dev_id ID of device that we want the session to be used on + * @param sess Session where the private data will be attached to + * @param xforms Symmetric crypto transform operations to apply on flow + * processed with this session + * @param mempool Mempool where the private data is allocated. * * @return - * Pointer to the created session or NULL + * - On success, zero. + * - On failure, a negative value. */ -extern struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_create(uint8_t dev_id, - struct rte_crypto_sym_xform *xform); +int +rte_cryptodev_sym_session_init(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, + struct rte_crypto_sym_xform *xforms, + struct rte_mempool *mempool); /** - * Free the memory associated with a previously allocated session. + * Frees private data for the device id, based on its device type, + * returning it to its mempool. * - * @param dev_id The device identifier. - * @param session Session pointer previously allocated by - * *rte_cryptodev_sym_session_create*. + * @param dev_id ID of device that uses the session. + * @param sess Session containing the reference to the private data * * @return - * NULL on successful freeing of session. - * Session pointer on failure to free session. + * - 0 if successful. + * - -EINVAL if device is invalid or session is NULL. */ -extern struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_free(uint8_t dev_id, - struct rte_cryptodev_sym_session *session); +int +rte_cryptodev_sym_session_clear(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess); + +/** + * Get the size of the header session, for all registered drivers. + * + * @return + * Size of the header session. + */ +unsigned int +rte_cryptodev_get_header_session_size(void); /** * Get the size of the private session data for a device. diff --git a/lib/librte_cryptodev/rte_cryptodev_pmd.h b/lib/librte_cryptodev/rte_cryptodev_pmd.h index 5911b838b7..2896171e45 100644 --- a/lib/librte_cryptodev/rte_cryptodev_pmd.h +++ b/lib/librte_cryptodev/rte_cryptodev_pmd.h @@ -285,20 +285,25 @@ typedef void (*cryptodev_sym_initialize_session_t)(struct rte_mempool *mempool, * @param dev Crypto device pointer * @param xform Single or chain of crypto xforms * @param priv_sess Pointer to cryptodev's private session structure + * @param mp Mempool where the private session is allocated * * @return - * - Returns private session structure on success. - * - Returns NULL on failure. + * - Returns 0 if private session structure have been created successfully. + * - Returns -1 on failure. */ -typedef void * (*cryptodev_sym_configure_session_t)(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private); +typedef int (*cryptodev_sym_configure_session_t)(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *session, + struct rte_mempool *mp); /** - * Free Crypto session. - * @param session Cryptodev session structure to free + * Free driver private session data. + * + * @param dev Crypto device pointer + * @param sess Cryptodev session structure */ typedef void (*cryptodev_sym_free_session_t)(struct rte_cryptodev *dev, - void *session_private); + struct rte_cryptodev_sym_session *sess); /** * Optional API for drivers to attach sessions with queue pair. @@ -413,6 +418,19 @@ void rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev, int rte_cryptodev_pmd_create_dev_name(char *name, const char *dev_name_prefix); +static inline void * +get_session_private_data(const struct rte_cryptodev_sym_session *sess, + uint8_t driver_id) { + return sess->sess_private_data[driver_id]; +} + +static inline void +set_session_private_data(struct rte_cryptodev_sym_session *sess, + uint8_t driver_id, void *private_data) +{ + sess->sess_private_data[driver_id] = private_data; +} + #ifdef __cplusplus } #endif diff --git a/lib/librte_cryptodev/rte_cryptodev_version.map b/lib/librte_cryptodev/rte_cryptodev_version.map index d817421824..e9ba88ac5b 100644 --- a/lib/librte_cryptodev/rte_cryptodev_version.map +++ b/lib/librte_cryptodev/rte_cryptodev_version.map @@ -66,10 +66,13 @@ DPDK_17.08 { rte_cryptodev_driver_id_get; rte_cryptodev_driver_name_get; rte_cryptodev_get_aead_algo_enum; + rte_cryptodev_get_header_session_size; rte_cryptodev_get_private_session_size; rte_cryptodev_pci_generic_probe; rte_cryptodev_pci_generic_remove; rte_cryptodev_sym_capability_check_aead; + rte_cryptodev_sym_session_init; + rte_cryptodev_sym_session_clear; rte_cryptodev_vdev_parse_init_params; rte_cryptodev_vdev_pmd_init; rte_crypto_aead_algorithm_strings; diff --git a/test/test/test_cryptodev.c b/test/test/test_cryptodev.c index e5d6c07eba..745f2611ac 100644 --- a/test/test/test_cryptodev.c +++ b/test/test/test_cryptodev.c @@ -68,7 +68,6 @@ struct crypto_testsuite_params { struct rte_mempool *large_mbuf_pool; struct rte_mempool *op_mpool; struct rte_mempool *session_mpool; - struct rte_mempool *slave_session_mpool; struct rte_cryptodev_config conf; struct rte_cryptodev_qp_conf qp_conf; @@ -387,12 +386,15 @@ testsuite_setup(void) ts_params->conf.nb_queue_pairs = info.max_nb_queue_pairs; ts_params->conf.socket_id = SOCKET_ID_ANY; - unsigned int session_size = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(dev_id); + unsigned int session_size = rte_cryptodev_get_private_session_size(dev_id); + /* + * Create mempool with maximum number of sessions * 2, + * to include the session headers + */ ts_params->session_mpool = rte_mempool_create( "test_sess_mp", - info.sym.max_nb_sessions, + info.sym.max_nb_sessions * 2, session_size, 0, 0, NULL, NULL, NULL, NULL, SOCKET_ID_ANY, @@ -439,11 +441,6 @@ testsuite_teardown(void) rte_mempool_free(ts_params->session_mpool); ts_params->session_mpool = NULL; } - - if (ts_params->slave_session_mpool != NULL) { - rte_mempool_free(ts_params->slave_session_mpool); - ts_params->slave_session_mpool = NULL; - } } static int @@ -494,8 +491,9 @@ ut_teardown(void) /* free crypto session structure */ if (ut_params->sess) { - rte_cryptodev_sym_session_free(ts_params->valid_devs[0], + rte_cryptodev_sym_session_clear(ts_params->valid_devs[0], ut_params->sess); + rte_cryptodev_sym_session_free(ut_params->sess); ut_params->sess = NULL; } @@ -1280,10 +1278,13 @@ test_AES_CBC_HMAC_SHA1_encrypt_digest(void) ut_params->auth_xform.auth.key.data = hmac_sha1_key; ut_params->auth_xform.auth.digest_length = DIGEST_BYTE_LENGTH_SHA1; - /* Create crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], - &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate crypto op data structure */ @@ -1496,7 +1497,9 @@ test_AES_cipheronly_mb_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1513,7 +1516,9 @@ test_AES_docsis_mb_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), BLKCIPHER_AES_DOCSIS_TYPE); @@ -1530,7 +1535,9 @@ test_AES_docsis_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_AES_DOCSIS_TYPE); @@ -1547,7 +1554,9 @@ test_DES_docsis_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_DES_DOCSIS_TYPE); @@ -1564,7 +1573,9 @@ test_authonly_mb_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), BLKCIPHER_AUTHONLY_TYPE); @@ -1581,7 +1592,9 @@ test_AES_chain_mb_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1600,7 +1613,9 @@ test_AES_cipheronly_scheduler_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1617,7 +1632,9 @@ test_AES_chain_scheduler_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1634,7 +1651,9 @@ test_authonly_scheduler_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)), BLKCIPHER_AUTHONLY_TYPE); @@ -1653,7 +1672,9 @@ test_AES_chain_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1670,7 +1691,9 @@ test_AES_cipheronly_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1687,7 +1710,9 @@ test_AES_chain_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1704,7 +1729,9 @@ test_AES_cipheronly_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1721,7 +1748,9 @@ test_AES_chain_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1738,7 +1767,9 @@ test_AES_cipheronly_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1755,7 +1786,9 @@ test_authonly_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(RTE_CRYPTODEV_DPAA2_SEC_PMD)), BLKCIPHER_AUTHONLY_TYPE); @@ -1772,7 +1805,9 @@ test_authonly_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_AUTHONLY_TYPE); @@ -1789,7 +1824,9 @@ test_AES_chain_armv8_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_ARMV8_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1809,6 +1846,7 @@ create_wireless_algo_hash_session(uint8_t dev_id, { uint8_t hash_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(hash_key, key, key_len); @@ -1826,8 +1864,11 @@ create_wireless_algo_hash_session(uint8_t dev_id, ut_params->auth_xform.auth.digest_length = auth_len; ut_params->auth_xform.auth.iv.offset = IV_OFFSET; ut_params->auth_xform.auth.iv.length = iv_len; - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); return 0; } @@ -1841,6 +1882,7 @@ create_wireless_algo_cipher_session(uint8_t dev_id, { uint8_t cipher_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(cipher_key, key, key_len); @@ -1859,9 +1901,11 @@ create_wireless_algo_cipher_session(uint8_t dev_id, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session */ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params-> - cipher_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->cipher_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); return 0; } @@ -1940,6 +1984,7 @@ create_wireless_algo_cipher_auth_session(uint8_t dev_id, { uint8_t cipher_auth_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(cipher_auth_key, key, key_len); @@ -1972,8 +2017,11 @@ create_wireless_algo_cipher_auth_session(uint8_t dev_id, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->cipher_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->cipher_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); return 0; @@ -1990,6 +2038,7 @@ create_wireless_cipher_auth_session(uint8_t dev_id, const uint8_t key_len = tdata->key.len; uint8_t cipher_auth_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; const uint8_t *key = tdata->key.data; const uint8_t auth_len = tdata->digest.len; @@ -2027,8 +2076,11 @@ create_wireless_cipher_auth_session(uint8_t dev_id, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->cipher_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->cipher_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); return 0; @@ -2056,6 +2108,7 @@ create_wireless_algo_auth_cipher_session(uint8_t dev_id, { uint8_t auth_cipher_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(auth_cipher_key, key, key_len); @@ -2085,8 +2138,11 @@ create_wireless_algo_auth_cipher_session(uint8_t dev_id, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -4536,7 +4592,9 @@ test_3DES_chain_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_3DES_CHAIN_TYPE); @@ -4553,7 +4611,9 @@ test_DES_cipheronly_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_DES_CIPHERONLY_TYPE); @@ -4570,7 +4630,9 @@ test_DES_docsis_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_DES_DOCSIS_TYPE); @@ -4587,7 +4649,9 @@ test_3DES_chain_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), BLKCIPHER_3DES_CHAIN_TYPE); @@ -4604,7 +4668,9 @@ test_3DES_cipheronly_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), BLKCIPHER_3DES_CIPHERONLY_TYPE); @@ -4621,7 +4687,9 @@ test_3DES_cipheronly_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_3DES_CIPHERONLY_TYPE); @@ -4638,7 +4706,9 @@ test_3DES_chain_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_3DES_CHAIN_TYPE); @@ -4655,7 +4725,9 @@ test_3DES_cipheronly_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_3DES_CIPHERONLY_TYPE); @@ -4675,6 +4747,7 @@ create_gcm_session(uint8_t dev_id, enum rte_crypto_aead_operation op, { uint8_t aead_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(aead_key, key, key_len); @@ -4694,8 +4767,11 @@ create_gcm_session(uint8_t dev_id, enum rte_crypto_aead_operation op, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->aead_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->aead_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -5656,7 +5732,11 @@ static int MD5_HMAC_create_session(struct crypto_testsuite_params *ts_params, ut_params->auth_xform.auth.key.data = key; ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->auth_xform); + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->auth_xform, + ts_params->session_mpool); if (ut_params->sess == NULL) return TEST_FAILED; @@ -5831,9 +5911,13 @@ test_multi_session(void) /* Create multiple crypto sessions*/ for (i = 0; i < dev_info.sym.max_nb_sessions; i++) { + sessions[i] = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], - &ut_params->auth_xform); + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + sessions[i], &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(sessions[i], "Session creation failed at session number %u", i); @@ -5869,14 +5953,17 @@ test_multi_session(void) } /* Next session create should fail */ - sessions[i] = rte_cryptodev_sym_session_create(ts_params->valid_devs[0], - &ut_params->auth_xform); + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + sessions[i], &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NULL(sessions[i], "Session creation succeeded unexpectedly!"); - for (i = 0; i < dev_info.sym.max_nb_sessions; i++) - rte_cryptodev_sym_session_free(ts_params->valid_devs[0], + for (i = 0; i < dev_info.sym.max_nb_sessions; i++) { + rte_cryptodev_sym_session_clear(ts_params->valid_devs[0], sessions[i]); + rte_cryptodev_sym_session_free(sessions[i]); + } rte_free(sessions); @@ -5934,6 +6021,9 @@ test_multi_session_random_usage(void) * dev_info.sym.max_nb_sessions) + 1, 0); for (i = 0; i < MB_SESSION_NUMBER; i++) { + sessions[i] = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + rte_memcpy(&ut_paramz[i].ut_params, &testsuite_params, sizeof(struct crypto_unittest_params)); @@ -5942,9 +6032,11 @@ test_multi_session_random_usage(void) ut_paramz[i].cipher_key, ut_paramz[i].hmac_key); /* Create multiple crypto sessions*/ - sessions[i] = rte_cryptodev_sym_session_create( + rte_cryptodev_sym_session_init( ts_params->valid_devs[0], - &ut_paramz[i].ut_params.auth_xform); + sessions[i], + &ut_paramz[i].ut_params.auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(sessions[i], "Session creation failed at session number %u", @@ -5987,9 +6079,11 @@ test_multi_session_random_usage(void) } } - for (i = 0; i < MB_SESSION_NUMBER; i++) - rte_cryptodev_sym_session_free(ts_params->valid_devs[0], + for (i = 0; i < MB_SESSION_NUMBER; i++) { + rte_cryptodev_sym_session_clear(ts_params->valid_devs[0], sessions[i]); + rte_cryptodev_sym_session_free(sessions[i]); + } rte_free(sessions); @@ -6013,9 +6107,14 @@ test_null_cipher_only_operation(void) ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, + &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate Crypto op data structure */ @@ -6070,9 +6169,13 @@ test_null_auth_only_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->auth_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate Crypto op data structure */ @@ -6126,9 +6229,13 @@ test_null_cipher_auth_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate Crypto op data structure */ @@ -6192,9 +6299,13 @@ test_null_auth_cipher_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate Crypto op data structure */ @@ -6240,6 +6351,7 @@ test_null_invalid_operation(void) { struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; + int ret; /* Setup Cipher Parameters */ ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; @@ -6248,10 +6360,14 @@ test_null_invalid_operation(void) ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); - TEST_ASSERT_NULL(ut_params->sess, + ts_params->session_mpool); + + /* Create Crypto session*/ + ret = rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); + TEST_ASSERT(ret == -1, "Session creation succeeded unexpectedly"); @@ -6262,10 +6378,14 @@ test_null_invalid_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->auth_xform); - TEST_ASSERT_NULL(ut_params->sess, + ts_params->session_mpool); + + /* Create Crypto session*/ + ret = rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->auth_xform, + ts_params->session_mpool); + TEST_ASSERT(ret == -1, "Session creation succeeded unexpectedly"); return TEST_SUCCESS; @@ -6299,9 +6419,13 @@ test_null_burst_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); TEST_ASSERT_EQUAL(rte_crypto_op_bulk_alloc(ts_params->op_mpool, @@ -6414,6 +6538,7 @@ static int create_gmac_session(uint8_t dev_id, { uint8_t auth_key[tdata->key.len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(auth_key, tdata->key.data, tdata->key.len); @@ -6430,8 +6555,12 @@ static int create_gmac_session(uint8_t dev_id, ut_params->auth_xform.auth.iv.length = tdata->iv.len; - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -6791,6 +6920,7 @@ create_auth_session(struct crypto_unittest_params *ut_params, const struct test_crypto_vector *reference, enum rte_crypto_auth_operation auth_op) { + struct crypto_testsuite_params *ts_params = &testsuite_params; uint8_t auth_key[reference->auth_key.len + 1]; memcpy(auth_key, reference->auth_key.data, reference->auth_key.len); @@ -6805,8 +6935,12 @@ create_auth_session(struct crypto_unittest_params *ut_params, ut_params->auth_xform.auth.digest_length = reference->digest.len; /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -6820,6 +6954,7 @@ create_auth_cipher_session(struct crypto_unittest_params *ut_params, enum rte_crypto_auth_operation auth_op, enum rte_crypto_cipher_operation cipher_op) { + struct crypto_testsuite_params *ts_params = &testsuite_params; uint8_t cipher_key[reference->cipher_key.len + 1]; uint8_t auth_key[reference->auth_key.len + 1]; @@ -6853,8 +6988,12 @@ create_auth_cipher_session(struct crypto_unittest_params *ut_params, } /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -7710,30 +7849,32 @@ test_scheduler_attach_slave_op(void) continue; /* - * Create a separate mempool for the slaves, as they need different - * session size and then configure them to store the pointer - * to this mempool + * Create the session mempool again, since now there are new devices + * to use the mempool. */ - unsigned int session_size = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(i); + if (ts_params->session_mpool) { + rte_mempool_free(ts_params->session_mpool); + ts_params->session_mpool = NULL; + } + unsigned int session_size = rte_cryptodev_get_private_session_size(i); - if (ts_params->slave_session_mpool == NULL) { - ts_params->slave_session_mpool = rte_mempool_create( - "test_slave_sess_mp", - info.sym.max_nb_sessions, - session_size, - 0, 0, NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); + /* + * Create mempool with maximum number of sessions * 2, + * to include the session headers + */ + if (ts_params->session_mpool == NULL) { + ts_params->session_mpool = rte_mempool_create( + "test_sess_mp", + info.sym.max_nb_sessions * 2, + session_size, + 0, 0, NULL, NULL, NULL, + NULL, SOCKET_ID_ANY, + 0); - TEST_ASSERT_NOT_NULL(ts_params->slave_session_mpool, + TEST_ASSERT_NOT_NULL(ts_params->session_mpool, "session mempool allocation failed"); } - TEST_ASSERT_SUCCESS(rte_cryptodev_configure(i, - &ts_params->conf, ts_params->slave_session_mpool), - "Failed to configure cryptodev %u with %u qps", - i, ts_params->conf.nb_queue_pairs); - ret = rte_cryptodev_scheduler_slave_attach(sched_id, (uint8_t)i); diff --git a/test/test/test_cryptodev_blockcipher.c b/test/test/test_cryptodev_blockcipher.c index 0b512c2e9d..e822c96c62 100644 --- a/test/test/test_cryptodev_blockcipher.c +++ b/test/test/test_cryptodev_blockcipher.c @@ -53,6 +53,7 @@ static int test_blockcipher_one_case(const struct blockcipher_test_case *t, struct rte_mempool *mbuf_pool, struct rte_mempool *op_mpool, + struct rte_mempool *sess_mpool, uint8_t dev_id, int driver_id, char *test_msg) @@ -65,8 +66,8 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t, struct rte_crypto_sym_xform *init_xform = NULL; struct rte_crypto_sym_op *sym_op = NULL; struct rte_crypto_op *op = NULL; - struct rte_cryptodev_sym_session *sess = NULL; struct rte_cryptodev_info dev_info; + struct rte_cryptodev_sym_session *sess = NULL; int status = TEST_SUCCESS; const struct blockcipher_test_data *tdata = t->test_data; @@ -340,8 +341,10 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t, /* create session for sessioned op */ if (!(t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS)) { - sess = rte_cryptodev_sym_session_create(dev_id, - init_xform); + sess = rte_cryptodev_sym_session_create(sess_mpool); + + rte_cryptodev_sym_session_init(dev_id, sess, init_xform, + sess_mpool); if (!sess) { snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " "FAILED: %s", __LINE__, @@ -561,8 +564,10 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t, error_exit: if (!(t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS)) { - if (sess) - rte_cryptodev_sym_session_free(dev_id, sess); + if (sess) { + rte_cryptodev_sym_session_clear(dev_id, sess); + rte_cryptodev_sym_session_free(sess); + } if (cipher_xform) rte_free(cipher_xform); if (auth_xform) @@ -584,6 +589,7 @@ error_exit: int test_blockcipher_all_tests(struct rte_mempool *mbuf_pool, struct rte_mempool *op_mpool, + struct rte_mempool *sess_mpool, uint8_t dev_id, int driver_id, enum blockcipher_test_type test_type) @@ -675,7 +681,7 @@ test_blockcipher_all_tests(struct rte_mempool *mbuf_pool, continue; status = test_blockcipher_one_case(tc, mbuf_pool, op_mpool, - dev_id, driver_id, test_msg); + sess_mpool, dev_id, driver_id, test_msg); printf(" %u) TestCase %s %s\n", test_index ++, tc->test_descr, test_msg); diff --git a/test/test/test_cryptodev_blockcipher.h b/test/test/test_cryptodev_blockcipher.h index 22fb4201e3..22b8d20612 100644 --- a/test/test/test_cryptodev_blockcipher.h +++ b/test/test/test_cryptodev_blockcipher.h @@ -125,6 +125,7 @@ struct blockcipher_test_data { int test_blockcipher_all_tests(struct rte_mempool *mbuf_pool, struct rte_mempool *op_mpool, + struct rte_mempool *sess_mpool, uint8_t dev_id, int driver_id, enum blockcipher_test_type test_type); diff --git a/test/test/test_cryptodev_perf.c b/test/test/test_cryptodev_perf.c index 9caba87605..e21f5e7af7 100644 --- a/test/test/test_cryptodev_perf.c +++ b/test/test/test_cryptodev_perf.c @@ -108,6 +108,8 @@ struct symmetric_session_attrs { uint32_t digest_len; }; +static struct rte_cryptodev_sym_session *test_crypto_session; + #define ALIGN_POW2_ROUNDUP(num, align) \ (((num) + (align) - 1) & ~((align) - 1)) @@ -156,18 +158,18 @@ struct crypto_unittest_params { uint8_t *digest; }; -static struct rte_cryptodev_sym_session * +static int test_perf_create_snow3g_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int cipher_key_len, enum rte_crypto_auth_algorithm auth_algo); -static struct rte_cryptodev_sym_session * +static int test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int cipher_key_len, enum rte_crypto_auth_algorithm auth_algo, enum rte_crypto_aead_algorithm aead_algo); -static struct rte_cryptodev_sym_session * +static int test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int cipher_key_len, @@ -487,9 +489,11 @@ ut_teardown(void) unsigned i; /* free crypto session structure */ - if (ut_params->sess) - rte_cryptodev_sym_session_free(ts_params->dev_id, + if (ut_params->sess) { + rte_cryptodev_sym_session_clear(ts_params->dev_id, ut_params->sess); + rte_cryptodev_sym_session_free(ut_params->sess); + } /* free crypto operation structure */ if (ut_params->op) @@ -1969,10 +1973,13 @@ test_perf_crypto_qp_vary_burst_size(uint16_t dev_num) ut_params->auth_xform.auth.digest_length = DIGEST_BYTE_LENGTH_SHA256; /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(ts_params->dev_id, - &ut_params->cipher_xform); - TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); + + rte_cryptodev_sym_session_init(ts_params->dev_id, test_crypto_session, + &ut_params->cipher_xform, ts_params->sess_mp); + + TEST_ASSERT_NOT_NULL(test_crypto_session, "Session creation failed"); /* Generate Crypto op data structure(s) */ for (i = 0; i < num_to_submit ; i++) { @@ -1994,7 +2001,7 @@ test_perf_crypto_qp_vary_burst_size(uint16_t dev_num) rte_crypto_op_alloc(ts_params->op_mpool, RTE_CRYPTO_OP_TYPE_SYMMETRIC); - rte_crypto_op_attach_sym_session(op, ut_params->sess); + rte_crypto_op_attach_sym_session(op, test_crypto_session); op->sym->auth.digest.data = ut_params->digest; op->sym->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(m, @@ -2105,9 +2112,12 @@ test_perf_snow3G_optimise_cyclecount(struct perf_test_params *pparams) } /* Create Crypto session*/ - sess = test_perf_create_snow3g_session(ts_params->dev_id, + if (test_perf_create_snow3g_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate Crypto op data structure(s)*/ @@ -2211,7 +2221,10 @@ test_perf_snow3G_optimise_cyclecount(struct perf_test_params *pparams) rte_pktmbuf_free(c_ops[i]->sym->m_src); rte_crypto_op_free(c_ops[i]); } - rte_cryptodev_sym_session_free(ts_params->dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, + sess); + rte_cryptodev_sym_session_free(sess); return TEST_SUCCESS; } @@ -2290,10 +2303,13 @@ test_perf_openssl_optimise_cyclecount(struct perf_test_params *pparams) } /* Create Crypto session*/ - sess = test_perf_create_openssl_session(ts_params->dev_id, + if (test_perf_create_openssl_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, pparams->key_length, pparams->auth_algo, - pparams->aead_algo); + pparams->aead_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate Crypto op data structure(s)*/ @@ -2425,7 +2441,9 @@ test_perf_openssl_optimise_cyclecount(struct perf_test_params *pparams) rte_pktmbuf_free(c_ops[i]->sym->m_src); rte_crypto_op_free(c_ops[i]); } - rte_cryptodev_sym_session_free(ts_params->dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); return TEST_SUCCESS; } @@ -2452,10 +2470,12 @@ test_perf_armv8_optimise_cyclecount(struct perf_test_params *pparams) } /* Create Crypto session*/ - sess = test_perf_create_armv8_session(ts_params->dev_id, + if (test_perf_create_armv8_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); - TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; /* Generate Crypto op data structure(s)*/ for (i = 0; i < num_to_submit ; i++) { @@ -2672,12 +2692,13 @@ static uint8_t snow3g_hash_key[] = { 0x1E, 0x26, 0x98, 0xD2, 0xE2, 0x2A, 0xD5, 0x7E }; -static struct rte_cryptodev_sym_session * +static int test_perf_create_aes_sha_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned cipher_key_len, enum rte_crypto_auth_algorithm auth_algo) { + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform cipher_xform = { 0 }; struct rte_crypto_sym_xform auth_xform = { 0 }; @@ -2701,33 +2722,42 @@ test_perf_create_aes_sha_session(uint8_t dev_id, enum chain_mode chain, auth_xform.auth.digest_length = get_auth_digest_length(auth_algo); } + + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); switch (chain) { case CIPHER_HASH: cipher_xform.next = &auth_xform; auth_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_CIPHER: auth_xform.next = &cipher_xform; cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); case CIPHER_ONLY: cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); default: - return NULL; + return -1; } } #define SNOW3G_CIPHER_IV_LENGTH 16 -static struct rte_cryptodev_sym_session * +static int test_perf_create_snow3g_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned cipher_key_len, enum rte_crypto_auth_algorithm auth_algo) { + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform cipher_xform = {0}; struct rte_crypto_sym_xform auth_xform = {0}; @@ -2755,37 +2785,47 @@ test_perf_create_snow3g_session(uint8_t dev_id, enum chain_mode chain, auth_xform.auth.iv.offset = IV_OFFSET + SNOW3G_CIPHER_IV_LENGTH; auth_xform.auth.iv.length = SNOW3G_CIPHER_IV_LENGTH; + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); switch (chain) { case CIPHER_HASH: cipher_xform.next = &auth_xform; auth_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_CIPHER: auth_xform.next = &cipher_xform; cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); case CIPHER_ONLY: cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_ONLY: auth_xform.next = NULL; /* Create Crypto session */ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); default: - return NULL; + return -1; } } -static struct rte_cryptodev_sym_session * +static int test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int key_len, enum rte_crypto_auth_algorithm auth_algo, enum rte_crypto_aead_algorithm aead_algo) { + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform cipher_xform = { 0 }; struct rte_crypto_sym_xform auth_xform = { 0 }; struct rte_crypto_sym_xform aead_xform = { 0 }; @@ -2809,7 +2849,7 @@ test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, cipher_xform.cipher.iv.length = AES_CIPHER_IV_LENGTH; break; default: - return NULL; + return -1; } cipher_xform.cipher.key.length = key_len; @@ -2824,7 +2864,7 @@ test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, auth_xform.auth.key.data = hmac_sha_key; break; default: - return NULL; + return -1; } auth_xform.auth.key.length = get_auth_key_max_length(auth_algo); @@ -2844,37 +2884,45 @@ test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, aead_xform.aead.digest_length = get_aead_digest_length(aead_algo); break; default: - return NULL; + return -1; } aead_xform.aead.key.length = key_len; } + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); switch (chain) { case CIPHER_HASH: cipher_xform.next = &auth_xform; auth_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_CIPHER: auth_xform.next = &cipher_xform; cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); case AEAD: /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &aead_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &aead_xform, + ts_params->sess_mp); default: - return NULL; + return -1; } } -static struct rte_cryptodev_sym_session * +static int test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int cipher_key_len, enum rte_crypto_auth_algorithm auth_algo) { + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform cipher_xform = { 0 }; struct rte_crypto_sym_xform auth_xform = { 0 }; @@ -2887,7 +2935,7 @@ test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, cipher_xform.cipher.key.data = aes_cbc_128_key; break; default: - return NULL; + return -1; } cipher_xform.cipher.key.length = cipher_key_len; @@ -2901,6 +2949,8 @@ test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, auth_xform.auth.digest_length = get_auth_digest_length(auth_algo); + rte_cryptodev_sym_session_create(ts_params->sess_mp); + switch (chain) { case CIPHER_HASH: cipher_xform.next = &auth_xform; @@ -2908,16 +2958,20 @@ test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, /* Encrypt and hash the result */ cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_CIPHER: auth_xform.next = &cipher_xform; cipher_xform.next = NULL; /* Hash encrypted message and decrypt */ cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); default: - return NULL; + return -1; } } @@ -3167,9 +3221,12 @@ test_perf_aes_sha(uint8_t dev_id, uint16_t queue_id, } /* Create Crypto session*/ - sess = test_perf_create_aes_sha_session(ts_params->dev_id, + if (test_perf_create_aes_sha_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate a burst of crypto operations */ @@ -3264,7 +3321,9 @@ test_perf_aes_sha(uint8_t dev_id, uint16_t queue_id, for (i = 0; i < pparams->burst_size * NUM_MBUF_SETS; i++) rte_pktmbuf_free(mbufs[i]); - rte_cryptodev_sym_session_free(dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); printf("\n"); return TEST_SUCCESS; @@ -3300,9 +3359,12 @@ test_perf_snow3g(uint8_t dev_id, uint16_t queue_id, } /* Create Crypto session*/ - sess = test_perf_create_snow3g_session(ts_params->dev_id, + if (test_perf_create_snow3g_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate a burst of crypto operations */ @@ -3429,7 +3491,9 @@ test_perf_snow3g(uint8_t dev_id, uint16_t queue_id, for (i = 0; i < pparams->burst_size * NUM_MBUF_SETS; i++) rte_pktmbuf_free(mbufs[i]); - rte_cryptodev_sym_session_free(dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); printf("\n"); return TEST_SUCCESS; @@ -3486,10 +3550,13 @@ test_perf_openssl(uint8_t dev_id, uint16_t queue_id, } /* Create Crypto session*/ - sess = test_perf_create_openssl_session(ts_params->dev_id, + if (test_perf_create_openssl_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, pparams->key_length, pparams->auth_algo, - pparams->aead_algo); + pparams->aead_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate a burst of crypto operations */ @@ -3582,7 +3649,9 @@ test_perf_openssl(uint8_t dev_id, uint16_t queue_id, for (i = 0; i < pparams->burst_size * NUM_MBUF_SETS; i++) rte_pktmbuf_free(mbufs[i]); - rte_cryptodev_sym_session_free(dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); printf("\n"); return TEST_SUCCESS; @@ -3617,9 +3686,12 @@ test_perf_armv8(uint8_t dev_id, uint16_t queue_id, } /* Create Crypto session*/ - sess = test_perf_create_armv8_session(ts_params->dev_id, + if (test_perf_create_armv8_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate a burst of crypto operations */ @@ -4177,7 +4249,7 @@ test_perf_aes_cbc_vary_burst_size(void) static struct rte_cryptodev_sym_session * test_perf_create_session(uint8_t dev_id, struct perf_test_params *pparams) { - static struct rte_cryptodev_sym_session *sess; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform aead_xform = { 0 }; uint8_t aead_key[pparams->session_attrs->key_aead_len]; @@ -4197,9 +4269,12 @@ test_perf_create_session(uint8_t dev_id, struct perf_test_params *pparams) aead_xform.aead.add_auth_data_length = pparams->session_attrs->aad_len; aead_xform.aead.digest_length = pparams->session_attrs->digest_len; - sess = rte_cryptodev_sym_session_create(dev_id, &aead_xform); + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); + + rte_cryptodev_sym_session_init(dev_id, test_crypto_session, + &aead_xform, ts_params->sess_mp); - return sess; + return test_crypto_session; } static inline struct rte_crypto_op * @@ -4418,7 +4493,9 @@ perf_AES_GCM(uint8_t dev_id, uint16_t queue_id, for (i = 0; i < burst; i++) rte_pktmbuf_free(mbufs[i]); - rte_cryptodev_sym_session_free(dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); return 0; }