X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Faesni_mb%2Frte_aesni_mb_pmd_ops.c;h=2362f0c3ceade8810193024531ecf8ff2c70de02;hb=f74904ce98e84f48e8f3a96b7ad6b6347c3f44b6;hp=01530523f18268cc6aba8dcc5607a55788e7a7d3;hpb=279e0f57b588dc5d2b876cf355d7de7c215d0563;p=dpdk.git diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index 01530523f1..2362f0c3ce 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -4,11 +4,13 @@ #include +#include #include #include +#include #include -#include "rte_aesni_mb_pmd_private.h" +#include "aesni_mb_pmd_private.h" static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { @@ -25,9 +27,9 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .increment = 1 }, .digest_size = { - .min = 12, - .max = 12, - .increment = 0 + .min = 1, + .max = 16, + .increment = 1 }, .iv_size = { 0 } }, } @@ -42,14 +44,35 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 64, .key_size = { .min = 1, - .max = 64, + .max = 65535, .increment = 1 }, .digest_size = { - .min = 12, - .max = 12, + .min = 1, + .max = 20, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA1 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA1, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, .increment = 0 }, + .digest_size = { + .min = 1, + .max = 20, + .increment = 1 + }, .iv_size = { 0 } }, } }, } @@ -63,14 +86,35 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 64, .key_size = { .min = 1, - .max = 64, + .max = 65535, .increment = 1 }, .digest_size = { - .min = 14, - .max = 14, + .min = 1, + .max = 28, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA224 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA224, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, .increment = 0 }, + .digest_size = { + .min = 1, + .max = 28, + .increment = 1 + }, .iv_size = { 0 } }, } }, } @@ -84,14 +128,35 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 64, .key_size = { .min = 1, - .max = 64, + .max = 65535, .increment = 1 }, .digest_size = { - .min = 16, - .max = 16, + .min = 1, + .max = 32, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA256 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA256, + .block_size = 64, + .key_size = { + .min = 0, + .max = 0, .increment = 0 }, + .digest_size = { + .min = 1, + .max = 32, + .increment = 1 + }, .iv_size = { 0 } }, } }, } @@ -105,14 +170,35 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 128, .key_size = { .min = 1, - .max = 128, + .max = 65535, .increment = 1 }, .digest_size = { - .min = 24, - .max = 24, + .min = 1, + .max = 48, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA384 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA384, + .block_size = 128, + .key_size = { + .min = 0, + .max = 0, .increment = 0 }, + .digest_size = { + .min = 1, + .max = 48, + .increment = 1 + }, .iv_size = { 0 } }, } }, } @@ -126,14 +212,35 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 128, .key_size = { .min = 1, - .max = 128, + .max = 65535, .increment = 1 }, .digest_size = { - .min = 32, - .max = 32, + .min = 1, + .max = 64, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* SHA512 */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_SHA512, + .block_size = 128, + .key_size = { + .min = 0, + .max = 0, .increment = 0 }, + .digest_size = { + .min = 1, + .max = 64, + .increment = 1 + }, .iv_size = { 0 } }, } }, } @@ -208,8 +315,13 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .block_size = 16, .key_size = { .min = 16, +#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3) + .max = 32, + .increment = 16 +#else .max = 16, .increment = 0 +#endif }, .iv_size = { .min = 16, @@ -239,6 +351,26 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { }, } }, } }, + { /* 3DES CBC */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_3DES_CBC, + .block_size = 8, + .key_size = { + .min = 8, + .max = 24, + .increment = 8 + }, + .iv_size = { + .min = 8, + .max = 8, + .increment = 0 + } + }, } + }, } + }, { /* DES DOCSIS BPI */ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, {.sym = { @@ -302,17 +434,121 @@ static const struct rte_cryptodev_capabilities aesni_mb_pmd_capabilities[] = { .increment = 0 }, .digest_size = { + .min = 1, + .max = 16, + .increment = 1 + }, + .iv_size = { 0 } + }, } + }, } + }, + { /* AES GCM */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, + {.aead = { + .algo = RTE_CRYPTO_AEAD_AES_GCM, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, + .max = 16, + .increment = 4 + }, + .aad_size = { + .min = 0, + .max = 65535, + .increment = 1 + }, + .iv_size = { .min = 12, + .max = 12, + .increment = 0 + } + }, } + }, } + }, + { /* AES GMAC (AUTH) */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, + {.auth = { + .algo = RTE_CRYPTO_AUTH_AES_GMAC, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 8 + }, + .digest_size = { + .min = 8, .max = 16, .increment = 4 }, - .iv_size = { 0 } + .iv_size = { + .min = 12, + .max = 12, + .increment = 0 + } }, } }, } }, RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED +static const struct rte_cryptodev_capabilities + aesni_mb_pmd_security_crypto_cap[] = { + { /* AES DOCSIS BPI */ + .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, + {.sym = { + .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, + {.cipher = { + .algo = RTE_CRYPTO_CIPHER_AES_DOCSISBPI, + .block_size = 16, + .key_size = { + .min = 16, + .max = 32, + .increment = 16 + }, + .iv_size = { + .min = 16, + .max = 16, + .increment = 0 + } + }, } + }, } + }, + + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + +static const struct rte_security_capability aesni_mb_pmd_security_cap[] = { + { /* DOCSIS Uplink */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_DOCSIS, + .docsis = { + .direction = RTE_SECURITY_DOCSIS_UPLINK + }, + .crypto_capabilities = aesni_mb_pmd_security_crypto_cap + }, + { /* DOCSIS Downlink */ + .action = RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL, + .protocol = RTE_SECURITY_PROTOCOL_DOCSIS, + .docsis = { + .direction = RTE_SECURITY_DOCSIS_DOWNLINK + }, + .crypto_capabilities = aesni_mb_pmd_security_crypto_cap + }, + { + .action = RTE_SECURITY_ACTION_TYPE_NONE + } +}; +#endif /** Configure device */ static int @@ -387,7 +623,8 @@ aesni_mb_pmd_info_get(struct rte_cryptodev *dev, dev_info->feature_flags = dev->feature_flags; dev_info->capabilities = aesni_mb_pmd_capabilities; dev_info->max_nb_queue_pairs = internals->max_nb_queue_pairs; - dev_info->sym.max_nb_sessions = internals->max_nb_sessions; + /* No limit of number of sessions */ + dev_info->sym.max_nb_sessions = 0; } } @@ -402,6 +639,8 @@ aesni_mb_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) r = rte_ring_lookup(qp->name); if (r) rte_ring_free(r); + if (qp->mb_mgr) + free_mb_mgr(qp->mb_mgr); rte_free(qp); dev->data->queue_pairs[qp_id] = NULL; } @@ -426,14 +665,12 @@ aesni_mb_pmd_qp_set_unique_name(struct rte_cryptodev *dev, /** Create a ring to place processed operations on */ static struct rte_ring * aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp, - const char *str, unsigned int ring_size, int socket_id) + unsigned int ring_size, int socket_id) { struct rte_ring *r; char ring_name[RTE_CRYPTODEV_NAME_MAX_LEN]; - unsigned int n = snprintf(ring_name, sizeof(ring_name), - "%s_%s", - qp->name, str); + unsigned int n = strlcpy(ring_name, qp->name, sizeof(ring_name)); if (n >= sizeof(ring_name)) return NULL; @@ -441,12 +678,12 @@ aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp, r = rte_ring_lookup(ring_name); if (r) { if (rte_ring_get_size(r) >= ring_size) { - MB_LOG_INFO("Reusing existing ring %s for processed ops", + AESNI_MB_LOG(INFO, "Reusing existing ring %s for processed ops", ring_name); return r; } - MB_LOG_ERR("Unable to reuse existing ring %s for processed ops", + AESNI_MB_LOG(ERR, "Unable to reuse existing ring %s for processed ops", ring_name); return NULL; } @@ -459,10 +696,11 @@ aesni_mb_pmd_qp_create_processed_ops_ring(struct aesni_mb_qp *qp, static int aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id, struct rte_mempool *session_pool) + int socket_id) { struct aesni_mb_qp *qp = NULL; struct aesni_mb_private *internals = dev->data->dev_private; + int ret = -1; /* Free memory prior to re-allocation if needed. */ if (dev->data->queue_pairs[qp_id] != NULL) @@ -481,14 +719,44 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, goto qp_setup_cleanup; - qp->op_fns = &job_ops[internals->vector_mode]; + qp->mb_mgr = alloc_mb_mgr(0); + if (qp->mb_mgr == NULL) { + ret = -ENOMEM; + goto qp_setup_cleanup; + } + + switch (internals->vector_mode) { + case RTE_AESNI_MB_SSE: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE; + init_mb_mgr_sse(qp->mb_mgr); + break; + case RTE_AESNI_MB_AVX: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX; + init_mb_mgr_avx(qp->mb_mgr); + break; + case RTE_AESNI_MB_AVX2: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + init_mb_mgr_avx2(qp->mb_mgr); + break; + case RTE_AESNI_MB_AVX512: + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512; + init_mb_mgr_avx512(qp->mb_mgr); + break; + default: + AESNI_MB_LOG(ERR, "Unsupported vector mode %u\n", + internals->vector_mode); + goto qp_setup_cleanup; + } qp->ingress_queue = aesni_mb_pmd_qp_create_processed_ops_ring(qp, - "ingress", qp_conf->nb_descriptors, socket_id); - if (qp->ingress_queue == NULL) + qp_conf->nb_descriptors, socket_id); + if (qp->ingress_queue == NULL) { + ret = -1; goto qp_setup_cleanup; + } - qp->sess_mp = session_pool; + qp->sess_mp = qp_conf->mp_session; + qp->sess_mp_priv = qp_conf->mp_session_private; memset(&qp->stats, 0, sizeof(qp->stats)); @@ -496,51 +764,28 @@ aesni_mb_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, "digest_mp_%u_%u", dev->data->dev_id, qp_id); - - /* Initialise multi-buffer manager */ - (*qp->op_fns->job.init_mgr)(&qp->mb_mgr); return 0; qp_setup_cleanup: - if (qp) + if (qp) { + if (qp->mb_mgr) + free_mb_mgr(qp->mb_mgr); rte_free(qp); + } - return -1; -} - -/** Start queue pair */ -static int -aesni_mb_pmd_qp_start(__rte_unused struct rte_cryptodev *dev, - __rte_unused uint16_t queue_pair_id) -{ - return -ENOTSUP; -} - -/** Stop queue pair */ -static int -aesni_mb_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev, - __rte_unused uint16_t queue_pair_id) -{ - return -ENOTSUP; -} - -/** Return the number of allocated queue pairs */ -static uint32_t -aesni_mb_pmd_qp_count(struct rte_cryptodev *dev) -{ - return dev->data->nb_queue_pairs; + return ret; } /** Returns the size of the aesni multi-buffer session structure */ static unsigned -aesni_mb_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) +aesni_mb_pmd_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) { return sizeof(struct aesni_mb_session); } /** Configure a aesni multi-buffer session from a crypto xform chain */ static int -aesni_mb_pmd_session_configure(struct rte_cryptodev *dev, +aesni_mb_pmd_sym_session_configure(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, struct rte_cryptodev_sym_session *sess, struct rte_mempool *mempool) @@ -550,27 +795,27 @@ aesni_mb_pmd_session_configure(struct rte_cryptodev *dev, int ret; if (unlikely(sess == NULL)) { - MB_LOG_ERR("invalid session struct"); + AESNI_MB_LOG(ERR, "invalid session struct"); return -EINVAL; } if (rte_mempool_get(mempool, &sess_private_data)) { - CDEV_LOG_ERR( - "Couldn't get object from session mempool"); + AESNI_MB_LOG(ERR, + "Couldn't get object from session mempool"); return -ENOMEM; } - ret = aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], + ret = aesni_mb_set_session_parameters(internals->mb_mgr, sess_private_data, xform); if (ret != 0) { - MB_LOG_ERR("failed configure session parameters"); + AESNI_MB_LOG(ERR, "failed configure session parameters"); /* Return session to mempool */ rte_mempool_put(mempool, sess_private_data); return ret; } - set_session_private_data(sess, dev->driver_id, + set_sym_session_private_data(sess, dev->driver_id, sess_private_data); return 0; @@ -578,17 +823,17 @@ aesni_mb_pmd_session_configure(struct rte_cryptodev *dev, /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_mb_pmd_session_clear(struct rte_cryptodev *dev, +aesni_mb_pmd_sym_session_clear(struct rte_cryptodev *dev, struct rte_cryptodev_sym_session *sess) { uint8_t index = dev->driver_id; - void *sess_priv = get_session_private_data(sess, index); + void *sess_priv = get_sym_session_private_data(sess, index); /* Zero out the whole structure */ if (sess_priv) { memset(sess_priv, 0, sizeof(struct aesni_mb_session)); struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); - set_session_private_data(sess, index, NULL); + set_sym_session_private_data(sess, index, NULL); rte_mempool_put(sess_mp, sess_priv); } } @@ -606,13 +851,88 @@ struct rte_cryptodev_ops aesni_mb_pmd_ops = { .queue_pair_setup = aesni_mb_pmd_qp_setup, .queue_pair_release = aesni_mb_pmd_qp_release, - .queue_pair_start = aesni_mb_pmd_qp_start, - .queue_pair_stop = aesni_mb_pmd_qp_stop, - .queue_pair_count = aesni_mb_pmd_qp_count, - .session_get_size = aesni_mb_pmd_session_get_size, - .session_configure = aesni_mb_pmd_session_configure, - .session_clear = aesni_mb_pmd_session_clear + .sym_cpu_process = aesni_mb_cpu_crypto_process_bulk, + + .sym_session_get_size = aesni_mb_pmd_sym_session_get_size, + .sym_session_configure = aesni_mb_pmd_sym_session_configure, + .sym_session_clear = aesni_mb_pmd_sym_session_clear }; struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops = &aesni_mb_pmd_ops; + +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED +/** + * Configure a aesni multi-buffer session from a security session + * configuration + */ +static int +aesni_mb_pmd_sec_sess_create(void *dev, struct rte_security_session_conf *conf, + struct rte_security_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; + int ret; + + if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL || + conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) { + AESNI_MB_LOG(ERR, "Invalid security protocol"); + return -EINVAL; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + AESNI_MB_LOG(ERR, "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = aesni_mb_set_docsis_sec_session_parameters(cdev, conf, + sess_private_data); + + if (ret != 0) { + AESNI_MB_LOG(ERR, "Failed to configure session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_sec_session_private_data(sess, sess_private_data); + + return ret; +} + +/** Clear the memory of session so it doesn't leave key material behind */ +static int +aesni_mb_pmd_sec_sess_destroy(void *dev __rte_unused, + struct rte_security_session *sess) +{ + void *sess_priv = get_sec_session_private_data(sess); + + if (sess_priv) { + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + memset(sess, 0, sizeof(struct aesni_mb_session)); + set_sec_session_private_data(sess, NULL); + rte_mempool_put(sess_mp, sess_priv); + } + return 0; +} + +/** Get security capabilities for aesni multi-buffer */ +static const struct rte_security_capability * +aesni_mb_pmd_sec_capa_get(void *device __rte_unused) +{ + return aesni_mb_pmd_security_cap; +} + +static struct rte_security_ops aesni_mb_pmd_sec_ops = { + .session_create = aesni_mb_pmd_sec_sess_create, + .session_update = NULL, + .session_stats_get = NULL, + .session_destroy = aesni_mb_pmd_sec_sess_destroy, + .set_pkt_metadata = NULL, + .capabilities_get = aesni_mb_pmd_sec_capa_get +}; + +struct rte_security_ops *rte_aesni_mb_pmd_sec_ops = &aesni_mb_pmd_sec_ops; +#endif