* Copyright(c) 2015-2020 Intel Corporation
*/
-#include <sys/types.h>
#include <sys/queue.h>
#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
-#include <stdarg.h>
#include <errno.h>
#include <stdint.h>
#include <inttypes.h>
-#include <rte_byteorder.h>
#include <rte_log.h>
#include <rte_debug.h>
#include <rte_dev.h>
-#include <rte_interrupts.h>
#include <rte_memory.h>
#include <rte_memcpy.h>
#include <rte_memzone.h>
-#include <rte_launch.h>
-#include <rte_tailq.h>
#include <rte_eal.h>
-#include <rte_per_lcore.h>
-#include <rte_lcore.h>
-#include <rte_atomic.h>
-#include <rte_branch_prediction.h>
#include <rte_common.h>
#include <rte_mempool.h>
#include <rte_malloc.h>
-#include <rte_mbuf.h>
#include <rte_errno.h>
#include <rte_spinlock.h>
#include <rte_string_fns.h>
+#include <rte_telemetry.h>
#include "rte_crypto.h"
#include "rte_cryptodev.h"
.nb_devs = 0
};
+/* Public fastpath APIs. */
+struct rte_crypto_fp_ops rte_crypto_fp_ops[RTE_CRYPTO_MAX_DEVS];
+
/* spinlock for crypto device callbacks */
static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
[RTE_CRYPTO_ASYM_OP_ENCRYPT] = "encrypt",
[RTE_CRYPTO_ASYM_OP_DECRYPT] = "decrypt",
[RTE_CRYPTO_ASYM_OP_SIGN] = "sign",
- [RTE_CRYPTO_ASYM_OP_VERIFY] = "verify",
- [RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE] = "priv_key_generate",
- [RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE] = "pub_key_generate",
- [RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE] = "sharedsecret_compute",
+ [RTE_CRYPTO_ASYM_OP_VERIFY] = "verify"
+};
+
+/**
+ * Asymmetric crypto key exchange operation strings identifiers.
+ */
+const char *rte_crypto_asym_ke_strings[] = {
+ [RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE] = "priv_key_generate",
+ [RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE] = "pub_key_generate",
+ [RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE] = "sharedsecret_compute",
+ [RTE_CRYPTO_ASYM_KE_PUB_KEY_VERIFY] = "pub_ec_key_verify"
};
/**
- * The private data structure stored in the session mempool private data.
+ * The private data structure stored in the sym session mempool private data.
*/
struct rte_cryptodev_sym_session_pool_private_data {
uint16_t nb_drivers;
/**< session user data will be placed after sess_data */
};
+/**
+ * The private data structure stored in the asym session mempool private data.
+ */
+struct rte_cryptodev_asym_session_pool_private_data {
+ uint16_t max_priv_session_sz;
+ /**< Size of private session data used when creating mempool */
+ uint16_t user_data_sz;
+ /**< Session user data will be placed after sess_private_data */
+};
+
int
rte_cryptodev_get_cipher_algo_enum(enum rte_crypto_cipher_algorithm *algo_enum,
const char *algo_string)
dev_id = cryptodev->data->dev_id;
+ cryptodev_fp_ops_reset(rte_crypto_fp_ops + dev_id);
+
/* Close device only if device operations have been set */
if (cryptodev->dev_ops) {
ret = rte_cryptodev_close(dev_id);
}
diag = (*dev->dev_ops->dev_start)(dev);
+ /* expose selection of PMD fast-path functions */
+ cryptodev_fp_ops_set(rte_crypto_fp_ops + dev_id, dev);
+
rte_cryptodev_trace_start(dev_id, diag);
if (diag == 0)
dev->data->dev_started = 1;
return;
}
+ /* point fast-path functions to dummy ones */
+ cryptodev_fp_ops_reset(rte_crypto_fp_ops + dev_id);
+
(*dev->dev_ops->dev_stop)(dev);
rte_cryptodev_trace_stop(dev_id);
dev->data->dev_started = 0;
return 0;
}
-int
-rte_cryptodev_asym_session_init(uint8_t dev_id,
- struct rte_cryptodev_asym_session *sess,
- struct rte_crypto_asym_xform *xforms,
- struct rte_mempool *mp)
-{
- struct rte_cryptodev *dev;
- uint8_t index;
- int ret;
-
- if (!rte_cryptodev_is_valid_dev(dev_id)) {
- CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
- return -EINVAL;
- }
-
- dev = rte_cryptodev_pmd_get_dev(dev_id);
-
- if (sess == NULL || xforms == NULL || dev == NULL)
- return -EINVAL;
-
- index = dev->driver_id;
-
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_configure,
- -ENOTSUP);
-
- if (sess->sess_private_data[index] == NULL) {
- ret = dev->dev_ops->asym_session_configure(dev,
- xforms,
- sess, mp);
- if (ret < 0) {
- CDEV_LOG_ERR(
- "dev_id %d failed to configure session details",
- dev_id);
- return ret;
- }
- }
-
- rte_cryptodev_trace_asym_session_init(dev_id, sess, xforms, mp);
- return 0;
-}
-
struct rte_mempool *
rte_cryptodev_sym_session_pool_create(const char *name, uint32_t nb_elts,
uint32_t elt_size, uint32_t cache_size, uint16_t user_data_size,
return mp;
}
+struct rte_mempool *
+rte_cryptodev_asym_session_pool_create(const char *name, uint32_t nb_elts,
+ uint32_t cache_size, uint16_t user_data_size, int socket_id)
+{
+ struct rte_mempool *mp;
+ struct rte_cryptodev_asym_session_pool_private_data *pool_priv;
+ uint32_t obj_sz, obj_sz_aligned;
+ uint8_t dev_id;
+ unsigned int priv_sz, max_priv_sz = 0;
+
+ for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++)
+ if (rte_cryptodev_is_valid_dev(dev_id)) {
+ priv_sz = rte_cryptodev_asym_get_private_session_size(dev_id);
+ if (priv_sz > max_priv_sz)
+ max_priv_sz = priv_sz;
+ }
+ if (max_priv_sz == 0) {
+ CDEV_LOG_INFO("Could not set max private session size\n");
+ return NULL;
+ }
+
+ obj_sz = rte_cryptodev_asym_get_header_session_size() + max_priv_sz +
+ user_data_size;
+ obj_sz_aligned = RTE_ALIGN_CEIL(obj_sz, RTE_CACHE_LINE_SIZE);
+
+ mp = rte_mempool_create(name, nb_elts, obj_sz_aligned, cache_size,
+ (uint32_t)(sizeof(*pool_priv)),
+ NULL, NULL, NULL, NULL,
+ socket_id, 0);
+ if (mp == NULL) {
+ CDEV_LOG_ERR("%s(name=%s) failed, rte_errno=%d\n",
+ __func__, name, rte_errno);
+ return NULL;
+ }
+
+ pool_priv = rte_mempool_get_priv(mp);
+ if (!pool_priv) {
+ CDEV_LOG_ERR("%s(name=%s) failed to get private data\n",
+ __func__, name);
+ rte_mempool_free(mp);
+ return NULL;
+ }
+ pool_priv->max_priv_session_sz = max_priv_sz;
+ pool_priv->user_data_sz = user_data_size;
+
+ rte_cryptodev_trace_asym_session_pool_create(name, nb_elts,
+ user_data_size, cache_size, mp);
+ return mp;
+}
+
static unsigned int
rte_cryptodev_sym_session_data_size(struct rte_cryptodev_sym_session *sess)
{
return sess;
}
-struct rte_cryptodev_asym_session *
-rte_cryptodev_asym_session_create(struct rte_mempool *mp)
+int
+rte_cryptodev_asym_session_create(uint8_t dev_id,
+ struct rte_crypto_asym_xform *xforms, struct rte_mempool *mp,
+ void **session)
{
struct rte_cryptodev_asym_session *sess;
- unsigned int session_size =
+ uint32_t session_priv_data_sz;
+ struct rte_cryptodev_asym_session_pool_private_data *pool_priv;
+ unsigned int session_header_size =
rte_cryptodev_asym_get_header_session_size();
+ struct rte_cryptodev *dev;
+ int ret;
+
+ if (!rte_cryptodev_is_valid_dev(dev_id)) {
+ CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
+ return -EINVAL;
+ }
+
+ dev = rte_cryptodev_pmd_get_dev(dev_id);
+
+ if (dev == NULL)
+ return -EINVAL;
if (!mp) {
CDEV_LOG_ERR("invalid mempool\n");
- return NULL;
+ return -EINVAL;
+ }
+
+ session_priv_data_sz = rte_cryptodev_asym_get_private_session_size(
+ dev_id);
+ pool_priv = rte_mempool_get_priv(mp);
+
+ if (pool_priv->max_priv_session_sz < session_priv_data_sz) {
+ CDEV_LOG_DEBUG(
+ "The private session data size used when creating the mempool is smaller than this device's private session data.");
+ return -EINVAL;
}
/* Verify if provided mempool can hold elements big enough. */
- if (mp->elt_size < session_size) {
+ if (mp->elt_size < session_header_size + session_priv_data_sz) {
CDEV_LOG_ERR(
"mempool elements too small to hold session objects");
- return NULL;
+ return -EINVAL;
}
/* Allocate a session structure from the session pool */
- if (rte_mempool_get(mp, (void **)&sess)) {
+ if (rte_mempool_get(mp, session)) {
CDEV_LOG_ERR("couldn't get object from session mempool");
- return NULL;
+ return -ENOMEM;
}
- /* Clear device session pointer.
- * Include the flag indicating presence of private data
- */
- memset(sess, 0, session_size);
+ sess = *session;
+ sess->driver_id = dev->driver_id;
+ sess->user_data_sz = pool_priv->user_data_sz;
+ sess->max_priv_data_sz = pool_priv->max_priv_session_sz;
- rte_cryptodev_trace_asym_session_create(mp, sess);
- return sess;
+ /* Clear device session pointer.*/
+ memset(sess->sess_private_data, 0, session_priv_data_sz + sess->user_data_sz);
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_configure, -ENOTSUP);
+
+ if (sess->sess_private_data[0] == 0) {
+ ret = dev->dev_ops->asym_session_configure(dev, xforms, sess);
+ if (ret < 0) {
+ CDEV_LOG_ERR(
+ "dev_id %d failed to configure session details",
+ dev_id);
+ return ret;
+ }
+ }
+
+ rte_cryptodev_trace_asym_session_create(dev_id, xforms, mp, sess);
+ return 0;
}
int
return 0;
}
-int
-rte_cryptodev_asym_session_clear(uint8_t dev_id,
- struct rte_cryptodev_asym_session *sess)
-{
- struct rte_cryptodev *dev;
-
- if (!rte_cryptodev_is_valid_dev(dev_id)) {
- CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
- return -EINVAL;
- }
-
- dev = rte_cryptodev_pmd_get_dev(dev_id);
-
- if (dev == NULL || sess == NULL)
- return -EINVAL;
-
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_clear, -ENOTSUP);
-
- dev->dev_ops->asym_session_clear(dev, sess);
-
- rte_cryptodev_trace_sym_session_clear(dev_id, sess);
- return 0;
-}
-
int
rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess)
{
}
int
-rte_cryptodev_asym_session_free(struct rte_cryptodev_asym_session *sess)
+rte_cryptodev_asym_session_free(uint8_t dev_id, void *sess)
{
- uint8_t i;
- void *sess_priv;
struct rte_mempool *sess_mp;
+ struct rte_cryptodev *dev;
- if (sess == NULL)
+ if (!rte_cryptodev_is_valid_dev(dev_id)) {
+ CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
return -EINVAL;
-
- /* Check that all device private data has been freed */
- for (i = 0; i < nb_drivers; i++) {
- sess_priv = get_asym_session_private_data(sess, i);
- if (sess_priv != NULL)
- return -EBUSY;
}
+ dev = rte_cryptodev_pmd_get_dev(dev_id);
+
+ if (dev == NULL || sess == NULL)
+ return -EINVAL;
+
+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->asym_session_clear, -ENOTSUP);
+
+ dev->dev_ops->asym_session_clear(dev, sess);
+
+ rte_free(((struct rte_cryptodev_asym_session *)sess)->event_mdata);
+
/* Return session to mempool */
sess_mp = rte_mempool_from_obj(sess);
rte_mempool_put(sess_mp, sess);
- rte_cryptodev_trace_asym_session_free(sess);
+ rte_cryptodev_trace_asym_session_free(dev_id, sess);
return 0;
}
unsigned int
rte_cryptodev_asym_get_header_session_size(void)
{
- /*
- * Header contains pointers to the private data
- * of all registered drivers, and a flag which
- * indicates presence of private data
- */
- return ((sizeof(void *) * nb_drivers) + sizeof(uint8_t));
+ return sizeof(struct rte_cryptodev_asym_session);
}
unsigned int
rte_cryptodev_asym_get_private_session_size(uint8_t dev_id)
{
struct rte_cryptodev *dev;
- unsigned int header_size = sizeof(void *) * nb_drivers;
unsigned int priv_sess_size;
if (!rte_cryptodev_is_valid_dev(dev_id))
return 0;
priv_sess_size = (*dev->dev_ops->asym_session_get_size)(dev);
- if (priv_sess_size < header_size)
- return header_size;
return priv_sess_size;
-
}
int
return (void *)(sess->sess_data + sess->nb_drivers);
}
+int
+rte_cryptodev_asym_session_set_user_data(void *session, void *data, uint16_t size)
+{
+ struct rte_cryptodev_asym_session *sess = session;
+ if (sess == NULL)
+ return -EINVAL;
+
+ if (sess->user_data_sz < size)
+ return -ENOMEM;
+
+ rte_memcpy(sess->sess_private_data +
+ sess->max_priv_data_sz,
+ data, size);
+ return 0;
+}
+
+void *
+rte_cryptodev_asym_session_get_user_data(void *session)
+{
+ struct rte_cryptodev_asym_session *sess = session;
+ if (sess == NULL || sess->user_data_sz == 0)
+ return NULL;
+
+ return (void *)(sess->sess_private_data +
+ sess->max_priv_data_sz);
+}
+
static inline void
sym_crypto_fill_status(struct rte_crypto_sym_vec *vec, int32_t errnum)
{
sess_type, session_ctx, is_update);
}
+int
+rte_cryptodev_session_event_mdata_set(uint8_t dev_id, void *sess,
+ enum rte_crypto_op_type op_type,
+ enum rte_crypto_op_sess_type sess_type,
+ void *ev_mdata,
+ uint16_t size)
+{
+ struct rte_cryptodev *dev;
+
+ if (sess == NULL || ev_mdata == NULL)
+ return -EINVAL;
+
+ if (!rte_cryptodev_is_valid_dev(dev_id))
+ goto skip_pmd_op;
+
+ dev = rte_cryptodev_pmd_get_dev(dev_id);
+ if (dev->dev_ops->session_ev_mdata_set == NULL)
+ goto skip_pmd_op;
+
+ return (*dev->dev_ops->session_ev_mdata_set)(dev, sess, op_type,
+ sess_type, ev_mdata);
+
+skip_pmd_op:
+ if (op_type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
+ return rte_cryptodev_sym_session_set_user_data(sess, ev_mdata,
+ size);
+ else if (op_type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) {
+ struct rte_cryptodev_asym_session *s = sess;
+
+ if (s->event_mdata == NULL) {
+ s->event_mdata = rte_malloc(NULL, size, 0);
+ if (s->event_mdata == NULL)
+ return -ENOMEM;
+ }
+ rte_memcpy(s->event_mdata, ev_mdata, size);
+
+ return 0;
+ } else
+ return -ENOTSUP;
+}
+
uint32_t
rte_cryptodev_raw_enqueue_burst(struct rte_crypto_raw_dp_ctx *ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
return nb_drivers++;
}
+
+RTE_INIT(cryptodev_init_fp_ops)
+{
+ uint32_t i;
+
+ for (i = 0; i != RTE_DIM(rte_crypto_fp_ops); i++)
+ cryptodev_fp_ops_reset(rte_crypto_fp_ops + i);
+}
+
+static int
+cryptodev_handle_dev_list(const char *cmd __rte_unused,
+ const char *params __rte_unused,
+ struct rte_tel_data *d)
+{
+ int dev_id;
+
+ if (rte_cryptodev_count() < 1)
+ return -EINVAL;
+
+ rte_tel_data_start_array(d, RTE_TEL_INT_VAL);
+ for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++)
+ if (rte_cryptodev_is_valid_dev(dev_id))
+ rte_tel_data_add_array_int(d, dev_id);
+
+ return 0;
+}
+
+static int
+cryptodev_handle_dev_info(const char *cmd __rte_unused,
+ const char *params, struct rte_tel_data *d)
+{
+ struct rte_cryptodev_info cryptodev_info;
+ int dev_id;
+ char *end_param;
+
+ if (params == NULL || strlen(params) == 0 || !isdigit(*params))
+ return -EINVAL;
+
+ dev_id = strtoul(params, &end_param, 0);
+ if (*end_param != '\0')
+ CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
+ if (!rte_cryptodev_is_valid_dev(dev_id))
+ return -EINVAL;
+
+ rte_cryptodev_info_get(dev_id, &cryptodev_info);
+
+ rte_tel_data_start_dict(d);
+ rte_tel_data_add_dict_string(d, "device_name",
+ cryptodev_info.device->name);
+ rte_tel_data_add_dict_int(d, "max_nb_queue_pairs",
+ cryptodev_info.max_nb_queue_pairs);
+
+ return 0;
+}
+
+#define ADD_DICT_STAT(s) rte_tel_data_add_dict_u64(d, #s, cryptodev_stats.s)
+
+static int
+cryptodev_handle_dev_stats(const char *cmd __rte_unused,
+ const char *params,
+ struct rte_tel_data *d)
+{
+ struct rte_cryptodev_stats cryptodev_stats;
+ int dev_id, ret;
+ char *end_param;
+
+ if (params == NULL || strlen(params) == 0 || !isdigit(*params))
+ return -EINVAL;
+
+ dev_id = strtoul(params, &end_param, 0);
+ if (*end_param != '\0')
+ CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
+ if (!rte_cryptodev_is_valid_dev(dev_id))
+ return -EINVAL;
+
+ ret = rte_cryptodev_stats_get(dev_id, &cryptodev_stats);
+ if (ret < 0)
+ return ret;
+
+ rte_tel_data_start_dict(d);
+ ADD_DICT_STAT(enqueued_count);
+ ADD_DICT_STAT(dequeued_count);
+ ADD_DICT_STAT(enqueue_err_count);
+ ADD_DICT_STAT(dequeue_err_count);
+
+ return 0;
+}
+
+#define CRYPTO_CAPS_SZ \
+ (RTE_ALIGN_CEIL(sizeof(struct rte_cryptodev_capabilities), \
+ sizeof(uint64_t)) / \
+ sizeof(uint64_t))
+
+static int
+crypto_caps_array(struct rte_tel_data *d,
+ const struct rte_cryptodev_capabilities *capabilities)
+{
+ const struct rte_cryptodev_capabilities *dev_caps;
+ uint64_t caps_val[CRYPTO_CAPS_SZ];
+ unsigned int i = 0, j;
+
+ rte_tel_data_start_array(d, RTE_TEL_U64_VAL);
+
+ while ((dev_caps = &capabilities[i++])->op !=
+ RTE_CRYPTO_OP_TYPE_UNDEFINED) {
+ memset(&caps_val, 0, CRYPTO_CAPS_SZ * sizeof(caps_val[0]));
+ rte_memcpy(caps_val, dev_caps, sizeof(capabilities[0]));
+ for (j = 0; j < CRYPTO_CAPS_SZ; j++)
+ rte_tel_data_add_array_u64(d, caps_val[j]);
+ }
+
+ return i;
+}
+
+static int
+cryptodev_handle_dev_caps(const char *cmd __rte_unused, const char *params,
+ struct rte_tel_data *d)
+{
+ struct rte_cryptodev_info dev_info;
+ struct rte_tel_data *crypto_caps;
+ int crypto_caps_n;
+ char *end_param;
+ int dev_id;
+
+ if (!params || strlen(params) == 0 || !isdigit(*params))
+ return -EINVAL;
+
+ dev_id = strtoul(params, &end_param, 0);
+ if (*end_param != '\0')
+ CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
+ if (!rte_cryptodev_is_valid_dev(dev_id))
+ return -EINVAL;
+
+ rte_tel_data_start_dict(d);
+ crypto_caps = rte_tel_data_alloc();
+ if (!crypto_caps)
+ return -ENOMEM;
+
+ rte_cryptodev_info_get(dev_id, &dev_info);
+ crypto_caps_n = crypto_caps_array(crypto_caps, dev_info.capabilities);
+ rte_tel_data_add_dict_container(d, "crypto_caps", crypto_caps, 0);
+ rte_tel_data_add_dict_int(d, "crypto_caps_n", crypto_caps_n);
+
+ return 0;
+}
+
+RTE_INIT(cryptodev_init_telemetry)
+{
+ rte_telemetry_register_cmd("/cryptodev/info", cryptodev_handle_dev_info,
+ "Returns information for a cryptodev. Parameters: int dev_id");
+ rte_telemetry_register_cmd("/cryptodev/list",
+ cryptodev_handle_dev_list,
+ "Returns list of available crypto devices by IDs. No parameters.");
+ rte_telemetry_register_cmd("/cryptodev/stats",
+ cryptodev_handle_dev_stats,
+ "Returns the stats for a cryptodev. Parameters: int dev_id");
+ rte_telemetry_register_cmd("/cryptodev/caps",
+ cryptodev_handle_dev_caps,
+ "Returns the capabilities for a cryptodev. Parameters: int dev_id");
+}
git.droids-corp.org / dpdk.git / blobdiff