eal: refactor --huge-unlink storage

[dpdk.git] / lib / cryptodev / rte_cryptodev.c

diff --git a/lib/cryptodev/rte_cryptodev.c b/lib/cryptodev/rte_cryptodev.c
index 447aa9d..a40536c 100644 (file)
--- a/lib/cryptodev/rte_cryptodev.c
+++ b/lib/cryptodev/rte_cryptodev.c
@@ -12,7 +12,6 @@
 #include <errno.h>
 #include <stdint.h>
 #include <inttypes.h>
-#include <netinet/in.h>
 
 #include <rte_byteorder.h>
 #include <rte_log.h>
@@ -36,10 +35,11 @@
 #include <rte_errno.h>
 #include <rte_spinlock.h>
 #include <rte_string_fns.h>
+#include <rte_telemetry.h>
 
 #include "rte_crypto.h"
 #include "rte_cryptodev.h"
-#include "rte_cryptodev_pmd.h"
+#include "cryptodev_pmd.h"
 #include "rte_cryptodev_trace.h"
 
 static uint8_t nb_drivers;
@@ -54,6 +54,9 @@ static struct rte_cryptodev_global cryptodev_globals = {
                .nb_devs                = 0
 };
 
+/* Public fastpath APIs. */
+struct rte_crypto_fp_ops rte_crypto_fp_ops[RTE_CRYPTO_MAX_DEVS];
+
 /* spinlock for crypto device callbacks */
 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
 
@@ -663,7 +666,7 @@ rte_cryptodev_is_valid_device_data(uint8_t dev_id)
 }
 
 unsigned int
-rte_cryptodev_pmd_is_valid_dev(uint8_t dev_id)
+rte_cryptodev_is_valid_dev(uint8_t dev_id)
 {
        struct rte_cryptodev *dev = NULL;
 
@@ -761,7 +764,7 @@ rte_cryptodev_socket_id(uint8_t dev_id)
 {
        struct rte_cryptodev *dev;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
+       if (!rte_cryptodev_is_valid_dev(dev_id))
                return -1;
 
        dev = rte_cryptodev_pmd_get_dev(dev_id);
@@ -918,6 +921,8 @@ rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev)
 
        dev_id = cryptodev->data->dev_id;
 
+       cryptodev_fp_ops_reset(rte_crypto_fp_ops + dev_id);
+
        /* Close device only if device operations have been set */
        if (cryptodev->dev_ops) {
                ret = rte_cryptodev_close(dev_id);
@@ -979,7 +984,8 @@ rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
        if (dev->data->queue_pairs == NULL) { /* first time configuration */
                dev->data->queue_pairs = rte_zmalloc_socket(
                                "cryptodev->queue_pairs",
-                               sizeof(dev->data->queue_pairs[0]) * nb_qpairs,
+                               sizeof(dev->data->queue_pairs[0]) *
+                               dev_info.max_nb_queue_pairs,
                                RTE_CACHE_LINE_SIZE, socket_id);
 
                if (dev->data->queue_pairs == NULL) {
@@ -1002,25 +1008,9 @@ rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
                        ret = (*dev->dev_ops->queue_pair_release)(dev, i);
                        if (ret < 0)
                                return ret;
+                       qp[i] = NULL;
                }
 
-               qp = rte_realloc(qp, sizeof(qp[0]) * nb_qpairs,
-                               RTE_CACHE_LINE_SIZE);
-               if (qp == NULL) {
-                       CDEV_LOG_ERR("failed to realloc qp meta data,"
-                                               " nb_queues %u", nb_qpairs);
-                       return -(ENOMEM);
-               }
-
-               if (nb_qpairs > old_nb_queues) {
-                       uint16_t new_qs = nb_qpairs - old_nb_queues;
-
-                       memset(qp + old_nb_queues, 0,
-                               sizeof(qp[0]) * new_qs);
-               }
-
-               dev->data->queue_pairs = qp;
-
        }
        dev->data->nb_queue_pairs = nb_qpairs;
        return 0;
@@ -1032,7 +1022,7 @@ rte_cryptodev_configure(uint8_t dev_id, struct rte_cryptodev_config *config)
        struct rte_cryptodev *dev;
        int diag;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -1080,7 +1070,7 @@ rte_cryptodev_start(uint8_t dev_id)
 
        CDEV_LOG_DEBUG("Start dev_id=%" PRIu8, dev_id);
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -1096,6 +1086,9 @@ rte_cryptodev_start(uint8_t dev_id)
        }
 
        diag = (*dev->dev_ops->dev_start)(dev);
+       /* expose selection of PMD fast-path functions */
+       cryptodev_fp_ops_set(rte_crypto_fp_ops + dev_id, dev);
+
        rte_cryptodev_trace_start(dev_id, diag);
        if (diag == 0)
                dev->data->dev_started = 1;
@@ -1110,7 +1103,7 @@ rte_cryptodev_stop(uint8_t dev_id)
 {
        struct rte_cryptodev *dev;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return;
        }
@@ -1125,6 +1118,9 @@ rte_cryptodev_stop(uint8_t dev_id)
                return;
        }
 
+       /* point fast-path functions to dummy ones */
+       cryptodev_fp_ops_reset(rte_crypto_fp_ops + dev_id);
+
        (*dev->dev_ops->dev_stop)(dev);
        rte_cryptodev_trace_stop(dev_id);
        dev->data->dev_started = 0;
@@ -1136,7 +1132,7 @@ rte_cryptodev_close(uint8_t dev_id)
        struct rte_cryptodev *dev;
        int retval;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -1;
        }
@@ -1176,7 +1172,7 @@ rte_cryptodev_get_qp_status(uint8_t dev_id, uint16_t queue_pair_id)
 {
        struct rte_cryptodev *dev;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -1207,7 +1203,7 @@ rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
 {
        struct rte_cryptodev *dev;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -1283,7 +1279,7 @@ rte_cryptodev_add_enq_callback(uint8_t dev_id,
                return NULL;
        }
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
                rte_errno = ENODEV;
                return NULL;
@@ -1349,7 +1345,7 @@ rte_cryptodev_remove_enq_callback(uint8_t dev_id,
                return -EINVAL;
        }
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
                return -ENODEV;
        }
@@ -1418,7 +1414,7 @@ rte_cryptodev_add_deq_callback(uint8_t dev_id,
                return NULL;
        }
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
                rte_errno = ENODEV;
                return NULL;
@@ -1484,7 +1480,7 @@ rte_cryptodev_remove_deq_callback(uint8_t dev_id,
                return -EINVAL;
        }
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
                return -ENODEV;
        }
@@ -1542,7 +1538,7 @@ rte_cryptodev_stats_get(uint8_t dev_id, struct rte_cryptodev_stats *stats)
 {
        struct rte_cryptodev *dev;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
                return -ENODEV;
        }
@@ -1565,7 +1561,7 @@ rte_cryptodev_stats_reset(uint8_t dev_id)
 {
        struct rte_cryptodev *dev;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return;
        }
@@ -1581,7 +1577,7 @@ rte_cryptodev_info_get(uint8_t dev_id, struct rte_cryptodev_info *dev_info)
 {
        struct rte_cryptodev *dev;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%d", dev_id);
                return;
        }
@@ -1608,7 +1604,7 @@ rte_cryptodev_callback_register(uint8_t dev_id,
        if (!cb_fn)
                return -EINVAL;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -1652,7 +1648,7 @@ rte_cryptodev_callback_unregister(uint8_t dev_id,
        if (!cb_fn)
                return -EINVAL;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -1720,7 +1716,7 @@ rte_cryptodev_sym_session_init(uint8_t dev_id,
        uint8_t index;
        int ret;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -1765,7 +1761,7 @@ rte_cryptodev_asym_session_init(uint8_t dev_id,
        uint8_t index;
        int ret;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -1939,7 +1935,7 @@ rte_cryptodev_sym_session_clear(uint8_t dev_id,
        struct rte_cryptodev *dev;
        uint8_t driver_id;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -1969,7 +1965,7 @@ rte_cryptodev_asym_session_clear(uint8_t dev_id,
 {
        struct rte_cryptodev *dev;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
                return -EINVAL;
        }
@@ -2079,7 +2075,7 @@ rte_cryptodev_sym_get_private_session_size(uint8_t dev_id)
        struct rte_cryptodev *dev;
        unsigned int priv_sess_size;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
+       if (!rte_cryptodev_is_valid_dev(dev_id))
                return 0;
 
        dev = rte_cryptodev_pmd_get_dev(dev_id);
@@ -2099,7 +2095,7 @@ rte_cryptodev_asym_get_private_session_size(uint8_t dev_id)
        unsigned int header_size = sizeof(void *) * nb_drivers;
        unsigned int priv_sess_size;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
+       if (!rte_cryptodev_is_valid_dev(dev_id))
                return 0;
 
        dev = rte_cryptodev_pmd_get_dev(dev_id);
@@ -2156,7 +2152,7 @@ rte_cryptodev_sym_cpu_crypto_process(uint8_t dev_id,
 {
        struct rte_cryptodev *dev;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) {
+       if (!rte_cryptodev_is_valid_dev(dev_id)) {
                sym_crypto_fill_status(vec, EINVAL);
                return 0;
        }
@@ -2179,7 +2175,7 @@ rte_cryptodev_get_raw_dp_ctx_size(uint8_t dev_id)
        int32_t size = sizeof(struct rte_crypto_raw_dp_ctx);
        int32_t priv_size;
 
-       if (!rte_cryptodev_pmd_is_valid_dev(dev_id))
+       if (!rte_cryptodev_is_valid_dev(dev_id))
                return -EINVAL;
 
        dev = rte_cryptodev_pmd_get_dev(dev_id);
@@ -2427,3 +2423,163 @@ rte_cryptodev_allocate_driver(struct cryptodev_driver *crypto_drv,
 
        return nb_drivers++;
 }
+
+RTE_INIT(cryptodev_init_fp_ops)
+{
+       uint32_t i;
+
+       for (i = 0; i != RTE_DIM(rte_crypto_fp_ops); i++)
+               cryptodev_fp_ops_reset(rte_crypto_fp_ops + i);
+}
+
+static int
+cryptodev_handle_dev_list(const char *cmd __rte_unused,
+               const char *params __rte_unused,
+               struct rte_tel_data *d)
+{
+       int dev_id;
+
+       if (rte_cryptodev_count() < 1)
+               return -EINVAL;
+
+       rte_tel_data_start_array(d, RTE_TEL_INT_VAL);
+       for (dev_id = 0; dev_id < RTE_CRYPTO_MAX_DEVS; dev_id++)
+               if (rte_cryptodev_is_valid_dev(dev_id))
+                       rte_tel_data_add_array_int(d, dev_id);
+
+       return 0;
+}
+
+static int
+cryptodev_handle_dev_info(const char *cmd __rte_unused,
+               const char *params, struct rte_tel_data *d)
+{
+       struct rte_cryptodev_info cryptodev_info;
+       int dev_id;
+       char *end_param;
+
+       if (params == NULL || strlen(params) == 0 || !isdigit(*params))
+               return -EINVAL;
+
+       dev_id = strtoul(params, &end_param, 0);
+       if (*end_param != '\0')
+               CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
+       if (!rte_cryptodev_is_valid_dev(dev_id))
+               return -EINVAL;
+
+       rte_cryptodev_info_get(dev_id, &cryptodev_info);
+
+       rte_tel_data_start_dict(d);
+       rte_tel_data_add_dict_string(d, "device_name",
+               cryptodev_info.device->name);
+       rte_tel_data_add_dict_int(d, "max_nb_queue_pairs",
+               cryptodev_info.max_nb_queue_pairs);
+
+       return 0;
+}
+
+#define ADD_DICT_STAT(s) rte_tel_data_add_dict_u64(d, #s, cryptodev_stats.s)
+
+static int
+cryptodev_handle_dev_stats(const char *cmd __rte_unused,
+               const char *params,
+               struct rte_tel_data *d)
+{
+       struct rte_cryptodev_stats cryptodev_stats;
+       int dev_id, ret;
+       char *end_param;
+
+       if (params == NULL || strlen(params) == 0 || !isdigit(*params))
+               return -EINVAL;
+
+       dev_id = strtoul(params, &end_param, 0);
+       if (*end_param != '\0')
+               CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
+       if (!rte_cryptodev_is_valid_dev(dev_id))
+               return -EINVAL;
+
+       ret = rte_cryptodev_stats_get(dev_id, &cryptodev_stats);
+       if (ret < 0)
+               return ret;
+
+       rte_tel_data_start_dict(d);
+       ADD_DICT_STAT(enqueued_count);
+       ADD_DICT_STAT(dequeued_count);
+       ADD_DICT_STAT(enqueue_err_count);
+       ADD_DICT_STAT(dequeue_err_count);
+
+       return 0;
+}
+
+#define CRYPTO_CAPS_SZ                                             \
+       (RTE_ALIGN_CEIL(sizeof(struct rte_cryptodev_capabilities), \
+                                       sizeof(uint64_t)) /        \
+        sizeof(uint64_t))
+
+static int
+crypto_caps_array(struct rte_tel_data *d,
+                 const struct rte_cryptodev_capabilities *capabilities)
+{
+       const struct rte_cryptodev_capabilities *dev_caps;
+       uint64_t caps_val[CRYPTO_CAPS_SZ];
+       unsigned int i = 0, j;
+
+       rte_tel_data_start_array(d, RTE_TEL_U64_VAL);
+
+       while ((dev_caps = &capabilities[i++])->op !=
+                       RTE_CRYPTO_OP_TYPE_UNDEFINED) {
+               memset(&caps_val, 0, CRYPTO_CAPS_SZ * sizeof(caps_val[0]));
+               rte_memcpy(caps_val, dev_caps, sizeof(capabilities[0]));
+               for (j = 0; j < CRYPTO_CAPS_SZ; j++)
+                       rte_tel_data_add_array_u64(d, caps_val[j]);
+       }
+
+       return i;
+}
+
+static int
+cryptodev_handle_dev_caps(const char *cmd __rte_unused, const char *params,
+                         struct rte_tel_data *d)
+{
+       struct rte_cryptodev_info dev_info;
+       struct rte_tel_data *crypto_caps;
+       int crypto_caps_n;
+       char *end_param;
+       int dev_id;
+
+       if (!params || strlen(params) == 0 || !isdigit(*params))
+               return -EINVAL;
+
+       dev_id = strtoul(params, &end_param, 0);
+       if (*end_param != '\0')
+               CDEV_LOG_ERR("Extra parameters passed to command, ignoring");
+       if (!rte_cryptodev_is_valid_dev(dev_id))
+               return -EINVAL;
+
+       rte_tel_data_start_dict(d);
+       crypto_caps = rte_tel_data_alloc();
+       if (!crypto_caps)
+               return -ENOMEM;
+
+       rte_cryptodev_info_get(dev_id, &dev_info);
+       crypto_caps_n = crypto_caps_array(crypto_caps, dev_info.capabilities);
+       rte_tel_data_add_dict_container(d, "crypto_caps", crypto_caps, 0);
+       rte_tel_data_add_dict_int(d, "crypto_caps_n", crypto_caps_n);
+
+       return 0;
+}
+
+RTE_INIT(cryptodev_init_telemetry)
+{
+       rte_telemetry_register_cmd("/cryptodev/info", cryptodev_handle_dev_info,
+                       "Returns information for a cryptodev. Parameters: int dev_id");
+       rte_telemetry_register_cmd("/cryptodev/list",
+                       cryptodev_handle_dev_list,
+                       "Returns list of available crypto devices by IDs. No parameters.");
+       rte_telemetry_register_cmd("/cryptodev/stats",
+                       cryptodev_handle_dev_stats,
+                       "Returns the stats for a cryptodev. Parameters: int dev_id");
+       rte_telemetry_register_cmd("/cryptodev/caps",
+                       cryptodev_handle_dev_caps,
+                       "Returns the capabilities for a cryptodev. Parameters: int dev_id");
+}