drivers/crypto/qat/qat_asym_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <rte_cryptodev_pmd.h>
   6
   7 #include "qat_logs.h"
   8
   9 #include "qat_asym.h"
  10 #include "qat_asym_pmd.h"
  11 #include "qat_sym_capabilities.h"
  12 #include "qat_asym_capabilities.h"
  13
  14 uint8_t qat_asym_driver_id;
  15
  16 static const struct rte_cryptodev_capabilities qat_gen1_asym_capabilities[] = {
  17         QAT_BASE_GEN1_ASYM_CAPABILITIES,
  18         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
  19 };
  20
  21 static int qat_asym_qp_release(struct rte_cryptodev *dev,
  22                                uint16_t queue_pair_id);
  23
  24 static int qat_asym_dev_config(__rte_unused struct rte_cryptodev *dev,
  25                                __rte_unused struct rte_cryptodev_config *config)
  26 {
  27         return 0;
  28 }
  29
  30 static int qat_asym_dev_start(__rte_unused struct rte_cryptodev *dev)
  31 {
  32         return 0;
  33 }
  34
  35 static void qat_asym_dev_stop(__rte_unused struct rte_cryptodev *dev)
  36 {
  37
  38 }
  39
  40 static int qat_asym_dev_close(struct rte_cryptodev *dev)
  41 {
  42         int i, ret;
  43
  44         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
  45                 ret = qat_asym_qp_release(dev, i);
  46                 if (ret < 0)
  47                         return ret;
  48         }
  49
  50         return 0;
  51 }
  52
  53 static void qat_asym_dev_info_get(struct rte_cryptodev *dev,
  54                                   struct rte_cryptodev_info *info)
  55 {
  56         struct qat_asym_dev_private *internals = dev->data->dev_private;
  57         const struct qat_qp_hw_data *asym_hw_qps =
  58                 qat_gen_config[internals->qat_dev->qat_dev_gen]
  59                               .qp_hw_data[QAT_SERVICE_ASYMMETRIC];
  60
  61         if (info != NULL) {
  62                 info->max_nb_queue_pairs = qat_qps_per_service(asym_hw_qps,
  63                                                         QAT_SERVICE_ASYMMETRIC);
  64                 info->feature_flags = dev->feature_flags;
  65                 info->capabilities = internals->qat_dev_capabilities;
  66                 info->driver_id = qat_asym_driver_id;
  67                 /* No limit of number of sessions */
  68                 info->sym.max_nb_sessions = 0;
  69         }
  70 }
  71
  72 static void qat_asym_stats_get(struct rte_cryptodev *dev,
  73                                struct rte_cryptodev_stats *stats)
  74 {
  75         struct qat_common_stats qat_stats = {0};
  76         struct qat_asym_dev_private *qat_priv;
  77
  78         if (stats == NULL || dev == NULL) {
  79                 QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
  80                 return;
  81         }
  82         qat_priv = dev->data->dev_private;
  83
  84         qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_ASYMMETRIC);
  85         stats->enqueued_count = qat_stats.enqueued_count;
  86         stats->dequeued_count = qat_stats.dequeued_count;
  87         stats->enqueue_err_count = qat_stats.enqueue_err_count;
  88         stats->dequeue_err_count = qat_stats.dequeue_err_count;
  89 }
  90
  91 static void qat_asym_stats_reset(struct rte_cryptodev *dev)
  92 {
  93         struct qat_asym_dev_private *qat_priv;
  94
  95         if (dev == NULL) {
  96                 QAT_LOG(ERR, "invalid asymmetric cryptodev ptr %p", dev);
  97                 return;
  98         }
  99         qat_priv = dev->data->dev_private;
 100
 101         qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_ASYMMETRIC);
 102 }
 103
 104 static int qat_asym_qp_release(struct rte_cryptodev *dev,
 105                                uint16_t queue_pair_id)
 106 {
 107         struct qat_asym_dev_private *qat_private = dev->data->dev_private;
 108
 109         QAT_LOG(DEBUG, "Release asym qp %u on device %d",
 110                                 queue_pair_id, dev->data->dev_id);
 111
 112         qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][queue_pair_id]
 113                                                 = NULL;
 114
 115         return qat_qp_release((struct qat_qp **)
 116                         &(dev->data->queue_pairs[queue_pair_id]));
 117 }
 118
 119 static int qat_asym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 120                              const struct rte_cryptodev_qp_conf *qp_conf,
 121                              int socket_id)
 122 {
 123         struct qat_qp_config qat_qp_conf;
 124         struct qat_qp *qp;
 125         int ret = 0;
 126         uint32_t i;
 127
 128         struct qat_qp **qp_addr =
 129                         (struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
 130         struct qat_asym_dev_private *qat_private = dev->data->dev_private;
 131         const struct qat_qp_hw_data *asym_hw_qps =
 132                         qat_gen_config[qat_private->qat_dev->qat_dev_gen]
 133                                       .qp_hw_data[QAT_SERVICE_ASYMMETRIC];
 134         const struct qat_qp_hw_data *qp_hw_data = asym_hw_qps + qp_id;
 135
 136         /* If qp is already in use free ring memory and qp metadata. */
 137         if (*qp_addr != NULL) {
 138                 ret = qat_asym_qp_release(dev, qp_id);
 139                 if (ret < 0)
 140                         return ret;
 141         }
 142         if (qp_id >= qat_qps_per_service(asym_hw_qps, QAT_SERVICE_ASYMMETRIC)) {
 143                 QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
 144                 return -EINVAL;
 145         }
 146
 147         qat_qp_conf.hw = qp_hw_data;
 148         qat_qp_conf.build_request = qat_asym_build_request;
 149         qat_qp_conf.cookie_size = sizeof(struct qat_asym_op_cookie);
 150         qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
 151         qat_qp_conf.socket_id = socket_id;
 152         qat_qp_conf.service_str = "asym";
 153
 154         ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf);
 155         if (ret != 0)
 156                 return ret;
 157
 158         /* store a link to the qp in the qat_pci_device */
 159         qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][qp_id]
 160                                                         = *qp_addr;
 161
 162         qp = (struct qat_qp *)*qp_addr;
 163         qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
 164
 165         for (i = 0; i < qp->nb_descriptors; i++) {
 166                 int j;
 167
 168                 struct qat_asym_op_cookie __rte_unused *cookie =
 169                                 qp->op_cookies[i];
 170                 cookie->input_addr = rte_mempool_virt2iova(cookie) +
 171                                 offsetof(struct qat_asym_op_cookie,
 172                                                 input_params_ptrs);
 173
 174                 cookie->output_addr = rte_mempool_virt2iova(cookie) +
 175                                 offsetof(struct qat_asym_op_cookie,
 176                                                 output_params_ptrs);
 177
 178                 for (j = 0; j < 8; j++) {
 179                         cookie->input_params_ptrs[j] =
 180                                         rte_mempool_virt2iova(cookie) +
 181                                         offsetof(struct qat_asym_op_cookie,
 182                                                         input_array[j]);
 183                         cookie->output_params_ptrs[j] =
 184                                         rte_mempool_virt2iova(cookie) +
 185                                         offsetof(struct qat_asym_op_cookie,
 186                                                         output_array[j]);
 187                 }
 188         }
 189
 190         return ret;
 191 }
 192
 193 struct rte_cryptodev_ops crypto_qat_ops = {
 194
 195         /* Device related operations */
 196         .dev_configure          = qat_asym_dev_config,
 197         .dev_start              = qat_asym_dev_start,
 198         .dev_stop               = qat_asym_dev_stop,
 199         .dev_close              = qat_asym_dev_close,
 200         .dev_infos_get          = qat_asym_dev_info_get,
 201
 202         .stats_get              = qat_asym_stats_get,
 203         .stats_reset            = qat_asym_stats_reset,
 204         .queue_pair_setup       = qat_asym_qp_setup,
 205         .queue_pair_release     = qat_asym_qp_release,
 206
 207         /* Crypto related operations */
 208         .asym_session_get_size  = qat_asym_session_get_private_size,
 209         .asym_session_configure = qat_asym_session_configure,
 210         .asym_session_clear     = qat_asym_session_clear
 211 };
 212
 213 uint16_t qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 214                                        uint16_t nb_ops)
 215 {
 216         return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
 217 }
 218
 219 uint16_t qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 220                                        uint16_t nb_ops)
 221 {
 222         return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
 223 }
 224
 225 /* An rte_driver is needed in the registration of both the device and the driver
 226  * with cryptodev.
 227  * The actual qat pci's rte_driver can't be used as its name represents
 228  * the whole pci device with all services. Think of this as a holder for a name
 229  * for the crypto part of the pci device.
 230  */
 231 static const char qat_asym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_ASYM_PMD);
 232 static const struct rte_driver cryptodev_qat_asym_driver = {
 233         .name = qat_asym_drv_name,
 234         .alias = qat_asym_drv_name
 235 };
 236
 237 int
 238 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
 239                 struct qat_dev_cmd_param *qat_dev_cmd_param)
 240 {
 241         int i = 0;
 242         struct qat_device_info *qat_dev_instance =
 243                         &qat_pci_devs[qat_pci_dev->qat_dev_id];
 244         struct rte_cryptodev_pmd_init_params init_params = {
 245                         .name = "",
 246                         .socket_id =
 247                                 qat_dev_instance->pci_dev->device.numa_node,
 248                         .private_data_size = sizeof(struct qat_asym_dev_private)
 249         };
 250         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 251         char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 252         struct rte_cryptodev *cryptodev;
 253         struct qat_asym_dev_private *internals;
 254
 255         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 256                 qat_pci_dev->qat_asym_driver_id =
 257                                 qat_asym_driver_id;
 258         } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 259                 if (qat_pci_dev->qat_asym_driver_id !=
 260                                 qat_asym_driver_id) {
 261                         QAT_LOG(ERR,
 262                                 "Device %s have different driver id than corresponding device in primary process",
 263                                 name);
 264                         return -(EFAULT);
 265                 }
 266         }
 267
 268         snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
 269                         qat_pci_dev->name, "asym");
 270         QAT_LOG(DEBUG, "Creating QAT ASYM device %s\n", name);
 271
 272         /* Populate subset device to use in cryptodev device creation */
 273         qat_dev_instance->asym_rte_dev.driver = &cryptodev_qat_asym_driver;
 274         qat_dev_instance->asym_rte_dev.numa_node =
 275                         qat_dev_instance->pci_dev->device.numa_node;
 276         qat_dev_instance->asym_rte_dev.devargs = NULL;
 277
 278         cryptodev = rte_cryptodev_pmd_create(name,
 279                         &(qat_dev_instance->asym_rte_dev), &init_params);
 280
 281         if (cryptodev == NULL)
 282                 return -ENODEV;
 283
 284         qat_dev_instance->asym_rte_dev.name = cryptodev->data->name;
 285         cryptodev->driver_id = qat_asym_driver_id;
 286         cryptodev->dev_ops = &crypto_qat_ops;
 287
 288         cryptodev->enqueue_burst = qat_asym_pmd_enqueue_op_burst;
 289         cryptodev->dequeue_burst = qat_asym_pmd_dequeue_op_burst;
 290
 291         cryptodev->feature_flags = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
 292                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
 293                         RTE_CRYPTODEV_FF_ASYM_SESSIONLESS |
 294                         RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP |
 295                         RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT;
 296
 297         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 298                 return 0;
 299
 300         snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 301                         "QAT_ASYM_CAPA_GEN_%d",
 302                         qat_pci_dev->qat_dev_gen);
 303
 304         internals = cryptodev->data->dev_private;
 305         internals->qat_dev = qat_pci_dev;
 306         internals->asym_dev_id = cryptodev->data->dev_id;
 307         internals->qat_dev_capabilities = qat_gen1_asym_capabilities;
 308
 309         internals->capa_mz = rte_memzone_lookup(capa_memz_name);
 310         if (internals->capa_mz == NULL) {
 311                 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
 312                         sizeof(qat_gen1_asym_capabilities),
 313                         rte_socket_id(), 0);
 314         }
 315         if (internals->capa_mz == NULL) {
 316                 QAT_LOG(DEBUG,
 317                         "Error allocating memzone for capabilities, destroying PMD for %s",
 318                         name);
 319                 rte_cryptodev_pmd_destroy(cryptodev);
 320                 memset(&qat_dev_instance->asym_rte_dev, 0,
 321                         sizeof(qat_dev_instance->asym_rte_dev));
 322                 return -EFAULT;
 323         }
 324
 325         memcpy(internals->capa_mz->addr, qat_gen1_asym_capabilities,
 326                         sizeof(qat_gen1_asym_capabilities));
 327         internals->qat_dev_capabilities = internals->capa_mz->addr;
 328
 329         while (1) {
 330                 if (qat_dev_cmd_param[i].name == NULL)
 331                         break;
 332                 if (!strcmp(qat_dev_cmd_param[i].name, ASYM_ENQ_THRESHOLD_NAME))
 333                         internals->min_enq_burst_threshold =
 334                                         qat_dev_cmd_param[i].val;
 335                 i++;
 336         }
 337
 338         qat_pci_dev->asym_dev = internals;
 339         QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
 340                         cryptodev->data->name, internals->asym_dev_id);
 341         return 0;
 342 }
 343
 344 int
 345 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev)
 346 {
 347         struct rte_cryptodev *cryptodev;
 348
 349         if (qat_pci_dev == NULL)
 350                 return -ENODEV;
 351         if (qat_pci_dev->asym_dev == NULL)
 352                 return 0;
 353         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 354                 rte_memzone_free(qat_pci_dev->asym_dev->capa_mz);
 355
 356         /* free crypto device */
 357         cryptodev = rte_cryptodev_pmd_get_dev(
 358                         qat_pci_dev->asym_dev->asym_dev_id);
 359         rte_cryptodev_pmd_destroy(cryptodev);
 360         qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL;
 361         qat_pci_dev->asym_dev = NULL;
 362
 363         return 0;
 364 }
 365
 366 static struct cryptodev_driver qat_crypto_drv;
 367 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
 368                 cryptodev_qat_asym_driver,
 369                 qat_asym_driver_id);