drivers/crypto/qat/qat_sym_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_bus_pci.h>
   6 #include <rte_common.h>
   7 #include <rte_dev.h>
   8 #include <rte_malloc.h>
   9 #include <rte_pci.h>
  10 #include <cryptodev_pmd.h>
  11 #ifdef RTE_LIB_SECURITY
  12 #include <rte_security_driver.h>
  13 #endif
  14
  15 #include "qat_logs.h"
  16 #include "qat_crypto.h"
  17 #include "qat_sym.h"
  18 #include "qat_sym_session.h"
  19 #include "qat_sym_pmd.h"
  20
  21 #define MIXED_CRYPTO_MIN_FW_VER 0x04090000
  22
  23 uint8_t qat_sym_driver_id;
  24
  25 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
  26
  27 void
  28 qat_sym_init_op_cookie(void *op_cookie)
  29 {
  30         struct qat_sym_op_cookie *cookie = op_cookie;
  31
  32         cookie->qat_sgl_src_phys_addr =
  33                         rte_mempool_virt2iova(cookie) +
  34                         offsetof(struct qat_sym_op_cookie,
  35                         qat_sgl_src);
  36
  37         cookie->qat_sgl_dst_phys_addr =
  38                         rte_mempool_virt2iova(cookie) +
  39                         offsetof(struct qat_sym_op_cookie,
  40                         qat_sgl_dst);
  41
  42         cookie->opt.spc_gmac.cd_phys_addr =
  43                         rte_mempool_virt2iova(cookie) +
  44                         offsetof(struct qat_sym_op_cookie,
  45                         opt.spc_gmac.cd_cipher);
  46 }
  47
  48 static uint16_t
  49 qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
  50                 uint16_t nb_ops)
  51 {
  52         return qat_enqueue_op_burst(qp, (void **)ops, nb_ops);
  53 }
  54
  55 static uint16_t
  56 qat_sym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
  57                 uint16_t nb_ops)
  58 {
  59         return qat_dequeue_op_burst(qp, (void **)ops, nb_ops);
  60 }
  61
  62 /* An rte_driver is needed in the registration of both the device and the driver
  63  * with cryptodev.
  64  * The actual qat pci's rte_driver can't be used as its name represents
  65  * the whole pci device with all services. Think of this as a holder for a name
  66  * for the crypto part of the pci device.
  67  */
  68 static const char qat_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD);
  69 static const struct rte_driver cryptodev_qat_sym_driver = {
  70         .name = qat_sym_drv_name,
  71         .alias = qat_sym_drv_name
  72 };
  73
  74 int
  75 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
  76                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
  77 {
  78         int i = 0, ret = 0;
  79         struct qat_device_info *qat_dev_instance =
  80                         &qat_pci_devs[qat_pci_dev->qat_dev_id];
  81         struct rte_cryptodev_pmd_init_params init_params = {
  82                 .name = "",
  83                 .socket_id = qat_dev_instance->pci_dev->device.numa_node,
  84                 .private_data_size = sizeof(struct qat_cryptodev_private)
  85         };
  86         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
  87         char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
  88         struct rte_cryptodev *cryptodev;
  89         struct qat_cryptodev_private *internals;
  90         struct qat_capabilities_info capa_info;
  91         const struct rte_cryptodev_capabilities *capabilities;
  92         const struct qat_crypto_gen_dev_ops *gen_dev_ops =
  93                 &qat_sym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
  94         uint64_t capa_size;
  95
  96         snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
  97                         qat_pci_dev->name, "sym");
  98         QAT_LOG(DEBUG, "Creating QAT SYM device %s", name);
  99
 100         if (gen_dev_ops->cryptodev_ops == NULL) {
 101                 QAT_LOG(ERR, "Device %s does not support symmetric crypto",
 102                                 name);
 103                 return -EFAULT;
 104         }
 105
 106         /*
 107          * All processes must use same driver id so they can share sessions.
 108          * Store driver_id so we can validate that all processes have the same
 109          * value, typically they have, but could differ if binaries built
 110          * separately.
 111          */
 112         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 113                 qat_pci_dev->qat_sym_driver_id =
 114                                 qat_sym_driver_id;
 115         } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 116                 if (qat_pci_dev->qat_sym_driver_id !=
 117                                 qat_sym_driver_id) {
 118                         QAT_LOG(ERR,
 119                                 "Device %s have different driver id than corresponding device in primary process",
 120                                 name);
 121                         return -(EFAULT);
 122                 }
 123         }
 124
 125         /* Populate subset device to use in cryptodev device creation */
 126         qat_dev_instance->sym_rte_dev.driver = &cryptodev_qat_sym_driver;
 127         qat_dev_instance->sym_rte_dev.numa_node =
 128                         qat_dev_instance->pci_dev->device.numa_node;
 129         qat_dev_instance->sym_rte_dev.devargs = NULL;
 130
 131         cryptodev = rte_cryptodev_pmd_create(name,
 132                         &(qat_dev_instance->sym_rte_dev), &init_params);
 133
 134         if (cryptodev == NULL)
 135                 return -ENODEV;
 136
 137         qat_dev_instance->sym_rte_dev.name = cryptodev->data->name;
 138         cryptodev->driver_id = qat_sym_driver_id;
 139         cryptodev->dev_ops = gen_dev_ops->cryptodev_ops;
 140
 141         cryptodev->enqueue_burst = qat_sym_pmd_enqueue_op_burst;
 142         cryptodev->dequeue_burst = qat_sym_pmd_dequeue_op_burst;
 143
 144         cryptodev->feature_flags = gen_dev_ops->get_feature_flags(qat_pci_dev);
 145
 146         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 147                 return 0;
 148
 149 #ifdef RTE_LIB_SECURITY
 150         if (gen_dev_ops->create_security_ctx) {
 151                 cryptodev->security_ctx =
 152                         gen_dev_ops->create_security_ctx((void *)cryptodev);
 153                 if (cryptodev->security_ctx == NULL) {
 154                         QAT_LOG(ERR, "rte_security_ctx memory alloc failed");
 155                         ret = -ENOMEM;
 156                         goto error;
 157                 }
 158
 159                 cryptodev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
 160                 QAT_LOG(INFO, "Device %s rte_security support enabled", name);
 161         } else
 162                 QAT_LOG(INFO, "Device %s rte_security support disabled", name);
 163
 164 #endif
 165         snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 166                         "QAT_SYM_CAPA_GEN_%d",
 167                         qat_pci_dev->qat_dev_gen);
 168
 169         internals = cryptodev->data->dev_private;
 170         internals->qat_dev = qat_pci_dev;
 171         internals->service_type = QAT_SERVICE_SYMMETRIC;
 172         internals->dev_id = cryptodev->data->dev_id;
 173
 174         capa_info = gen_dev_ops->get_capabilities(qat_pci_dev);
 175         capabilities = capa_info.data;
 176         capa_size = capa_info.size;
 177
 178         internals->capa_mz = rte_memzone_lookup(capa_memz_name);
 179         if (internals->capa_mz == NULL) {
 180                 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
 181                                 capa_size, rte_socket_id(), 0);
 182                 if (internals->capa_mz == NULL) {
 183                         QAT_LOG(DEBUG,
 184                                 "Error allocating capability memzon for %s",
 185                                 name);
 186                         ret = -EFAULT;
 187                         goto error;
 188                 }
 189         }
 190
 191         memcpy(internals->capa_mz->addr, capabilities, capa_size);
 192         internals->qat_dev_capabilities = internals->capa_mz->addr;
 193
 194         while (1) {
 195                 if (qat_dev_cmd_param[i].name == NULL)
 196                         break;
 197                 if (!strcmp(qat_dev_cmd_param[i].name, SYM_ENQ_THRESHOLD_NAME))
 198                         internals->min_enq_burst_threshold =
 199                                         qat_dev_cmd_param[i].val;
 200                 i++;
 201         }
 202
 203         qat_pci_dev->sym_dev = internals;
 204         QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
 205                         cryptodev->data->name, internals->dev_id);
 206
 207         rte_cryptodev_pmd_probing_finish(cryptodev);
 208
 209         return 0;
 210
 211 error:
 212 #ifdef RTE_LIB_SECURITY
 213         rte_free(cryptodev->security_ctx);
 214         cryptodev->security_ctx = NULL;
 215 #endif
 216         rte_cryptodev_pmd_destroy(cryptodev);
 217         memset(&qat_dev_instance->sym_rte_dev, 0,
 218                 sizeof(qat_dev_instance->sym_rte_dev));
 219
 220         return ret;
 221 }
 222
 223 int
 224 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
 225 {
 226         struct rte_cryptodev *cryptodev;
 227
 228         if (qat_pci_dev == NULL)
 229                 return -ENODEV;
 230         if (qat_pci_dev->sym_dev == NULL)
 231                 return 0;
 232         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 233                 rte_memzone_free(qat_pci_dev->sym_dev->capa_mz);
 234
 235         /* free crypto device */
 236         cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->dev_id);
 237 #ifdef RTE_LIB_SECURITY
 238         rte_free(cryptodev->security_ctx);
 239         cryptodev->security_ctx = NULL;
 240 #endif
 241         rte_cryptodev_pmd_destroy(cryptodev);
 242         qat_pci_devs[qat_pci_dev->qat_dev_id].sym_rte_dev.name = NULL;
 243         qat_pci_dev->sym_dev = NULL;
 244
 245         return 0;
 246 }
 247
 248 static struct cryptodev_driver qat_crypto_drv;
 249 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
 250                 cryptodev_qat_sym_driver,
 251                 qat_sym_driver_id);