drivers/crypto/qat/qat_sym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2022 Intel Corporation
   3  */
   4
   5 #include <openssl/evp.h>
   6
   7 #include <rte_mempool.h>
   8 #include <rte_mbuf.h>
   9 #include <rte_crypto_sym.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_byteorder.h>
  12
  13 #include "qat_sym.h"
  14 #include "qat_crypto.h"
  15 #include "qat_qp.h"
  16
  17 uint8_t qat_sym_driver_id;
  18 int qat_ipsec_mb_lib;
  19
  20 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
  21
  22 /* An rte_driver is needed in the registration of both the device and the driver
  23  * with cryptodev.
  24  * The actual qat pci's rte_driver can't be used as its name represents
  25  * the whole pci device with all services. Think of this as a holder for a name
  26  * for the crypto part of the pci device.
  27  */
  28 static const char qat_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD);
  29 static const struct rte_driver cryptodev_qat_sym_driver = {
  30         .name = qat_sym_drv_name,
  31         .alias = qat_sym_drv_name
  32 };
  33
  34 void
  35 qat_sym_init_op_cookie(void *op_cookie)
  36 {
  37         struct qat_sym_op_cookie *cookie = op_cookie;
  38
  39         cookie->qat_sgl_src_phys_addr =
  40                         rte_mempool_virt2iova(cookie) +
  41                         offsetof(struct qat_sym_op_cookie,
  42                         qat_sgl_src);
  43
  44         cookie->qat_sgl_dst_phys_addr =
  45                         rte_mempool_virt2iova(cookie) +
  46                         offsetof(struct qat_sym_op_cookie,
  47                         qat_sgl_dst);
  48
  49         cookie->opt.spc_gmac.cd_phys_addr =
  50                         rte_mempool_virt2iova(cookie) +
  51                         offsetof(struct qat_sym_op_cookie,
  52                         opt.spc_gmac.cd_cipher);
  53 }
  54
  55 static __rte_always_inline int
  56 qat_sym_build_request(void *in_op, uint8_t *out_msg,
  57                 void *op_cookie, uint64_t *opaque, enum qat_device_gen dev_gen)
  58 {
  59         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
  60         uintptr_t sess = (uintptr_t)opaque[0];
  61         uintptr_t build_request_p = (uintptr_t)opaque[1];
  62         qat_sym_build_request_t build_request = (void *)build_request_p;
  63         struct qat_sym_session *ctx = NULL;
  64         enum rte_proc_type_t proc_type = rte_eal_process_type();
  65
  66         if (proc_type == RTE_PROC_AUTO || proc_type == RTE_PROC_INVALID)
  67                 return -EINVAL;
  68
  69         if (likely(op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)) {
  70                 ctx = get_sym_session_private_data(op->sym->session,
  71                                 qat_sym_driver_id);
  72                 if (unlikely(!ctx)) {
  73                         QAT_DP_LOG(ERR, "No session for this device");
  74                         return -EINVAL;
  75                 }
  76                 if (sess != (uintptr_t)ctx) {
  77                         struct rte_cryptodev *cdev;
  78                         struct qat_cryptodev_private *internals;
  79
  80                         cdev = rte_cryptodev_pmd_get_dev(ctx->dev_id);
  81                         internals = cdev->data->dev_private;
  82
  83                         if (internals->qat_dev->qat_dev_gen != dev_gen) {
  84                                 op->status =
  85                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
  86                                 return -EINVAL;
  87                         }
  88
  89                         if (unlikely(ctx->build_request[proc_type] == NULL)) {
  90                                 int ret =
  91                                 qat_sym_gen_dev_ops[dev_gen].set_session(
  92                                         (void *)cdev, (void *)sess);
  93                                 if (ret < 0) {
  94                                         op->status =
  95                                                 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
  96                                         return -EINVAL;
  97                                 }
  98                         }
  99
 100                         build_request = ctx->build_request[proc_type];
 101                         opaque[0] = (uintptr_t)ctx;
 102                         opaque[1] = (uintptr_t)build_request;
 103                 }
 104         }
 105
 106 #ifdef RTE_LIB_SECURITY
 107         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 108                 if ((void *)sess != (void *)op->sym->sec_session) {
 109                         struct rte_cryptodev *cdev;
 110                         struct qat_cryptodev_private *internals;
 111
 112                         ctx = get_sec_session_private_data(
 113                                         op->sym->sec_session);
 114                         if (unlikely(!ctx)) {
 115                                 QAT_DP_LOG(ERR, "No session for this device");
 116                                 return -EINVAL;
 117                         }
 118                         if (unlikely(ctx->bpi_ctx == NULL)) {
 119                                 QAT_DP_LOG(ERR, "QAT PMD only supports security"
 120                                                 " operation requests for"
 121                                                 " DOCSIS, op (%p) is not for"
 122                                                 " DOCSIS.", op);
 123                                 return -EINVAL;
 124                         } else if (unlikely(((op->sym->m_dst != NULL) &&
 125                                         (op->sym->m_dst != op->sym->m_src)) ||
 126                                         op->sym->m_src->nb_segs > 1)) {
 127                                 QAT_DP_LOG(ERR, "OOP and/or multi-segment"
 128                                                 " buffers not supported for"
 129                                                 " DOCSIS security.");
 130                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 131                                 return -EINVAL;
 132                         }
 133                         cdev = rte_cryptodev_pmd_get_dev(ctx->dev_id);
 134                         internals = cdev->data->dev_private;
 135
 136                         if (internals->qat_dev->qat_dev_gen != dev_gen) {
 137                                 op->status =
 138                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 139                                 return -EINVAL;
 140                         }
 141
 142                         if (unlikely(ctx->build_request[proc_type] == NULL)) {
 143                                 int ret =
 144                                 qat_sym_gen_dev_ops[dev_gen].set_session(
 145                                         (void *)cdev, (void *)sess);
 146                                 if (ret < 0) {
 147                                         op->status =
 148                                                 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 149                                         return -EINVAL;
 150                                 }
 151                         }
 152
 153                         sess = (uintptr_t)op->sym->sec_session;
 154                         build_request = ctx->build_request[proc_type];
 155                         opaque[0] = sess;
 156                         opaque[1] = (uintptr_t)build_request;
 157                 }
 158         }
 159 #endif
 160         else { /* RTE_CRYPTO_OP_SESSIONLESS */
 161                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 162                 QAT_LOG(DEBUG, "QAT does not support sessionless operation");
 163                 return -1;
 164         }
 165
 166         return build_request(op, (void *)ctx, out_msg, op_cookie);
 167 }
 168
 169 uint16_t
 170 qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 171                 uint16_t nb_ops)
 172 {
 173         return qat_enqueue_op_burst(qp, qat_sym_build_request,
 174                         (void **)ops, nb_ops);
 175 }
 176
 177 uint16_t
 178 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 179                 uint16_t nb_ops)
 180 {
 181         return qat_dequeue_op_burst(qp, (void **)ops,
 182                                 qat_sym_process_response, nb_ops);
 183 }
 184
 185 int
 186 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
 187                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 188 {
 189         int i = 0, ret = 0;
 190         struct qat_device_info *qat_dev_instance =
 191                         &qat_pci_devs[qat_pci_dev->qat_dev_id];
 192         struct rte_cryptodev_pmd_init_params init_params = {
 193                 .name = "",
 194                 .socket_id = qat_dev_instance->pci_dev->device.numa_node,
 195                 .private_data_size = sizeof(struct qat_cryptodev_private)
 196         };
 197         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 198         char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 199         struct rte_cryptodev *cryptodev;
 200         struct qat_cryptodev_private *internals;
 201         struct qat_capabilities_info capa_info;
 202         const struct rte_cryptodev_capabilities *capabilities;
 203         const struct qat_crypto_gen_dev_ops *gen_dev_ops =
 204                 &qat_sym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
 205         uint64_t capa_size;
 206
 207         snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
 208                         qat_pci_dev->name, "sym");
 209         QAT_LOG(DEBUG, "Creating QAT SYM device %s", name);
 210
 211         if (gen_dev_ops->cryptodev_ops == NULL) {
 212                 QAT_LOG(ERR, "Device %s does not support symmetric crypto",
 213                                 name);
 214                 return -(EFAULT);
 215         }
 216
 217         /*
 218          * All processes must use same driver id so they can share sessions.
 219          * Store driver_id so we can validate that all processes have the same
 220          * value, typically they have, but could differ if binaries built
 221          * separately.
 222          */
 223         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 224                 qat_pci_dev->qat_sym_driver_id =
 225                                 qat_sym_driver_id;
 226         } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 227                 if (qat_pci_dev->qat_sym_driver_id !=
 228                                 qat_sym_driver_id) {
 229                         QAT_LOG(ERR,
 230                                 "Device %s have different driver id than corresponding device in primary process",
 231                                 name);
 232                         return -(EFAULT);
 233                 }
 234         }
 235
 236         /* Populate subset device to use in cryptodev device creation */
 237         qat_dev_instance->sym_rte_dev.driver = &cryptodev_qat_sym_driver;
 238         qat_dev_instance->sym_rte_dev.numa_node =
 239                         qat_dev_instance->pci_dev->device.numa_node;
 240         qat_dev_instance->sym_rte_dev.devargs = NULL;
 241
 242         cryptodev = rte_cryptodev_pmd_create(name,
 243                         &(qat_dev_instance->sym_rte_dev), &init_params);
 244
 245         if (cryptodev == NULL)
 246                 return -ENODEV;
 247
 248         qat_dev_instance->sym_rte_dev.name = cryptodev->data->name;
 249         cryptodev->driver_id = qat_sym_driver_id;
 250         cryptodev->dev_ops = gen_dev_ops->cryptodev_ops;
 251
 252         cryptodev->enqueue_burst = qat_sym_enqueue_burst;
 253         cryptodev->dequeue_burst = qat_sym_dequeue_burst;
 254
 255         cryptodev->feature_flags = gen_dev_ops->get_feature_flags(qat_pci_dev);
 256
 257         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 258                 return 0;
 259
 260 #ifdef RTE_LIB_SECURITY
 261         if (gen_dev_ops->create_security_ctx) {
 262                 cryptodev->security_ctx =
 263                         gen_dev_ops->create_security_ctx((void *)cryptodev);
 264                 if (cryptodev->security_ctx == NULL) {
 265                         QAT_LOG(ERR, "rte_security_ctx memory alloc failed");
 266                         ret = -ENOMEM;
 267                         goto error;
 268                 }
 269
 270                 cryptodev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
 271                 QAT_LOG(INFO, "Device %s rte_security support ensabled", name);
 272         } else {
 273                 QAT_LOG(INFO, "Device %s rte_security support disabled", name);
 274         }
 275 #endif
 276         snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 277                         "QAT_SYM_CAPA_GEN_%d",
 278                         qat_pci_dev->qat_dev_gen);
 279
 280         internals = cryptodev->data->dev_private;
 281         internals->qat_dev = qat_pci_dev;
 282
 283         internals->dev_id = cryptodev->data->dev_id;
 284
 285         capa_info = gen_dev_ops->get_capabilities(qat_pci_dev);
 286         capabilities = capa_info.data;
 287         capa_size = capa_info.size;
 288
 289         internals->capa_mz = rte_memzone_lookup(capa_memz_name);
 290         if (internals->capa_mz == NULL) {
 291                 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
 292                                 capa_size, rte_socket_id(), 0);
 293                 if (internals->capa_mz == NULL) {
 294                         QAT_LOG(DEBUG,
 295                                 "Error allocating memzone for capabilities, "
 296                                 "destroying PMD for %s", name);
 297                         ret = -EFAULT;
 298                         goto error;
 299                 }
 300         }
 301
 302         memcpy(internals->capa_mz->addr, capabilities, capa_size);
 303         internals->qat_dev_capabilities = internals->capa_mz->addr;
 304
 305         while (1) {
 306                 if (qat_dev_cmd_param[i].name == NULL)
 307                         break;
 308                 if (!strcmp(qat_dev_cmd_param[i].name, SYM_ENQ_THRESHOLD_NAME))
 309                         internals->min_enq_burst_threshold =
 310                                         qat_dev_cmd_param[i].val;
 311                 if (!strcmp(qat_dev_cmd_param[i].name, QAT_IPSEC_MB_LIB))
 312                         qat_ipsec_mb_lib = qat_dev_cmd_param[i].val;
 313                 i++;
 314         }
 315
 316         internals->service_type = QAT_SERVICE_SYMMETRIC;
 317         qat_pci_dev->sym_dev = internals;
 318         QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
 319                         cryptodev->data->name, internals->dev_id);
 320
 321         return 0;
 322
 323 error:
 324 #ifdef RTE_LIB_SECURITY
 325         rte_free(cryptodev->security_ctx);
 326         cryptodev->security_ctx = NULL;
 327 #endif
 328         rte_cryptodev_pmd_destroy(cryptodev);
 329         memset(&qat_dev_instance->sym_rte_dev, 0,
 330                 sizeof(qat_dev_instance->sym_rte_dev));
 331
 332         return ret;
 333 }
 334
 335 int
 336 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
 337 {
 338         struct rte_cryptodev *cryptodev;
 339
 340         if (qat_pci_dev == NULL)
 341                 return -ENODEV;
 342         if (qat_pci_dev->sym_dev == NULL)
 343                 return 0;
 344         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 345                 rte_memzone_free(qat_pci_dev->sym_dev->capa_mz);
 346
 347         /* free crypto device */
 348         cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->dev_id);
 349 #ifdef RTE_LIB_SECURITY
 350         rte_free(cryptodev->security_ctx);
 351         cryptodev->security_ctx = NULL;
 352 #endif
 353         rte_cryptodev_pmd_destroy(cryptodev);
 354         qat_pci_devs[qat_pci_dev->qat_dev_id].sym_rte_dev.name = NULL;
 355         qat_pci_dev->sym_dev = NULL;
 356
 357         return 0;
 358 }
 359
 360 int
 361 qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
 362         struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
 363         enum rte_crypto_op_sess_type sess_type,
 364         union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
 365 {
 366         struct qat_cryptodev_private *internals = dev->data->dev_private;
 367         enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
 368         struct qat_crypto_gen_dev_ops *gen_dev_ops =
 369                         &qat_sym_gen_dev_ops[qat_dev_gen];
 370         struct qat_qp *qp;
 371         struct qat_sym_session *ctx;
 372         struct qat_sym_dp_ctx *dp_ctx;
 373
 374         if (!gen_dev_ops->set_raw_dp_ctx) {
 375                 QAT_LOG(ERR, "Device GEN %u does not support raw data path",
 376                                 qat_dev_gen);
 377                 return -ENOTSUP;
 378         }
 379
 380         qp = dev->data->queue_pairs[qp_id];
 381         dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
 382
 383         if (!is_update) {
 384                 memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
 385                                 sizeof(struct qat_sym_dp_ctx));
 386                 raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
 387                 dp_ctx->tail = qp->tx_q.tail;
 388                 dp_ctx->head = qp->rx_q.head;
 389                 dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
 390         }
 391
 392         if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
 393                 return -EINVAL;
 394
 395         ctx = (struct qat_sym_session *)get_sym_session_private_data(
 396                         session_ctx.crypto_sess, qat_sym_driver_id);
 397
 398         dp_ctx->session = ctx;
 399
 400         return gen_dev_ops->set_raw_dp_ctx(raw_dp_ctx, ctx);
 401 }
 402
 403 int
 404 qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev __rte_unused)
 405 {
 406         return sizeof(struct qat_sym_dp_ctx);
 407 }
 408
 409 static struct cryptodev_driver qat_crypto_drv;
 410 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
 411                 cryptodev_qat_sym_driver,
 412                 qat_sym_driver_id);