drivers/crypto/qat/qat_sym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2022 Intel Corporation
   3  */
   4
   5 #define OPENSSL_API_COMPAT 0x10100000L
   6
   7 #include <openssl/evp.h>
   8
   9 #include <rte_mempool.h>
  10 #include <rte_mbuf.h>
  11 #include <rte_crypto_sym.h>
  12 #include <rte_bus_pci.h>
  13 #include <rte_byteorder.h>
  14
  15 #include "qat_sym.h"
  16 #include "qat_crypto.h"
  17 #include "qat_qp.h"
  18
  19 uint8_t qat_sym_driver_id;
  20
  21 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
  22
  23 /* An rte_driver is needed in the registration of both the device and the driver
  24  * with cryptodev.
  25  * The actual qat pci's rte_driver can't be used as its name represents
  26  * the whole pci device with all services. Think of this as a holder for a name
  27  * for the crypto part of the pci device.
  28  */
  29 static const char qat_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD);
  30 static const struct rte_driver cryptodev_qat_sym_driver = {
  31         .name = qat_sym_drv_name,
  32         .alias = qat_sym_drv_name
  33 };
  34
  35 void
  36 qat_sym_init_op_cookie(void *op_cookie)
  37 {
  38         struct qat_sym_op_cookie *cookie = op_cookie;
  39
  40         cookie->qat_sgl_src_phys_addr =
  41                         rte_mempool_virt2iova(cookie) +
  42                         offsetof(struct qat_sym_op_cookie,
  43                         qat_sgl_src);
  44
  45         cookie->qat_sgl_dst_phys_addr =
  46                         rte_mempool_virt2iova(cookie) +
  47                         offsetof(struct qat_sym_op_cookie,
  48                         qat_sgl_dst);
  49
  50         cookie->opt.spc_gmac.cd_phys_addr =
  51                         rte_mempool_virt2iova(cookie) +
  52                         offsetof(struct qat_sym_op_cookie,
  53                         opt.spc_gmac.cd_cipher);
  54 }
  55
  56 static __rte_always_inline int
  57 qat_sym_build_request(void *in_op, uint8_t *out_msg,
  58                 void *op_cookie, uint64_t *opaque, enum qat_device_gen dev_gen)
  59 {
  60         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
  61         uintptr_t sess = (uintptr_t)opaque[0];
  62         uintptr_t build_request_p = (uintptr_t)opaque[1];
  63         qat_sym_build_request_t build_request = (void *)build_request_p;
  64         struct qat_sym_session *ctx = NULL;
  65         enum rte_proc_type_t proc_type = rte_eal_process_type();
  66
  67         if (proc_type == RTE_PROC_AUTO || proc_type == RTE_PROC_INVALID)
  68                 return -EINVAL;
  69
  70         if (likely(op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)) {
  71                 ctx = get_sym_session_private_data(op->sym->session,
  72                                 qat_sym_driver_id);
  73                 if (unlikely(!ctx)) {
  74                         QAT_DP_LOG(ERR, "No session for this device");
  75                         return -EINVAL;
  76                 }
  77                 if (sess != (uintptr_t)ctx) {
  78                         struct rte_cryptodev *cdev;
  79                         struct qat_cryptodev_private *internals;
  80
  81                         cdev = rte_cryptodev_pmd_get_dev(ctx->dev_id);
  82                         internals = cdev->data->dev_private;
  83
  84                         if (internals->qat_dev->qat_dev_gen != dev_gen) {
  85                                 op->status =
  86                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
  87                                 return -EINVAL;
  88                         }
  89
  90                         if (unlikely(ctx->build_request[proc_type] == NULL)) {
  91                                 int ret =
  92                                 qat_sym_gen_dev_ops[dev_gen].set_session(
  93                                         (void *)cdev, (void *)sess);
  94                                 if (ret < 0) {
  95                                         op->status =
  96                                                 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
  97                                         return -EINVAL;
  98                                 }
  99                         }
 100
 101                         build_request = ctx->build_request[proc_type];
 102                         opaque[0] = (uintptr_t)ctx;
 103                         opaque[1] = (uintptr_t)build_request;
 104                 }
 105         }
 106
 107 #ifdef RTE_LIB_SECURITY
 108         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 109                 if ((void *)sess != (void *)op->sym->sec_session) {
 110                         struct rte_cryptodev *cdev;
 111                         struct qat_cryptodev_private *internals;
 112
 113                         ctx = get_sec_session_private_data(
 114                                         op->sym->sec_session);
 115                         if (unlikely(!ctx)) {
 116                                 QAT_DP_LOG(ERR, "No session for this device");
 117                                 return -EINVAL;
 118                         }
 119                         if (unlikely(ctx->bpi_ctx == NULL)) {
 120                                 QAT_DP_LOG(ERR, "QAT PMD only supports security"
 121                                                 " operation requests for"
 122                                                 " DOCSIS, op (%p) is not for"
 123                                                 " DOCSIS.", op);
 124                                 return -EINVAL;
 125                         } else if (unlikely(((op->sym->m_dst != NULL) &&
 126                                         (op->sym->m_dst != op->sym->m_src)) ||
 127                                         op->sym->m_src->nb_segs > 1)) {
 128                                 QAT_DP_LOG(ERR, "OOP and/or multi-segment"
 129                                                 " buffers not supported for"
 130                                                 " DOCSIS security.");
 131                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 132                                 return -EINVAL;
 133                         }
 134                         cdev = rte_cryptodev_pmd_get_dev(ctx->dev_id);
 135                         internals = cdev->data->dev_private;
 136
 137                         if (internals->qat_dev->qat_dev_gen != dev_gen) {
 138                                 op->status =
 139                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 140                                 return -EINVAL;
 141                         }
 142
 143                         if (unlikely(ctx->build_request[proc_type] == NULL)) {
 144                                 int ret =
 145                                 qat_sym_gen_dev_ops[dev_gen].set_session(
 146                                         (void *)cdev, (void *)sess);
 147                                 if (ret < 0) {
 148                                         op->status =
 149                                                 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 150                                         return -EINVAL;
 151                                 }
 152                         }
 153
 154                         sess = (uintptr_t)op->sym->sec_session;
 155                         build_request = ctx->build_request[proc_type];
 156                         opaque[0] = sess;
 157                         opaque[1] = (uintptr_t)build_request;
 158                 }
 159         }
 160 #endif
 161         else { /* RTE_CRYPTO_OP_SESSIONLESS */
 162                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 163                 QAT_LOG(DEBUG, "QAT does not support sessionless operation");
 164                 return -1;
 165         }
 166
 167         return build_request(op, (void *)ctx, out_msg, op_cookie);
 168 }
 169
 170 uint16_t
 171 qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 172                 uint16_t nb_ops)
 173 {
 174         return qat_enqueue_op_burst(qp, qat_sym_build_request,
 175                         (void **)ops, nb_ops);
 176 }
 177
 178 uint16_t
 179 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 180                 uint16_t nb_ops)
 181 {
 182         return qat_dequeue_op_burst(qp, (void **)ops,
 183                                 qat_sym_process_response, nb_ops);
 184 }
 185
 186 int
 187 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
 188                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 189 {
 190         int i = 0, ret = 0;
 191         struct qat_device_info *qat_dev_instance =
 192                         &qat_pci_devs[qat_pci_dev->qat_dev_id];
 193         struct rte_cryptodev_pmd_init_params init_params = {
 194                 .name = "",
 195                 .socket_id = qat_dev_instance->pci_dev->device.numa_node,
 196                 .private_data_size = sizeof(struct qat_cryptodev_private)
 197         };
 198         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 199         char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 200         struct rte_cryptodev *cryptodev;
 201         struct qat_cryptodev_private *internals;
 202         struct qat_capabilities_info capa_info;
 203         const struct rte_cryptodev_capabilities *capabilities;
 204         const struct qat_crypto_gen_dev_ops *gen_dev_ops =
 205                 &qat_sym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
 206         uint64_t capa_size;
 207
 208         snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
 209                         qat_pci_dev->name, "sym");
 210         QAT_LOG(DEBUG, "Creating QAT SYM device %s", name);
 211
 212         if (gen_dev_ops->cryptodev_ops == NULL) {
 213                 QAT_LOG(ERR, "Device %s does not support symmetric crypto",
 214                                 name);
 215                 return -(EFAULT);
 216         }
 217
 218         /*
 219          * All processes must use same driver id so they can share sessions.
 220          * Store driver_id so we can validate that all processes have the same
 221          * value, typically they have, but could differ if binaries built
 222          * separately.
 223          */
 224         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 225                 qat_pci_dev->qat_sym_driver_id =
 226                                 qat_sym_driver_id;
 227         } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 228                 if (qat_pci_dev->qat_sym_driver_id !=
 229                                 qat_sym_driver_id) {
 230                         QAT_LOG(ERR,
 231                                 "Device %s have different driver id than corresponding device in primary process",
 232                                 name);
 233                         return -(EFAULT);
 234                 }
 235         }
 236
 237         /* Populate subset device to use in cryptodev device creation */
 238         qat_dev_instance->sym_rte_dev.driver = &cryptodev_qat_sym_driver;
 239         qat_dev_instance->sym_rte_dev.numa_node =
 240                         qat_dev_instance->pci_dev->device.numa_node;
 241         qat_dev_instance->sym_rte_dev.devargs = NULL;
 242
 243         cryptodev = rte_cryptodev_pmd_create(name,
 244                         &(qat_dev_instance->sym_rte_dev), &init_params);
 245
 246         if (cryptodev == NULL)
 247                 return -ENODEV;
 248
 249         qat_dev_instance->sym_rte_dev.name = cryptodev->data->name;
 250         cryptodev->driver_id = qat_sym_driver_id;
 251         cryptodev->dev_ops = gen_dev_ops->cryptodev_ops;
 252
 253         cryptodev->enqueue_burst = qat_sym_enqueue_burst;
 254         cryptodev->dequeue_burst = qat_sym_dequeue_burst;
 255
 256         cryptodev->feature_flags = gen_dev_ops->get_feature_flags(qat_pci_dev);
 257
 258         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 259                 return 0;
 260
 261 #ifdef RTE_LIB_SECURITY
 262         if (gen_dev_ops->create_security_ctx) {
 263                 cryptodev->security_ctx =
 264                         gen_dev_ops->create_security_ctx((void *)cryptodev);
 265                 if (cryptodev->security_ctx == NULL) {
 266                         QAT_LOG(ERR, "rte_security_ctx memory alloc failed");
 267                         ret = -ENOMEM;
 268                         goto error;
 269                 }
 270
 271                 cryptodev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
 272                 QAT_LOG(INFO, "Device %s rte_security support ensabled", name);
 273         } else {
 274                 QAT_LOG(INFO, "Device %s rte_security support disabled", name);
 275         }
 276 #endif
 277         snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 278                         "QAT_SYM_CAPA_GEN_%d",
 279                         qat_pci_dev->qat_dev_gen);
 280
 281         internals = cryptodev->data->dev_private;
 282         internals->qat_dev = qat_pci_dev;
 283
 284         internals->dev_id = cryptodev->data->dev_id;
 285
 286         capa_info = gen_dev_ops->get_capabilities(qat_pci_dev);
 287         capabilities = capa_info.data;
 288         capa_size = capa_info.size;
 289
 290         internals->capa_mz = rte_memzone_lookup(capa_memz_name);
 291         if (internals->capa_mz == NULL) {
 292                 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
 293                                 capa_size, rte_socket_id(), 0);
 294                 if (internals->capa_mz == NULL) {
 295                         QAT_LOG(DEBUG,
 296                                 "Error allocating memzone for capabilities, "
 297                                 "destroying PMD for %s", name);
 298                         ret = -EFAULT;
 299                         goto error;
 300                 }
 301         }
 302
 303         memcpy(internals->capa_mz->addr, capabilities, capa_size);
 304         internals->qat_dev_capabilities = internals->capa_mz->addr;
 305
 306         while (1) {
 307                 if (qat_dev_cmd_param[i].name == NULL)
 308                         break;
 309                 if (!strcmp(qat_dev_cmd_param[i].name, SYM_ENQ_THRESHOLD_NAME))
 310                         internals->min_enq_burst_threshold =
 311                                         qat_dev_cmd_param[i].val;
 312                 i++;
 313         }
 314
 315         internals->service_type = QAT_SERVICE_SYMMETRIC;
 316         qat_pci_dev->sym_dev = internals;
 317         QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
 318                         cryptodev->data->name, internals->dev_id);
 319
 320         return 0;
 321
 322 error:
 323 #ifdef RTE_LIB_SECURITY
 324         rte_free(cryptodev->security_ctx);
 325         cryptodev->security_ctx = NULL;
 326 #endif
 327         rte_cryptodev_pmd_destroy(cryptodev);
 328         memset(&qat_dev_instance->sym_rte_dev, 0,
 329                 sizeof(qat_dev_instance->sym_rte_dev));
 330
 331         return ret;
 332 }
 333
 334 int
 335 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
 336 {
 337         struct rte_cryptodev *cryptodev;
 338
 339         if (qat_pci_dev == NULL)
 340                 return -ENODEV;
 341         if (qat_pci_dev->sym_dev == NULL)
 342                 return 0;
 343         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 344                 rte_memzone_free(qat_pci_dev->sym_dev->capa_mz);
 345
 346         /* free crypto device */
 347         cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->dev_id);
 348 #ifdef RTE_LIB_SECURITY
 349         rte_free(cryptodev->security_ctx);
 350         cryptodev->security_ctx = NULL;
 351 #endif
 352         rte_cryptodev_pmd_destroy(cryptodev);
 353         qat_pci_devs[qat_pci_dev->qat_dev_id].sym_rte_dev.name = NULL;
 354         qat_pci_dev->sym_dev = NULL;
 355
 356         return 0;
 357 }
 358
 359 int
 360 qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
 361         struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
 362         enum rte_crypto_op_sess_type sess_type,
 363         union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
 364 {
 365         struct qat_cryptodev_private *internals = dev->data->dev_private;
 366         enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
 367         struct qat_crypto_gen_dev_ops *gen_dev_ops =
 368                         &qat_sym_gen_dev_ops[qat_dev_gen];
 369         struct qat_qp *qp;
 370         struct qat_sym_session *ctx;
 371         struct qat_sym_dp_ctx *dp_ctx;
 372
 373         if (!gen_dev_ops->set_raw_dp_ctx) {
 374                 QAT_LOG(ERR, "Device GEN %u does not support raw data path",
 375                                 qat_dev_gen);
 376                 return -ENOTSUP;
 377         }
 378
 379         qp = dev->data->queue_pairs[qp_id];
 380         dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
 381
 382         if (!is_update) {
 383                 memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
 384                                 sizeof(struct qat_sym_dp_ctx));
 385                 raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
 386                 dp_ctx->tail = qp->tx_q.tail;
 387                 dp_ctx->head = qp->rx_q.head;
 388                 dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
 389         }
 390
 391         if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
 392                 return -EINVAL;
 393
 394         ctx = (struct qat_sym_session *)get_sym_session_private_data(
 395                         session_ctx.crypto_sess, qat_sym_driver_id);
 396
 397         dp_ctx->session = ctx;
 398
 399         return gen_dev_ops->set_raw_dp_ctx(raw_dp_ctx, ctx);
 400 }
 401
 402 int
 403 qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev __rte_unused)
 404 {
 405         return sizeof(struct qat_sym_dp_ctx);
 406 }
 407
 408 static struct cryptodev_driver qat_crypto_drv;
 409 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
 410                 cryptodev_qat_sym_driver,
 411                 qat_sym_driver_id);