drivers/crypto/qat/qat_sym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2022 Intel Corporation
   3  */
   4
   5 #include <openssl/evp.h>
   6
   7 #include <rte_mempool.h>
   8 #include <rte_mbuf.h>
   9 #include <rte_crypto_sym.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_byteorder.h>
  12
  13 #include "qat_sym.h"
  14 #include "qat_crypto.h"
  15 #include "qat_qp.h"
  16
  17 uint8_t qat_sym_driver_id;
  18
  19 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
  20
  21 /* An rte_driver is needed in the registration of both the device and the driver
  22  * with cryptodev.
  23  * The actual qat pci's rte_driver can't be used as its name represents
  24  * the whole pci device with all services. Think of this as a holder for a name
  25  * for the crypto part of the pci device.
  26  */
  27 static const char qat_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD);
  28 static const struct rte_driver cryptodev_qat_sym_driver = {
  29         .name = qat_sym_drv_name,
  30         .alias = qat_sym_drv_name
  31 };
  32
  33 void
  34 qat_sym_init_op_cookie(void *op_cookie)
  35 {
  36         struct qat_sym_op_cookie *cookie = op_cookie;
  37
  38         cookie->qat_sgl_src_phys_addr =
  39                         rte_mempool_virt2iova(cookie) +
  40                         offsetof(struct qat_sym_op_cookie,
  41                         qat_sgl_src);
  42
  43         cookie->qat_sgl_dst_phys_addr =
  44                         rte_mempool_virt2iova(cookie) +
  45                         offsetof(struct qat_sym_op_cookie,
  46                         qat_sgl_dst);
  47
  48         cookie->opt.spc_gmac.cd_phys_addr =
  49                         rte_mempool_virt2iova(cookie) +
  50                         offsetof(struct qat_sym_op_cookie,
  51                         opt.spc_gmac.cd_cipher);
  52 }
  53
  54 static __rte_always_inline int
  55 qat_sym_build_request(void *in_op, uint8_t *out_msg,
  56                 void *op_cookie, uint64_t *opaque, enum qat_device_gen dev_gen)
  57 {
  58         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
  59         uintptr_t sess = (uintptr_t)opaque[0];
  60         uintptr_t build_request_p = (uintptr_t)opaque[1];
  61         qat_sym_build_request_t build_request = (void *)build_request_p;
  62         struct qat_sym_session *ctx = NULL;
  63
  64         if (likely(op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)) {
  65                 ctx = get_sym_session_private_data(op->sym->session,
  66                                 qat_sym_driver_id);
  67                 if (unlikely(!ctx)) {
  68                         QAT_DP_LOG(ERR, "No session for this device");
  69                         return -EINVAL;
  70                 }
  71                 if (sess != (uintptr_t)ctx) {
  72                         struct rte_cryptodev *cdev;
  73                         struct qat_cryptodev_private *internals;
  74                         enum rte_proc_type_t proc_type;
  75
  76                         cdev = rte_cryptodev_pmd_get_dev(ctx->dev_id);
  77                         internals = cdev->data->dev_private;
  78                         proc_type = rte_eal_process_type();
  79
  80                         if (internals->qat_dev->qat_dev_gen != dev_gen) {
  81                                 op->status =
  82                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
  83                                 return -EINVAL;
  84                         }
  85
  86                         if (unlikely(ctx->build_request[proc_type] == NULL)) {
  87                                 int ret =
  88                                 qat_sym_gen_dev_ops[dev_gen].set_session(
  89                                         (void *)cdev, (void *)sess);
  90                                 if (ret < 0) {
  91                                         op->status =
  92                                                 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
  93                                         return -EINVAL;
  94                                 }
  95                         }
  96
  97                         build_request = ctx->build_request[proc_type];
  98                         opaque[0] = (uintptr_t)ctx;
  99                         opaque[1] = (uintptr_t)build_request;
 100                 }
 101         }
 102
 103 #ifdef RTE_LIB_SECURITY
 104         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 105                 if ((void *)sess != (void *)op->sym->sec_session) {
 106                         struct rte_cryptodev *cdev;
 107                         struct qat_cryptodev_private *internals;
 108                         enum rte_proc_type_t proc_type;
 109
 110                         ctx = get_sec_session_private_data(
 111                                         op->sym->sec_session);
 112                         if (unlikely(!ctx)) {
 113                                 QAT_DP_LOG(ERR, "No session for this device");
 114                                 return -EINVAL;
 115                         }
 116                         if (unlikely(ctx->bpi_ctx == NULL)) {
 117                                 QAT_DP_LOG(ERR, "QAT PMD only supports security"
 118                                                 " operation requests for"
 119                                                 " DOCSIS, op (%p) is not for"
 120                                                 " DOCSIS.", op);
 121                                 return -EINVAL;
 122                         } else if (unlikely(((op->sym->m_dst != NULL) &&
 123                                         (op->sym->m_dst != op->sym->m_src)) ||
 124                                         op->sym->m_src->nb_segs > 1)) {
 125                                 QAT_DP_LOG(ERR, "OOP and/or multi-segment"
 126                                                 " buffers not supported for"
 127                                                 " DOCSIS security.");
 128                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 129                                 return -EINVAL;
 130                         }
 131                         cdev = rte_cryptodev_pmd_get_dev(ctx->dev_id);
 132                         internals = cdev->data->dev_private;
 133                         proc_type = rte_eal_process_type();
 134
 135                         if (internals->qat_dev->qat_dev_gen != dev_gen) {
 136                                 op->status =
 137                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 138                                 return -EINVAL;
 139                         }
 140
 141                         if (unlikely(ctx->build_request[proc_type] == NULL)) {
 142                                 int ret =
 143                                 qat_sym_gen_dev_ops[dev_gen].set_session(
 144                                         (void *)cdev, (void *)sess);
 145                                 if (ret < 0) {
 146                                         op->status =
 147                                                 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 148                                         return -EINVAL;
 149                                 }
 150                         }
 151
 152                         sess = (uintptr_t)op->sym->sec_session;
 153                         build_request = ctx->build_request[proc_type];
 154                         opaque[0] = sess;
 155                         opaque[1] = (uintptr_t)build_request;
 156                 }
 157         }
 158 #endif
 159         else { /* RTE_CRYPTO_OP_SESSIONLESS */
 160                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 161                 QAT_LOG(DEBUG, "QAT does not support sessionless operation");
 162                 return -1;
 163         }
 164
 165         return build_request(op, (void *)ctx, out_msg, op_cookie);
 166 }
 167
 168 uint16_t
 169 qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 170                 uint16_t nb_ops)
 171 {
 172         return qat_enqueue_op_burst(qp, qat_sym_build_request,
 173                         (void **)ops, nb_ops);
 174 }
 175
 176 uint16_t
 177 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 178                 uint16_t nb_ops)
 179 {
 180         return qat_dequeue_op_burst(qp, (void **)ops,
 181                                 qat_sym_process_response, nb_ops);
 182 }
 183
 184 int
 185 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
 186                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 187 {
 188         int i = 0, ret = 0;
 189         struct qat_device_info *qat_dev_instance =
 190                         &qat_pci_devs[qat_pci_dev->qat_dev_id];
 191         struct rte_cryptodev_pmd_init_params init_params = {
 192                 .name = "",
 193                 .socket_id = qat_dev_instance->pci_dev->device.numa_node,
 194                 .private_data_size = sizeof(struct qat_cryptodev_private)
 195         };
 196         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 197         char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 198         struct rte_cryptodev *cryptodev;
 199         struct qat_cryptodev_private *internals;
 200         struct qat_capabilities_info capa_info;
 201         const struct rte_cryptodev_capabilities *capabilities;
 202         const struct qat_crypto_gen_dev_ops *gen_dev_ops =
 203                 &qat_sym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
 204         uint64_t capa_size;
 205
 206         snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
 207                         qat_pci_dev->name, "sym");
 208         QAT_LOG(DEBUG, "Creating QAT SYM device %s", name);
 209
 210         if (gen_dev_ops->cryptodev_ops == NULL) {
 211                 QAT_LOG(ERR, "Device %s does not support symmetric crypto",
 212                                 name);
 213                 return -(EFAULT);
 214         }
 215
 216         /*
 217          * All processes must use same driver id so they can share sessions.
 218          * Store driver_id so we can validate that all processes have the same
 219          * value, typically they have, but could differ if binaries built
 220          * separately.
 221          */
 222         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 223                 qat_pci_dev->qat_sym_driver_id =
 224                                 qat_sym_driver_id;
 225         } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 226                 if (qat_pci_dev->qat_sym_driver_id !=
 227                                 qat_sym_driver_id) {
 228                         QAT_LOG(ERR,
 229                                 "Device %s have different driver id than corresponding device in primary process",
 230                                 name);
 231                         return -(EFAULT);
 232                 }
 233         }
 234
 235         /* Populate subset device to use in cryptodev device creation */
 236         qat_dev_instance->sym_rte_dev.driver = &cryptodev_qat_sym_driver;
 237         qat_dev_instance->sym_rte_dev.numa_node =
 238                         qat_dev_instance->pci_dev->device.numa_node;
 239         qat_dev_instance->sym_rte_dev.devargs = NULL;
 240
 241         cryptodev = rte_cryptodev_pmd_create(name,
 242                         &(qat_dev_instance->sym_rte_dev), &init_params);
 243
 244         if (cryptodev == NULL)
 245                 return -ENODEV;
 246
 247         qat_dev_instance->sym_rte_dev.name = cryptodev->data->name;
 248         cryptodev->driver_id = qat_sym_driver_id;
 249         cryptodev->dev_ops = gen_dev_ops->cryptodev_ops;
 250
 251         cryptodev->enqueue_burst = qat_sym_enqueue_burst;
 252         cryptodev->dequeue_burst = qat_sym_dequeue_burst;
 253
 254         cryptodev->feature_flags = gen_dev_ops->get_feature_flags(qat_pci_dev);
 255
 256         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 257                 return 0;
 258
 259 #ifdef RTE_LIB_SECURITY
 260         if (gen_dev_ops->create_security_ctx) {
 261                 cryptodev->security_ctx =
 262                         gen_dev_ops->create_security_ctx((void *)cryptodev);
 263                 if (cryptodev->security_ctx == NULL) {
 264                         QAT_LOG(ERR, "rte_security_ctx memory alloc failed");
 265                         ret = -ENOMEM;
 266                         goto error;
 267                 }
 268
 269                 cryptodev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
 270                 QAT_LOG(INFO, "Device %s rte_security support ensabled", name);
 271         } else {
 272                 QAT_LOG(INFO, "Device %s rte_security support disabled", name);
 273         }
 274 #endif
 275         snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 276                         "QAT_SYM_CAPA_GEN_%d",
 277                         qat_pci_dev->qat_dev_gen);
 278
 279         internals = cryptodev->data->dev_private;
 280         internals->qat_dev = qat_pci_dev;
 281
 282         internals->dev_id = cryptodev->data->dev_id;
 283
 284         capa_info = gen_dev_ops->get_capabilities(qat_pci_dev);
 285         capabilities = capa_info.data;
 286         capa_size = capa_info.size;
 287
 288         internals->capa_mz = rte_memzone_lookup(capa_memz_name);
 289         if (internals->capa_mz == NULL) {
 290                 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
 291                                 capa_size, rte_socket_id(), 0);
 292                 if (internals->capa_mz == NULL) {
 293                         QAT_LOG(DEBUG,
 294                                 "Error allocating memzone for capabilities, "
 295                                 "destroying PMD for %s", name);
 296                         ret = -EFAULT;
 297                         goto error;
 298                 }
 299         }
 300
 301         memcpy(internals->capa_mz->addr, capabilities, capa_size);
 302         internals->qat_dev_capabilities = internals->capa_mz->addr;
 303
 304         while (1) {
 305                 if (qat_dev_cmd_param[i].name == NULL)
 306                         break;
 307                 if (!strcmp(qat_dev_cmd_param[i].name, SYM_ENQ_THRESHOLD_NAME))
 308                         internals->min_enq_burst_threshold =
 309                                         qat_dev_cmd_param[i].val;
 310                 i++;
 311         }
 312
 313         internals->service_type = QAT_SERVICE_SYMMETRIC;
 314         qat_pci_dev->sym_dev = internals;
 315         QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
 316                         cryptodev->data->name, internals->dev_id);
 317
 318         return 0;
 319
 320 error:
 321 #ifdef RTE_LIB_SECURITY
 322         rte_free(cryptodev->security_ctx);
 323         cryptodev->security_ctx = NULL;
 324 #endif
 325         rte_cryptodev_pmd_destroy(cryptodev);
 326         memset(&qat_dev_instance->sym_rte_dev, 0,
 327                 sizeof(qat_dev_instance->sym_rte_dev));
 328
 329         return ret;
 330 }
 331
 332 int
 333 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
 334 {
 335         struct rte_cryptodev *cryptodev;
 336
 337         if (qat_pci_dev == NULL)
 338                 return -ENODEV;
 339         if (qat_pci_dev->sym_dev == NULL)
 340                 return 0;
 341         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 342                 rte_memzone_free(qat_pci_dev->sym_dev->capa_mz);
 343
 344         /* free crypto device */
 345         cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->dev_id);
 346 #ifdef RTE_LIB_SECURITY
 347         rte_free(cryptodev->security_ctx);
 348         cryptodev->security_ctx = NULL;
 349 #endif
 350         rte_cryptodev_pmd_destroy(cryptodev);
 351         qat_pci_devs[qat_pci_dev->qat_dev_id].sym_rte_dev.name = NULL;
 352         qat_pci_dev->sym_dev = NULL;
 353
 354         return 0;
 355 }
 356
 357 int
 358 qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
 359         struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
 360         enum rte_crypto_op_sess_type sess_type,
 361         union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
 362 {
 363         struct qat_cryptodev_private *internals = dev->data->dev_private;
 364         enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
 365         struct qat_crypto_gen_dev_ops *gen_dev_ops =
 366                         &qat_sym_gen_dev_ops[qat_dev_gen];
 367         struct qat_qp *qp;
 368         struct qat_sym_session *ctx;
 369         struct qat_sym_dp_ctx *dp_ctx;
 370
 371         if (!gen_dev_ops->set_raw_dp_ctx) {
 372                 QAT_LOG(ERR, "Device GEN %u does not support raw data path",
 373                                 qat_dev_gen);
 374                 return -ENOTSUP;
 375         }
 376
 377         qp = dev->data->queue_pairs[qp_id];
 378         dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
 379
 380         if (!is_update) {
 381                 memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
 382                                 sizeof(struct qat_sym_dp_ctx));
 383                 raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
 384                 dp_ctx->tail = qp->tx_q.tail;
 385                 dp_ctx->head = qp->rx_q.head;
 386                 dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
 387         }
 388
 389         if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
 390                 return -EINVAL;
 391
 392         ctx = (struct qat_sym_session *)get_sym_session_private_data(
 393                         session_ctx.crypto_sess, qat_sym_driver_id);
 394
 395         dp_ctx->session = ctx;
 396
 397         return gen_dev_ops->set_raw_dp_ctx(raw_dp_ctx, ctx);
 398 }
 399
 400 int
 401 qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev __rte_unused)
 402 {
 403         return sizeof(struct qat_sym_dp_ctx);
 404 }
 405
 406 static struct cryptodev_driver qat_crypto_drv;
 407 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
 408                 cryptodev_qat_sym_driver,
 409                 qat_sym_driver_id);