drivers/crypto/qat/qat_sym.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2022 Intel Corporation
   3  */
   4
   5 #include <openssl/evp.h>
   6
   7 #include <rte_mempool.h>
   8 #include <rte_mbuf.h>
   9 #include <rte_crypto_sym.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_byteorder.h>
  12
  13 #include "qat_sym.h"
  14 #include "qat_crypto.h"
  15 #include "qat_qp.h"
  16
  17 uint8_t qat_sym_driver_id;
  18 int qat_ipsec_mb_lib;
  19
  20 struct qat_crypto_gen_dev_ops qat_sym_gen_dev_ops[QAT_N_GENS];
  21
  22 /* An rte_driver is needed in the registration of both the device and the driver
  23  * with cryptodev.
  24  * The actual qat pci's rte_driver can't be used as its name represents
  25  * the whole pci device with all services. Think of this as a holder for a name
  26  * for the crypto part of the pci device.
  27  */
  28 static const char qat_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD);
  29 static const struct rte_driver cryptodev_qat_sym_driver = {
  30         .name = qat_sym_drv_name,
  31         .alias = qat_sym_drv_name
  32 };
  33
  34 void
  35 qat_sym_init_op_cookie(void *op_cookie)
  36 {
  37         struct qat_sym_op_cookie *cookie = op_cookie;
  38
  39         cookie->qat_sgl_src_phys_addr =
  40                         rte_mempool_virt2iova(cookie) +
  41                         offsetof(struct qat_sym_op_cookie,
  42                         qat_sgl_src);
  43
  44         cookie->qat_sgl_dst_phys_addr =
  45                         rte_mempool_virt2iova(cookie) +
  46                         offsetof(struct qat_sym_op_cookie,
  47                         qat_sgl_dst);
  48
  49         cookie->opt.spc_gmac.cd_phys_addr =
  50                         rte_mempool_virt2iova(cookie) +
  51                         offsetof(struct qat_sym_op_cookie,
  52                         opt.spc_gmac.cd_cipher);
  53 }
  54
  55 static __rte_always_inline int
  56 qat_sym_build_request(void *in_op, uint8_t *out_msg,
  57                 void *op_cookie, uint64_t *opaque, enum qat_device_gen dev_gen)
  58 {
  59         struct rte_crypto_op *op = (struct rte_crypto_op *)in_op;
  60         uintptr_t sess = (uintptr_t)opaque[0];
  61         uintptr_t build_request_p = (uintptr_t)opaque[1];
  62         qat_sym_build_request_t build_request = (void *)build_request_p;
  63         struct qat_sym_session *ctx = NULL;
  64         enum rte_proc_type_t proc_type = rte_eal_process_type();
  65
  66         if (proc_type == RTE_PROC_AUTO || proc_type == RTE_PROC_INVALID)
  67                 return -EINVAL;
  68
  69         if (likely(op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)) {
  70                 ctx = get_sym_session_private_data(op->sym->session,
  71                                 qat_sym_driver_id);
  72                 if (unlikely(!ctx)) {
  73                         QAT_DP_LOG(ERR, "No session for this device");
  74                         return -EINVAL;
  75                 }
  76                 if (sess != (uintptr_t)ctx) {
  77                         struct rte_cryptodev *cdev;
  78                         struct qat_cryptodev_private *internals;
  79
  80                         cdev = rte_cryptodev_pmd_get_dev(ctx->dev_id);
  81                         internals = cdev->data->dev_private;
  82
  83                         if (internals->qat_dev->qat_dev_gen != dev_gen) {
  84                                 op->status =
  85                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
  86                                 return -EINVAL;
  87                         }
  88
  89                         if (unlikely(ctx->build_request[proc_type] == NULL)) {
  90                                 int ret =
  91                                 qat_sym_gen_dev_ops[dev_gen].set_session(
  92                                         (void *)cdev, (void *)sess);
  93                                 if (ret < 0) {
  94                                         op->status =
  95                                                 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
  96                                         return -EINVAL;
  97                                 }
  98                         }
  99
 100                         build_request = ctx->build_request[proc_type];
 101                         opaque[0] = (uintptr_t)ctx;
 102                         opaque[1] = (uintptr_t)build_request;
 103                 }
 104         }
 105
 106 #ifdef RTE_LIB_SECURITY
 107         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 108                 ctx = get_sec_session_private_data(op->sym->sec_session);
 109                 if (unlikely(!ctx)) {
 110                         QAT_DP_LOG(ERR, "No session for this device");
 111                         return -EINVAL;
 112                 }
 113                 if (sess != (uintptr_t)ctx) {
 114                         struct rte_cryptodev *cdev;
 115                         struct qat_cryptodev_private *internals;
 116
 117                         if (unlikely(ctx->bpi_ctx == NULL)) {
 118                                 QAT_DP_LOG(ERR, "QAT PMD only supports security"
 119                                                 " operation requests for"
 120                                                 " DOCSIS, op (%p) is not for"
 121                                                 " DOCSIS.", op);
 122                                 return -EINVAL;
 123                         } else if (unlikely(((op->sym->m_dst != NULL) &&
 124                                         (op->sym->m_dst != op->sym->m_src)) ||
 125                                         op->sym->m_src->nb_segs > 1)) {
 126                                 QAT_DP_LOG(ERR, "OOP and/or multi-segment"
 127                                                 " buffers not supported for"
 128                                                 " DOCSIS security.");
 129                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 130                                 return -EINVAL;
 131                         }
 132                         cdev = rte_cryptodev_pmd_get_dev(ctx->dev_id);
 133                         internals = cdev->data->dev_private;
 134
 135                         if (internals->qat_dev->qat_dev_gen != dev_gen) {
 136                                 op->status =
 137                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 138                                 return -EINVAL;
 139                         }
 140
 141                         if (unlikely(ctx->build_request[proc_type] == NULL)) {
 142                                 int ret =
 143                                 qat_sym_gen_dev_ops[dev_gen].set_session(
 144                                         (void *)cdev, (void *)sess);
 145                                 if (ret < 0) {
 146                                         op->status =
 147                                                 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 148                                         return -EINVAL;
 149                                 }
 150                         }
 151
 152                         sess = (uintptr_t)op->sym->sec_session;
 153                         build_request = ctx->build_request[proc_type];
 154                         opaque[0] = sess;
 155                         opaque[1] = (uintptr_t)build_request;
 156                 }
 157         }
 158 #endif
 159         else { /* RTE_CRYPTO_OP_SESSIONLESS */
 160                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 161                 QAT_LOG(DEBUG, "QAT does not support sessionless operation");
 162                 return -1;
 163         }
 164
 165         return build_request(op, (void *)ctx, out_msg, op_cookie);
 166 }
 167
 168 uint16_t
 169 qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
 170                 uint16_t nb_ops)
 171 {
 172         return qat_enqueue_op_burst(qp, qat_sym_build_request,
 173                         (void **)ops, nb_ops);
 174 }
 175
 176 uint16_t
 177 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 178                 uint16_t nb_ops)
 179 {
 180         return qat_dequeue_op_burst(qp, (void **)ops,
 181                                 qat_sym_process_response, nb_ops);
 182 }
 183
 184 int
 185 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
 186                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 187 {
 188         int i = 0, ret = 0;
 189         struct qat_device_info *qat_dev_instance =
 190                         &qat_pci_devs[qat_pci_dev->qat_dev_id];
 191         struct rte_cryptodev_pmd_init_params init_params = {
 192                 .name = "",
 193                 .socket_id = qat_dev_instance->pci_dev->device.numa_node,
 194                 .private_data_size = sizeof(struct qat_cryptodev_private)
 195         };
 196         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 197         char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 198         struct rte_cryptodev *cryptodev;
 199         struct qat_cryptodev_private *internals;
 200         struct qat_capabilities_info capa_info;
 201         const struct rte_cryptodev_capabilities *capabilities;
 202         const struct qat_crypto_gen_dev_ops *gen_dev_ops =
 203                 &qat_sym_gen_dev_ops[qat_pci_dev->qat_dev_gen];
 204         uint64_t capa_size;
 205
 206         snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
 207                         qat_pci_dev->name, "sym");
 208         QAT_LOG(DEBUG, "Creating QAT SYM device %s", name);
 209
 210         if (gen_dev_ops->cryptodev_ops == NULL) {
 211                 QAT_LOG(ERR, "Device %s does not support symmetric crypto",
 212                                 name);
 213                 return -(EFAULT);
 214         }
 215
 216         /*
 217          * All processes must use same driver id so they can share sessions.
 218          * Store driver_id so we can validate that all processes have the same
 219          * value, typically they have, but could differ if binaries built
 220          * separately.
 221          */
 222         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 223                 qat_pci_dev->qat_sym_driver_id =
 224                                 qat_sym_driver_id;
 225         } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 226                 if (qat_pci_dev->qat_sym_driver_id !=
 227                                 qat_sym_driver_id) {
 228                         QAT_LOG(ERR,
 229                                 "Device %s have different driver id than corresponding device in primary process",
 230                                 name);
 231                         return -(EFAULT);
 232                 }
 233         }
 234
 235         /* Populate subset device to use in cryptodev device creation */
 236         qat_dev_instance->sym_rte_dev.driver = &cryptodev_qat_sym_driver;
 237         qat_dev_instance->sym_rte_dev.numa_node =
 238                         qat_dev_instance->pci_dev->device.numa_node;
 239         qat_dev_instance->sym_rte_dev.devargs = NULL;
 240
 241         cryptodev = rte_cryptodev_pmd_create(name,
 242                         &(qat_dev_instance->sym_rte_dev), &init_params);
 243
 244         if (cryptodev == NULL)
 245                 return -ENODEV;
 246
 247         qat_dev_instance->sym_rte_dev.name = cryptodev->data->name;
 248         cryptodev->driver_id = qat_sym_driver_id;
 249         cryptodev->dev_ops = gen_dev_ops->cryptodev_ops;
 250
 251         cryptodev->enqueue_burst = qat_sym_enqueue_burst;
 252         cryptodev->dequeue_burst = qat_sym_dequeue_burst;
 253
 254         cryptodev->feature_flags = gen_dev_ops->get_feature_flags(qat_pci_dev);
 255
 256         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 257                 return 0;
 258
 259 #ifdef RTE_LIB_SECURITY
 260         if (gen_dev_ops->create_security_ctx) {
 261                 cryptodev->security_ctx =
 262                         gen_dev_ops->create_security_ctx((void *)cryptodev);
 263                 if (cryptodev->security_ctx == NULL) {
 264                         QAT_LOG(ERR, "rte_security_ctx memory alloc failed");
 265                         ret = -ENOMEM;
 266                         goto error;
 267                 }
 268
 269                 cryptodev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY;
 270                 QAT_LOG(INFO, "Device %s rte_security support ensabled", name);
 271         } else {
 272                 QAT_LOG(INFO, "Device %s rte_security support disabled", name);
 273         }
 274 #endif
 275         snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 276                         "QAT_SYM_CAPA_GEN_%d",
 277                         qat_pci_dev->qat_dev_gen);
 278
 279         internals = cryptodev->data->dev_private;
 280         internals->qat_dev = qat_pci_dev;
 281
 282         internals->dev_id = cryptodev->data->dev_id;
 283
 284         capa_info = gen_dev_ops->get_capabilities(qat_pci_dev);
 285         capabilities = capa_info.data;
 286         capa_size = capa_info.size;
 287
 288         internals->capa_mz = rte_memzone_lookup(capa_memz_name);
 289         if (internals->capa_mz == NULL) {
 290                 internals->capa_mz = rte_memzone_reserve(capa_memz_name,
 291                                 capa_size, rte_socket_id(), 0);
 292                 if (internals->capa_mz == NULL) {
 293                         QAT_LOG(DEBUG,
 294                                 "Error allocating memzone for capabilities, "
 295                                 "destroying PMD for %s", name);
 296                         ret = -EFAULT;
 297                         goto error;
 298                 }
 299         }
 300
 301         memcpy(internals->capa_mz->addr, capabilities, capa_size);
 302         internals->qat_dev_capabilities = internals->capa_mz->addr;
 303
 304         while (1) {
 305                 if (qat_dev_cmd_param[i].name == NULL)
 306                         break;
 307                 if (!strcmp(qat_dev_cmd_param[i].name, SYM_ENQ_THRESHOLD_NAME))
 308                         internals->min_enq_burst_threshold =
 309                                         qat_dev_cmd_param[i].val;
 310                 if (!strcmp(qat_dev_cmd_param[i].name, QAT_IPSEC_MB_LIB))
 311                         qat_ipsec_mb_lib = qat_dev_cmd_param[i].val;
 312                 i++;
 313         }
 314
 315         internals->service_type = QAT_SERVICE_SYMMETRIC;
 316         qat_pci_dev->sym_dev = internals;
 317         QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
 318                         cryptodev->data->name, internals->dev_id);
 319
 320         return 0;
 321
 322 error:
 323 #ifdef RTE_LIB_SECURITY
 324         rte_free(cryptodev->security_ctx);
 325         cryptodev->security_ctx = NULL;
 326 #endif
 327         rte_cryptodev_pmd_destroy(cryptodev);
 328         memset(&qat_dev_instance->sym_rte_dev, 0,
 329                 sizeof(qat_dev_instance->sym_rte_dev));
 330
 331         return ret;
 332 }
 333
 334 int
 335 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
 336 {
 337         struct rte_cryptodev *cryptodev;
 338
 339         if (qat_pci_dev == NULL)
 340                 return -ENODEV;
 341         if (qat_pci_dev->sym_dev == NULL)
 342                 return 0;
 343         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 344                 rte_memzone_free(qat_pci_dev->sym_dev->capa_mz);
 345
 346         /* free crypto device */
 347         cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->dev_id);
 348 #ifdef RTE_LIB_SECURITY
 349         rte_free(cryptodev->security_ctx);
 350         cryptodev->security_ctx = NULL;
 351 #endif
 352         rte_cryptodev_pmd_destroy(cryptodev);
 353         qat_pci_devs[qat_pci_dev->qat_dev_id].sym_rte_dev.name = NULL;
 354         qat_pci_dev->sym_dev = NULL;
 355
 356         return 0;
 357 }
 358
 359 int
 360 qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
 361         struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
 362         enum rte_crypto_op_sess_type sess_type,
 363         union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
 364 {
 365         struct qat_cryptodev_private *internals = dev->data->dev_private;
 366         enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
 367         struct qat_crypto_gen_dev_ops *gen_dev_ops =
 368                         &qat_sym_gen_dev_ops[qat_dev_gen];
 369         struct qat_qp *qp;
 370         struct qat_sym_session *ctx;
 371         struct qat_sym_dp_ctx *dp_ctx;
 372
 373         if (!gen_dev_ops->set_raw_dp_ctx) {
 374                 QAT_LOG(ERR, "Device GEN %u does not support raw data path",
 375                                 qat_dev_gen);
 376                 return -ENOTSUP;
 377         }
 378
 379         qp = dev->data->queue_pairs[qp_id];
 380         dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
 381
 382         if (!is_update) {
 383                 memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
 384                                 sizeof(struct qat_sym_dp_ctx));
 385                 raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
 386                 dp_ctx->tail = qp->tx_q.tail;
 387                 dp_ctx->head = qp->rx_q.head;
 388                 dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
 389         }
 390
 391         if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
 392                 return -EINVAL;
 393
 394         ctx = (struct qat_sym_session *)get_sym_session_private_data(
 395                         session_ctx.crypto_sess, qat_sym_driver_id);
 396
 397         dp_ctx->session = ctx;
 398
 399         return gen_dev_ops->set_raw_dp_ctx(raw_dp_ctx, ctx);
 400 }
 401
 402 int
 403 qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev __rte_unused)
 404 {
 405         return sizeof(struct qat_sym_dp_ctx);
 406 }
 407
 408 static struct cryptodev_driver qat_crypto_drv;
 409 RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
 410                 cryptodev_qat_sym_driver,
 411                 qat_sym_driver_id);