X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fqat%2Fqat_sym_pmd.c;h=c7e323cceb6f820d177896ba72492b9ef51afc0a;hb=faa57df0b458d325aa90bc8e0de67987e8acc095;hp=e6760b8f84dbf4a8c334c1fa54619ee2ad0c154f;hpb=850d2304ee0527856cfebe07404715b7a8bd6c10;p=dpdk.git diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c index e6760b8f84..c7e323cceb 100644 --- a/drivers/crypto/qat/qat_sym_pmd.c +++ b/drivers/crypto/qat/qat_sym_pmd.c @@ -8,13 +8,18 @@ #include #include #include +#ifdef RTE_LIBRTE_SECURITY +#include +#endif #include "qat_logs.h" #include "qat_sym.h" #include "qat_sym_session.h" #include "qat_sym_pmd.h" -uint8_t cryptodev_qat_driver_id; +#define MIXED_CRYPTO_MIN_FW_VER 0x04090000 + +uint8_t qat_sym_driver_id; static const struct rte_cryptodev_capabilities qat_gen1_sym_capabilities[] = { QAT_BASE_GEN1_SYM_CAPABILITIES, @@ -27,33 +32,51 @@ static const struct rte_cryptodev_capabilities qat_gen2_sym_capabilities[] = { RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; +static const struct rte_cryptodev_capabilities qat_gen3_sym_capabilities[] = { + QAT_BASE_GEN1_SYM_CAPABILITIES, + QAT_EXTRA_GEN2_SYM_CAPABILITIES, + QAT_EXTRA_GEN3_SYM_CAPABILITIES, + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + +#ifdef RTE_LIBRTE_SECURITY +static const struct rte_cryptodev_capabilities + qat_security_sym_capabilities[] = { + QAT_SECURITY_SYM_CAPABILITIES, + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + +static const struct rte_security_capability qat_security_capabilities[] = { + QAT_SECURITY_CAPABILITIES(qat_security_sym_capabilities), + { + .action = RTE_SECURITY_ACTION_TYPE_NONE + } +}; +#endif + static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id); static int qat_sym_dev_config(__rte_unused struct rte_cryptodev *dev, __rte_unused struct rte_cryptodev_config *config) { - PMD_INIT_FUNC_TRACE(); return 0; } static int qat_sym_dev_start(__rte_unused struct rte_cryptodev *dev) { - PMD_INIT_FUNC_TRACE(); return 0; } static void qat_sym_dev_stop(__rte_unused struct rte_cryptodev *dev) { - PMD_INIT_FUNC_TRACE(); + return; } static int qat_sym_dev_close(struct rte_cryptodev *dev) { int i, ret; - PMD_INIT_FUNC_TRACE(); - for (i = 0; i < dev->data->nb_queue_pairs; i++) { ret = qat_sym_qp_release(dev, i); if (ret < 0) @@ -71,15 +94,14 @@ static void qat_sym_dev_info_get(struct rte_cryptodev *dev, qat_gen_config[internals->qat_dev->qat_dev_gen] .qp_hw_data[QAT_SERVICE_SYMMETRIC]; - PMD_INIT_FUNC_TRACE(); if (info != NULL) { info->max_nb_queue_pairs = qat_qps_per_service(sym_hw_qps, QAT_SERVICE_SYMMETRIC); info->feature_flags = dev->feature_flags; info->capabilities = internals->qat_dev_capabilities; - info->sym.max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS; - info->driver_id = cryptodev_qat_driver_id; - info->pci_dev = RTE_DEV_TO_PCI(dev->device); + info->driver_id = qat_sym_driver_id; + /* No limit of number of sessions */ + info->sym.max_nb_sessions = 0; } } @@ -90,7 +112,7 @@ static void qat_sym_stats_get(struct rte_cryptodev *dev, struct qat_sym_dev_private *qat_priv; if (stats == NULL || dev == NULL) { - PMD_DRV_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); + QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); return; } qat_priv = dev->data->dev_private; @@ -107,7 +129,7 @@ static void qat_sym_stats_reset(struct rte_cryptodev *dev) struct qat_sym_dev_private *qat_priv; if (dev == NULL) { - PMD_DRV_LOG(ERR, "invalid cryptodev ptr %p", dev); + QAT_LOG(ERR, "invalid cryptodev ptr %p", dev); return; } qat_priv = dev->data->dev_private; @@ -120,7 +142,7 @@ static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) { struct qat_sym_dev_private *qat_private = dev->data->dev_private; - PMD_DRV_LOG(DEBUG, "Release sym qp %u on device %d", + QAT_LOG(DEBUG, "Release sym qp %u on device %d", queue_pair_id, dev->data->dev_id); qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][queue_pair_id] @@ -132,7 +154,7 @@ static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id, struct rte_mempool *session_pool __rte_unused) + int socket_id) { struct qat_qp *qp; int ret = 0; @@ -154,13 +176,12 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, return ret; } if (qp_id >= qat_qps_per_service(sym_hw_qps, QAT_SERVICE_SYMMETRIC)) { - PMD_DRV_LOG(ERR, "qp_id %u invalid for this device", qp_id); + QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id); return -EINVAL; } qat_qp_conf.hw = qp_hw_data; qat_qp_conf.build_request = qat_sym_build_request; - qat_qp_conf.process_response = qat_sym_process_response; qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie); qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors; qat_qp_conf.socket_id = socket_id; @@ -175,6 +196,7 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, = *qp_addr; qp = (struct qat_qp *)*qp_addr; + qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold; for (i = 0; i < qp->nb_descriptors; i++) { @@ -192,6 +214,31 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, qat_sgl_dst); } + /* Get fw version from QAT (GEN2), skip if we've got it already */ + if (qp->qat_dev_gen == QAT_GEN2 && !(qat_private->internal_capabilities + & QAT_SYM_CAP_VALID)) { + ret = qat_cq_get_fw_version(qp); + + if (ret < 0) { + qat_sym_qp_release(dev, qp_id); + return ret; + } + + if (ret != 0) + QAT_LOG(DEBUG, "QAT firmware version: %d.%d.%d", + (ret >> 24) & 0xff, + (ret >> 16) & 0xff, + (ret >> 8) & 0xff); + else + QAT_LOG(DEBUG, "unknown QAT firmware version"); + + /* set capabilities based on the fw version */ + qat_private->internal_capabilities = QAT_SYM_CAP_VALID | + ((ret >= MIXED_CRYPTO_MIN_FW_VER) ? + QAT_SYM_CAP_MIXED_CRYPTO : 0); + ret = 0; + } + return ret; } @@ -208,15 +255,30 @@ static struct rte_cryptodev_ops crypto_qat_ops = { .stats_reset = qat_sym_stats_reset, .queue_pair_setup = qat_sym_qp_setup, .queue_pair_release = qat_sym_qp_release, - .queue_pair_start = NULL, - .queue_pair_stop = NULL, - .queue_pair_count = NULL, /* Crypto related operations */ - .session_get_size = qat_sym_session_get_private_size, - .session_configure = qat_sym_session_configure, - .session_clear = qat_sym_session_clear + .sym_session_get_size = qat_sym_session_get_private_size, + .sym_session_configure = qat_sym_session_configure, + .sym_session_clear = qat_sym_session_clear +}; + +#ifdef RTE_LIBRTE_SECURITY +static const struct rte_security_capability * +qat_security_cap_get(void *device __rte_unused) +{ + return qat_security_capabilities; +} + +static struct rte_security_ops security_qat_ops = { + + .session_create = qat_security_session_create, + .session_update = NULL, + .session_stats_get = NULL, + .session_destroy = qat_security_session_destroy, + .set_pkt_metadata = NULL, + .capabilities_get = qat_security_cap_get }; +#endif static uint16_t qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, @@ -232,30 +294,80 @@ qat_sym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, return qat_dequeue_op_burst(qp, (void **)ops, nb_ops); } +/* An rte_driver is needed in the registration of both the device and the driver + * with cryptodev. + * The actual qat pci's rte_driver can't be used as its name represents + * the whole pci device with all services. Think of this as a holder for a name + * for the crypto part of the pci device. + */ +static const char qat_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD); +static const struct rte_driver cryptodev_qat_sym_driver = { + .name = qat_sym_drv_name, + .alias = qat_sym_drv_name +}; + int -qat_sym_dev_create(struct qat_pci_device *qat_pci_dev) +qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, + struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused) { + int i = 0; + struct qat_device_info *qat_dev_instance = + &qat_pci_devs[qat_pci_dev->qat_dev_id]; + struct rte_cryptodev_pmd_init_params init_params = { .name = "", - .socket_id = qat_pci_dev->pci_dev->device.numa_node, - .private_data_size = sizeof(struct qat_sym_dev_private), - .max_nb_sessions = RTE_QAT_PMD_MAX_NB_SESSIONS + .socket_id = + qat_dev_instance->pci_dev->device.numa_node, + .private_data_size = sizeof(struct qat_sym_dev_private) }; char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct rte_cryptodev *cryptodev; struct qat_sym_dev_private *internals; + const struct rte_cryptodev_capabilities *capabilities; + uint64_t capa_size; + + /* + * All processes must use same driver id so they can share sessions. + * Store driver_id so we can validate that all processes have the same + * value, typically they have, but could differ if binaries built + * separately. + */ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + qat_pci_dev->qat_sym_driver_id = + qat_sym_driver_id; + } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + if (qat_pci_dev->qat_sym_driver_id != + qat_sym_driver_id) { + QAT_LOG(ERR, + "Device %s have different driver id than corresponding device in primary process", + name); + return -(EFAULT); + } + } + +#ifdef RTE_LIBRTE_SECURITY + struct rte_security_ctx *security_instance; +#endif snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s", qat_pci_dev->name, "sym"); - PMD_DRV_LOG(DEBUG, "Creating QAT SYM device %s", name); + QAT_LOG(DEBUG, "Creating QAT SYM device %s", name); + + /* Populate subset device to use in cryptodev device creation */ + qat_dev_instance->sym_rte_dev.driver = &cryptodev_qat_sym_driver; + qat_dev_instance->sym_rte_dev.numa_node = + qat_dev_instance->pci_dev->device.numa_node; + qat_dev_instance->sym_rte_dev.devargs = NULL; cryptodev = rte_cryptodev_pmd_create(name, - &qat_pci_dev->pci_dev->device, &init_params); + &(qat_dev_instance->sym_rte_dev), &init_params); if (cryptodev == NULL) return -ENODEV; - cryptodev->driver_id = cryptodev_qat_driver_id; + qat_dev_instance->sym_rte_dev.name = cryptodev->data->name; + cryptodev->driver_id = qat_sym_driver_id; cryptodev->dev_ops = &crypto_qat_ops; cryptodev->enqueue_burst = qat_sym_pmd_enqueue_op_burst; @@ -264,30 +376,96 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev) cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | - RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; + RTE_CRYPTODEV_FF_IN_PLACE_SGL | + RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | + RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | + RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | + RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED | + RTE_CRYPTODEV_FF_SECURITY; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN, + "QAT_SYM_CAPA_GEN_%d", + qat_pci_dev->qat_dev_gen); + +#ifdef RTE_LIBRTE_SECURITY + security_instance = rte_malloc("qat_sec", + sizeof(struct rte_security_ctx), + RTE_CACHE_LINE_SIZE); + if (security_instance == NULL) { + QAT_LOG(ERR, "rte_security_ctx memory alloc failed"); + rte_cryptodev_pmd_destroy(cryptodev); + return -ENOMEM; + } + + security_instance->device = (void *)cryptodev; + security_instance->ops = &security_qat_ops; + security_instance->sess_cnt = 0; + cryptodev->security_ctx = security_instance; +#endif internals = cryptodev->data->dev_private; internals->qat_dev = qat_pci_dev; - qat_pci_dev->sym_dev = internals; internals->sym_dev_id = cryptodev->data->dev_id; switch (qat_pci_dev->qat_dev_gen) { case QAT_GEN1: - internals->qat_dev_capabilities = qat_gen1_sym_capabilities; + capabilities = qat_gen1_sym_capabilities; + capa_size = sizeof(qat_gen1_sym_capabilities); break; case QAT_GEN2: - internals->qat_dev_capabilities = qat_gen2_sym_capabilities; + capabilities = qat_gen2_sym_capabilities; + capa_size = sizeof(qat_gen2_sym_capabilities); break; - default: - internals->qat_dev_capabilities = qat_gen2_sym_capabilities; - PMD_DRV_LOG(DEBUG, - "QAT gen %d capabilities unknown, default to GEN2", - qat_pci_dev->qat_dev_gen); + case QAT_GEN3: + capabilities = qat_gen3_sym_capabilities; + capa_size = sizeof(qat_gen3_sym_capabilities); break; + default: + QAT_LOG(DEBUG, + "QAT gen %d capabilities unknown", + qat_pci_dev->qat_dev_gen); + rte_cryptodev_pmd_destroy(cryptodev); + memset(&qat_dev_instance->sym_rte_dev, 0, + sizeof(qat_dev_instance->sym_rte_dev)); + return -(EINVAL); + } + + internals->capa_mz = rte_memzone_lookup(capa_memz_name); + if (internals->capa_mz == NULL) { + internals->capa_mz = rte_memzone_reserve(capa_memz_name, + capa_size, + rte_socket_id(), 0); + } + if (internals->capa_mz == NULL) { + QAT_LOG(DEBUG, + "Error allocating memzone for capabilities, destroying PMD for %s", + name); + rte_cryptodev_pmd_destroy(cryptodev); + memset(&qat_dev_instance->sym_rte_dev, 0, + sizeof(qat_dev_instance->sym_rte_dev)); + return -EFAULT; } - PMD_DRV_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d", - name, internals->sym_dev_id); + memcpy(internals->capa_mz->addr, capabilities, capa_size); + internals->qat_dev_capabilities = internals->capa_mz->addr; + + while (1) { + if (qat_dev_cmd_param[i].name == NULL) + break; + if (!strcmp(qat_dev_cmd_param[i].name, SYM_ENQ_THRESHOLD_NAME)) + internals->min_enq_burst_threshold = + qat_dev_cmd_param[i].val; + i++; + } + + qat_pci_dev->sym_dev = internals; + QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d", + cryptodev->data->name, internals->sym_dev_id); + return 0; } @@ -300,27 +478,22 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev) return -ENODEV; if (qat_pci_dev->sym_dev == NULL) return 0; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_memzone_free(qat_pci_dev->sym_dev->capa_mz); /* free crypto device */ cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->sym_dev_id); +#ifdef RTE_LIBRTE_SECURITY + rte_free(cryptodev->security_ctx); +#endif rte_cryptodev_pmd_destroy(cryptodev); + qat_pci_devs[qat_pci_dev->qat_dev_id].sym_rte_dev.name = NULL; qat_pci_dev->sym_dev = NULL; return 0; } - -/* An rte_driver is needed in the registration of both the device and the driver - * with cryptodev. - * The actual qat pci's rte_driver can't be used as its name represents - * the whole pci device with all services. Think of this as a holder for a name - * for the crypto part of the pci device. - */ -static const char qat_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD); -static struct rte_driver cryptodev_qat_sym_driver = { - .name = qat_sym_drv_name, - .alias = qat_sym_drv_name -}; static struct cryptodev_driver qat_crypto_drv; -RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv, cryptodev_qat_sym_driver, - cryptodev_qat_driver_id); +RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv, + cryptodev_qat_sym_driver, + qat_sym_driver_id);