From: Arek Kusztal Date: Mon, 28 Jun 2021 16:34:20 +0000 (+0100) Subject: common/qat: support GEN4 devices X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=8f393c4ffdc1ff9b46702708781723ca0f17f5ac;p=dpdk.git common/qat: support GEN4 devices This commit adds support for fourth generation (GEN4) of Intel QuickAssist (QAT) Technology devices. Signed-off-by: Arek Kusztal Acked-by: Fan Zhang Acked-by: Akhil Goyal --- diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst index 96f5ab6afe..666a01df33 100644 --- a/doc/guides/cryptodevs/qat.rst +++ b/doc/guides/cryptodevs/qat.rst @@ -25,6 +25,7 @@ poll mode crypto driver support for the following hardware accelerator devices: * ``Intel QuickAssist Technology 200xx`` * ``Intel QuickAssist Technology D15xx`` * ``Intel QuickAssist Technology C4xxx`` +* ``Intel QuickAssist Technology 4xxx`` Features @@ -94,15 +95,16 @@ All the usual chains are supported and also some mixed chains: +==================+===========+=============+==========+==========+ | NULL CIPHER | Y | 2&3 | 2&3 | Y | +------------------+-----------+-------------+----------+----------+ - | SNOW3G UEA2 | 2&3 | Y | 2&3 | 2&3 | + | SNOW3G UEA2 | 2&3 | 1&2&3 | 2&3 | 2&3 | +------------------+-----------+-------------+----------+----------+ | ZUC EEA3 | 2&3 | 2&3 | 2&3 | 2&3 | +------------------+-----------+-------------+----------+----------+ - | AES CTR | Y | 2&3 | 2&3 | Y | + | AES CTR | 1&2&3 | 2&3 | 2&3 | Y | +------------------+-----------+-------------+----------+----------+ * The combinations marked as "Y" are supported on all QAT hardware versions. -* The combinations marked as "2&3" are supported on GEN2/GEN3 QAT hardware only. +* The combinations marked as "2&3" are supported on GEN2 and GEN3 QAT hardware only. +* The combinations marked as "1&2&3" are supported on GEN1, GEN2 and GEN3 QAT hardware only. Limitations @@ -373,6 +375,8 @@ to see the full table) +-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+ | Yes | No | No | 3 | C4xxx | p | qat_c4xxx | c4xxx | 18a0 | 1 | 18a1 | 128 | +-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+ + | Yes | No | No | 4 | 4xxx | N/A | qat_4xxx | 4xxx | 4940 | 4 | 4941 | 16 | + +-----+-----+-----+-----+----------+---------------+---------------+------------+--------+------+--------+--------+ * Note: Symmetric mixed crypto algorithms feature on Gen 2 works only with 01.org driver version 4.9.0+ diff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst index e0d6a1e0e6..d21b3138a7 100644 --- a/doc/guides/rel_notes/release_21_08.rst +++ b/doc/guides/rel_notes/release_21_08.rst @@ -105,6 +105,12 @@ New Features * Added COUNT action support for SN1000 NICs +* **Updated Intel QuickAssist crypto PMD.** + + Added fourth generation of QuickAssist Technology(QAT) devices support. + Only symmetric crypto has been currently enabled, compression and asymmetric + crypto PMD will fail to create. + * **Added support for Marvell CNXK crypto driver.** * Added cnxk crypto PMD which provides support for an integrated diff --git a/drivers/common/qat/qat_adf/adf_transport_access_macros_gen4.h b/drivers/common/qat/qat_adf/adf_transport_access_macros_gen4.h new file mode 100644 index 0000000000..3ab873db5e --- /dev/null +++ b/drivers/common/qat/qat_adf/adf_transport_access_macros_gen4.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) + * Copyright(c) 2021 Intel Corporation + */ + +#ifndef ADF_TRANSPORT_ACCESS_MACROS_GEN4_H +#define ADF_TRANSPORT_ACCESS_MACROS_GEN4_H + +#include "adf_transport_access_macros.h" + +#define ADF_RINGS_PER_INT_SRCSEL_GEN4 2 +#define ADF_BANK_INT_SRC_SEL_MASK_GEN4 0x44UL +#define ADF_BANK_INT_FLAG_CLEAR_MASK_GEN4 0x3 +#define ADF_RING_BUNDLE_SIZE_GEN4 0x2000 +#define ADF_RING_CSR_ADDR_OFFSET_GEN4 0x100000 +#define ADF_RING_CSR_RING_CONFIG_GEN4 0x1000 +#define ADF_RING_CSR_RING_LBASE_GEN4 0x1040 +#define ADF_RING_CSR_RING_UBASE_GEN4 0x1080 + +#define BUILD_RING_BASE_ADDR_GEN4(addr, size) \ + ((((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) << 6) + +#define WRITE_CSR_RING_BASE_GEN4(csr_base_addr, bank, ring, value) \ +do { \ + uint32_t l_base = 0, u_base = 0; \ + l_base = (uint32_t)(value & 0xFFFFFFFF); \ + u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \ + ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \ + ADF_RING_CSR_RING_LBASE_GEN4 + (ring << 2), \ + l_base); \ + ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \ + ADF_RING_CSR_RING_UBASE_GEN4 + (ring << 2), \ + u_base); \ +} while (0) + +#define WRITE_CSR_RING_CONFIG_GEN4(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \ + ADF_RING_CSR_RING_CONFIG_GEN4 + (ring << 2), value) + +#define WRITE_CSR_RING_TAIL_GEN4(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((u8 *)(csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) + +#define WRITE_CSR_RING_HEAD_GEN4(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((u8 *)(csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) + +#endif diff --git a/drivers/common/qat/qat_adf/adf_transport_access_macros_gen4vf.h b/drivers/common/qat/qat_adf/adf_transport_access_macros_gen4vf.h new file mode 100644 index 0000000000..37e113c443 --- /dev/null +++ b/drivers/common/qat/qat_adf/adf_transport_access_macros_gen4vf.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) + * Copyright(c) 2021 Intel Corporation + */ + +#ifndef ADF_TRANSPORT_ACCESS_MACROS_GEN4VF_H +#define ADF_TRANSPORT_ACCESS_MACROS_GEN4VF_H + +#include "adf_transport_access_macros.h" +#include "adf_transport_access_macros_gen4.h" + +#define ADF_RING_CSR_ADDR_OFFSET_GEN4VF 0x0 + +#define WRITE_CSR_RING_BASE_GEN4VF(csr_base_addr, bank, ring, value) \ +do { \ + uint32_t l_base = 0, u_base = 0; \ + l_base = (uint32_t)(value & 0xFFFFFFFF); \ + u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \ + ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \ + ADF_RING_CSR_RING_LBASE_GEN4 + (ring << 2), \ + l_base); \ + ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \ + ADF_RING_CSR_RING_UBASE_GEN4 + (ring << 2), \ + u_base); \ +} while (0) + +#define WRITE_CSR_RING_CONFIG_GEN4VF(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * bank) + \ + ADF_RING_CSR_RING_CONFIG_GEN4 + (ring << 2), value) + +#define WRITE_CSR_RING_TAIL_GEN4VF(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), (value)) + +#define WRITE_CSR_RING_HEAD_GEN4VF(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), (value)) + +#define WRITE_CSR_RING_SRV_ARB_EN_GEN4VF(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, \ + (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \ + ADF_RING_CSR_RING_SRV_ARB_EN, (value)) + +#endif diff --git a/drivers/common/qat/qat_common.h b/drivers/common/qat/qat_common.h index cf840fea9b..845c8d99ab 100644 --- a/drivers/common/qat/qat_common.h +++ b/drivers/common/qat/qat_common.h @@ -18,7 +18,8 @@ enum qat_device_gen { QAT_GEN1 = 1, QAT_GEN2, - QAT_GEN3 + QAT_GEN3, + QAT_GEN4 }; enum qat_service_type { diff --git a/drivers/common/qat/qat_device.c b/drivers/common/qat/qat_device.c index 9fa142b5e5..932d7110f7 100644 --- a/drivers/common/qat/qat_device.c +++ b/drivers/common/qat/qat_device.c @@ -30,6 +30,11 @@ struct qat_gen_hw_data qat_gen_config[] = { .qp_hw_data = qat_gen3_qps, .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3 }, + [QAT_GEN4] = { + .dev_gen = QAT_GEN4, + .qp_hw_data = NULL, + .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3 + }, }; /* per-process array of device data */ @@ -59,6 +64,9 @@ static const struct rte_pci_id pci_id_qat_map[] = { { RTE_PCI_DEVICE(0x8086, 0x18a1), }, + { + RTE_PCI_DEVICE(0x8086, 0x4941), + }, {.device_id = 0}, }; @@ -232,6 +240,9 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev, case 0x18a1: qat_dev->qat_dev_gen = QAT_GEN3; break; + case 0x4941: + qat_dev->qat_dev_gen = QAT_GEN4; + break; default: QAT_LOG(ERR, "Invalid dev_id, can't determine generation"); rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz); @@ -241,6 +252,17 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev, if (devargs && devargs->drv_str) qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param); + if (qat_dev->qat_dev_gen >= QAT_GEN4) { + int ret = qat_read_qp_config(qat_dev, qat_dev->qat_dev_gen); + + if (ret) { + QAT_LOG(ERR, + "Cannot acquire ring configuration for QAT_%d", + qat_dev_id); + return NULL; + } + } + rte_spinlock_init(&qat_dev->arb_csr_lock); qat_nb_pci_devices++; diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h index 9c6a3ca4e6..f4fecc9517 100644 --- a/drivers/common/qat/qat_device.h +++ b/drivers/common/qat/qat_device.h @@ -105,6 +105,9 @@ struct qat_pci_device { /* Data relating to compression service */ struct qat_comp_dev_private *comp_dev; /**< link back to compressdev private data */ + struct qat_qp_hw_data qp_gen4_data[QAT_GEN4_BUNDLE_NUM] + [QAT_GEN4_QPS_PER_BUNDLE_NUM]; + /**< Data of ring configuration on gen4 */ }; struct qat_gen_hw_data { diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c index aa64d2e168..8be59779f9 100644 --- a/drivers/common/qat/qat_qp.c +++ b/drivers/common/qat/qat_qp.c @@ -19,6 +19,7 @@ #include "qat_asym.h" #include "qat_comp.h" #include "adf_transport_access_macros.h" +#include "adf_transport_access_macros_gen4vf.h" #define QAT_CQ_MAX_DEQ_RETRIES 10 @@ -138,25 +139,33 @@ static int qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, struct qat_qp_config *, uint8_t dir); static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num, uint32_t *queue_size_for_csr); -static void adf_configure_queues(struct qat_qp *queue); -static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr, - rte_spinlock_t *lock); -static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr, - rte_spinlock_t *lock); - +static void adf_configure_queues(struct qat_qp *queue, + enum qat_device_gen qat_dev_gen); +static void adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, + struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock); +static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen, + struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock); int qat_qps_per_service(struct qat_pci_device *qat_dev, enum qat_service_type service) { int i = 0, count = 0, max_ops_per_srv = 0; - const struct qat_qp_hw_data* - sym_hw_qps = qat_gen_config[qat_dev->qat_dev_gen] - .qp_hw_data[service]; - max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE; - for (; i < max_ops_per_srv; i++) - if (sym_hw_qps[i].service_type == service) - count++; + if (qat_dev->qat_dev_gen == QAT_GEN4) { + max_ops_per_srv = QAT_GEN4_BUNDLE_NUM; + for (i = 0, count = 0; i < max_ops_per_srv; i++) + if (qat_dev->qp_gen4_data[i][0].service_type == service) + count++; + } else { + const struct qat_qp_hw_data *sym_hw_qps = + qat_gen_config[qat_dev->qat_dev_gen] + .qp_hw_data[service]; + + max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE; + for (i = 0, count = 0; i < max_ops_per_srv; i++) + if (sym_hw_qps[i].service_type == service) + count++; + } return count; } @@ -195,12 +204,12 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, struct qat_qp **qp_addr, uint16_t queue_pair_id, struct qat_qp_config *qat_qp_conf) - { struct qat_qp *qp; struct rte_pci_device *pci_dev = qat_pci_devs[qat_dev->qat_dev_id].pci_dev; char op_cookie_pool_name[RTE_RING_NAMESIZE]; + enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen; uint32_t i; QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d", @@ -264,8 +273,8 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, goto create_err; } - adf_configure_queues(qp); - adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr, + adf_configure_queues(qp, qat_dev_gen); + adf_queue_arb_enable(qat_dev_gen, &qp->tx_q, qp->mmap_bar_addr, &qat_dev->arb_csr_lock); snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, @@ -314,7 +323,8 @@ create_err: return -EFAULT; } -int qat_qp_release(struct qat_qp **qp_addr) + +int qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr) { struct qat_qp *qp = *qp_addr; uint32_t i; @@ -335,8 +345,8 @@ int qat_qp_release(struct qat_qp **qp_addr) return -EAGAIN; } - adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr, - &qp->qat_dev->arb_csr_lock); + adf_queue_arb_disable(qat_dev_gen, &(qp->tx_q), qp->mmap_bar_addr, + &qp->qat_dev->arb_csr_lock); for (i = 0; i < qp->nb_descriptors; i++) rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]); @@ -386,6 +396,7 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, const struct rte_memzone *qp_mz; struct rte_pci_device *pci_dev = qat_pci_devs[qat_dev->qat_dev_id].pci_dev; + enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen; int ret = 0; uint16_t desc_size = (dir == ADF_RING_DIR_TX ? qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size); @@ -445,14 +456,19 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, * Write an unused pattern to the queue memory. */ memset(queue->base_addr, 0x7F, queue_size_bytes); - - queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr, - queue->queue_size); - io_addr = pci_dev->mem_resource[0].addr; - WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number, + if (qat_dev_gen == QAT_GEN4) { + queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr, + queue->queue_size); + WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number, + queue->hw_queue_number, queue_base); + } else { + queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr, + queue->queue_size); + WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number, queue->hw_queue_number, queue_base); + } QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u," " nb msgs %u, msg_size %u, modulo mask %u", @@ -468,6 +484,61 @@ queue_create_err: return ret; } +int +qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id, + enum qat_service_type service_type) +{ + if (qat_dev->qat_dev_gen == QAT_GEN4) { + int i = 0, valid_qps = 0; + + for (; i < QAT_GEN4_BUNDLE_NUM; i++) { + if (qat_dev->qp_gen4_data[i][0].service_type == + service_type) { + if (valid_qps == qp_id) + return i; + ++valid_qps; + } + } + } + return -1; +} + +int +qat_read_qp_config(struct qat_pci_device *qat_dev, + enum qat_device_gen qat_dev_gen) +{ + if (qat_dev_gen == QAT_GEN4) { + /* Read default configuration, + * until some probe of it can be done + */ + int i = 0; + + for (; i < QAT_GEN4_BUNDLE_NUM; i++) { + struct qat_qp_hw_data *hw_data = + &qat_dev->qp_gen4_data[i][0]; + enum qat_service_type service_type = + (QAT_GEN4_QP_DEFCON >> (8 * i)) & 0xFF; + + memset(hw_data, 0, sizeof(*hw_data)); + hw_data->service_type = service_type; + if (service_type == QAT_SERVICE_ASYMMETRIC) { + hw_data->tx_msg_size = 64; + hw_data->rx_msg_size = 32; + } else if (service_type == QAT_SERVICE_SYMMETRIC || + service_type == + QAT_SERVICE_COMPRESSION) { + hw_data->tx_msg_size = 128; + hw_data->rx_msg_size = 32; + } + hw_data->tx_ring_num = 0; + hw_data->rx_ring_num = 1; + hw_data->hw_bundle_num = i; + } + } + /* With default config will always return success */ + return 0; +} + static int qat_qp_check_queue_alignment(uint64_t phys_addr, uint32_t queue_size_bytes) { @@ -491,54 +562,81 @@ static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num, return -EINVAL; } -static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr, - rte_spinlock_t *lock) +static void +adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq, + void *base_addr, rte_spinlock_t *lock) { - uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + - (ADF_ARB_REG_SLOT * - txq->hw_bundle_number); - uint32_t value; + uint32_t arb_csr_offset = 0, value; rte_spinlock_lock(lock); - value = ADF_CSR_RD(base_addr, arb_csr_offset); + if (qat_dev_gen == QAT_GEN4) { + arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + + (ADF_RING_BUNDLE_SIZE_GEN4 * + txq->hw_bundle_number); + value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, + arb_csr_offset); + } else { + arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + + (ADF_ARB_REG_SLOT * + txq->hw_bundle_number); + value = ADF_CSR_RD(base_addr, + arb_csr_offset); + } value |= (0x01 << txq->hw_queue_number); ADF_CSR_WR(base_addr, arb_csr_offset, value); rte_spinlock_unlock(lock); } -static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr, - rte_spinlock_t *lock) +static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen, + struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock) { - uint32_t arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + - (ADF_ARB_REG_SLOT * - txq->hw_bundle_number); - uint32_t value; + uint32_t arb_csr_offset = 0, value; rte_spinlock_lock(lock); - value = ADF_CSR_RD(base_addr, arb_csr_offset); + if (qat_dev_gen == QAT_GEN4) { + arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + + (ADF_RING_BUNDLE_SIZE_GEN4 * + txq->hw_bundle_number); + value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF, + arb_csr_offset); + } else { + arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET + + (ADF_ARB_REG_SLOT * + txq->hw_bundle_number); + value = ADF_CSR_RD(base_addr, + arb_csr_offset); + } value &= ~(0x01 << txq->hw_queue_number); ADF_CSR_WR(base_addr, arb_csr_offset, value); rte_spinlock_unlock(lock); } -static void adf_configure_queues(struct qat_qp *qp) +static void adf_configure_queues(struct qat_qp *qp, + enum qat_device_gen qat_dev_gen) { - uint32_t queue_config; - struct qat_queue *queue = &qp->tx_q; - - queue_config = BUILD_RING_CONFIG(queue->queue_size); - - WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number, - queue->hw_queue_number, queue_config); - - queue = &qp->rx_q; - queue_config = - BUILD_RESP_RING_CONFIG(queue->queue_size, - ADF_RING_NEAR_WATERMARK_512, - ADF_RING_NEAR_WATERMARK_0); - - WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number, - queue->hw_queue_number, queue_config); + uint32_t q_tx_config, q_resp_config; + struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q; + + q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size); + q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size, + ADF_RING_NEAR_WATERMARK_512, + ADF_RING_NEAR_WATERMARK_0); + + if (qat_dev_gen == QAT_GEN4) { + WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr, + q_tx->hw_bundle_number, q_tx->hw_queue_number, + q_tx_config); + WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr, + q_rx->hw_bundle_number, q_rx->hw_queue_number, + q_resp_config); + } else { + WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, + q_tx->hw_bundle_number, q_tx->hw_queue_number, + q_tx_config); + WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, + q_rx->hw_bundle_number, q_rx->hw_queue_number, + q_resp_config); + } } static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask) @@ -547,14 +645,21 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask) } static inline void -txq_write_tail(struct qat_qp *qp, struct qat_queue *q) { - WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, +txq_write_tail(enum qat_device_gen qat_dev_gen, + struct qat_qp *qp, struct qat_queue *q) { + + if (qat_dev_gen == QAT_GEN4) { + WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr, + q->hw_bundle_number, q->hw_queue_number, q->tail); + } else { + WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number, q->hw_queue_number, q->tail); - q->csr_tail = q->tail; + } } static inline -void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q) +void rxq_free_desc(enum qat_device_gen qat_dev_gen, struct qat_qp *qp, + struct qat_queue *q) { uint32_t old_head, new_head; uint32_t max_head; @@ -576,8 +681,14 @@ void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q) q->csr_head = new_head; /* write current head to CSR */ - WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number, - q->hw_queue_number, new_head); + if (qat_dev_gen == QAT_GEN4) { + WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr, + q->hw_bundle_number, q->hw_queue_number, new_head); + } else { + WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number, + q->hw_queue_number, new_head); + } + } uint16_t @@ -670,7 +781,7 @@ kick_tail: queue->tail = tail; tmp_qp->enqueued += nb_ops_sent; tmp_qp->stats.enqueued_count += nb_ops_sent; - txq_write_tail(tmp_qp, queue); + txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue); return nb_ops_sent; } @@ -843,7 +954,7 @@ kick_tail: queue->tail = tail; tmp_qp->enqueued += total_descriptors_built; tmp_qp->stats.enqueued_count += nb_ops_sent; - txq_write_tail(tmp_qp, queue); + txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue); return nb_ops_sent; } @@ -909,7 +1020,7 @@ qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops) rx_queue->head = head; if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) - rxq_free_desc(tmp_qp, rx_queue); + rxq_free_desc(tmp_qp->qat_dev_gen, tmp_qp, rx_queue); QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u", op_resp_counter, fw_resp_counter); @@ -951,7 +1062,7 @@ qat_cq_dequeue_response(struct qat_qp *qp, void *out_data) queue->head = adf_modulo(queue->head + queue->msg_size, queue->modulo_mask); - rxq_free_desc(qp, queue); + rxq_free_desc(qp->qat_dev_gen, qp, queue); } return result; @@ -986,7 +1097,7 @@ qat_cq_get_fw_version(struct qat_qp *qp) memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg)); queue->tail = adf_modulo(queue->tail + queue->msg_size, queue->modulo_mask); - txq_write_tail(qp, queue); + txq_write_tail(qp->qat_dev_gen, qp, queue); /* receive a response */ if (qat_cq_dequeue_response(qp, &response)) { diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h index d353e8552b..3d9a757349 100644 --- a/drivers/common/qat/qat_qp.h +++ b/drivers/common/qat/qat_qp.h @@ -14,6 +14,16 @@ struct qat_pci_device; #define QAT_QP_MIN_INFL_THRESHOLD 256 +/* Default qp configuration for GEN4 devices */ +#define QAT_GEN4_QP_DEFCON (QAT_SERVICE_SYMMETRIC | \ + QAT_SERVICE_SYMMETRIC << 8 | \ + QAT_SERVICE_SYMMETRIC << 16 | \ + QAT_SERVICE_SYMMETRIC << 24) + +/* QAT GEN 4 specific macros */ +#define QAT_GEN4_BUNDLE_NUM 4 +#define QAT_GEN4_QPS_PER_BUNDLE_NUM 1 + /** * Structure with data needed for creation of queue pair. */ @@ -26,6 +36,15 @@ struct qat_qp_hw_data { uint16_t rx_msg_size; }; +/** + * Structure with data needed for creation of queue pair on gen4. + */ +struct qat_qp_gen4_data { + struct qat_qp_hw_data qat_qp_hw_data; + uint8_t reserved; + uint8_t valid; +}; + /** * Structure with data needed for creation of queue pair. */ @@ -90,7 +109,7 @@ uint16_t qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops); int -qat_qp_release(struct qat_qp **qp_addr); +qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr); int qat_qp_setup(struct qat_pci_device *qat_dev, @@ -110,4 +129,12 @@ qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused, void *op_cookie __rte_unused, uint64_t *dequeue_err_count __rte_unused); +int +qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id, + enum qat_service_type service_type); + +int +qat_read_qp_config(struct qat_pci_device *qat_dev, + enum qat_device_gen qat_dev_gen); + #endif /* _QAT_QP_H_ */ diff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c index 6eb1ae3a21..cfdcb6b3d1 100644 --- a/drivers/compress/qat/qat_comp_pmd.c +++ b/drivers/compress/qat/qat_comp_pmd.c @@ -74,6 +74,7 @@ qat_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id) struct qat_qp **qp_addr = (struct qat_qp **)&(dev->data->queue_pairs[queue_pair_id]); struct qat_qp *qp = (struct qat_qp *)*qp_addr; + enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen; uint32_t i; QAT_LOG(DEBUG, "Release comp qp %u on device %d", @@ -90,7 +91,7 @@ qat_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id) rte_free(cookie->qat_sgl_dst_d); } - return qat_qp_release((struct qat_qp **) + return qat_qp_release(qat_dev_gen, (struct qat_qp **) &(dev->data->queue_pairs[queue_pair_id])); } @@ -710,6 +711,10 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev, const struct rte_compressdev_capabilities *capabilities; uint64_t capa_size; + if (qat_pci_dev->qat_dev_gen == QAT_GEN4) { + QAT_LOG(ERR, "Compression PMD not supported on QAT 4xxx"); + return 0; + } snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, "%s_%s", qat_pci_dev->name, "comp"); QAT_LOG(DEBUG, "Creating QAT COMP device %s", name); diff --git a/drivers/crypto/qat/qat_asym_pmd.c b/drivers/crypto/qat/qat_asym_pmd.c index f0c8ed1bcf..56ccca36d1 100644 --- a/drivers/crypto/qat/qat_asym_pmd.c +++ b/drivers/crypto/qat/qat_asym_pmd.c @@ -103,6 +103,7 @@ static int qat_asym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) { struct qat_asym_dev_private *qat_private = dev->data->dev_private; + enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen; QAT_LOG(DEBUG, "Release asym qp %u on device %d", queue_pair_id, dev->data->dev_id); @@ -110,7 +111,7 @@ static int qat_asym_qp_release(struct rte_cryptodev *dev, qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][queue_pair_id] = NULL; - return qat_qp_release((struct qat_qp **) + return qat_qp_release(qat_dev_gen, (struct qat_qp **) &(dev->data->queue_pairs[queue_pair_id])); } @@ -250,6 +251,10 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev, struct rte_cryptodev *cryptodev; struct qat_asym_dev_private *internals; + if (qat_pci_dev->qat_dev_gen == QAT_GEN4) { + QAT_LOG(ERR, "Asymmetric crypto PMD not supported on QAT 4xxx"); + return 0; + } snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s", qat_pci_dev->name, "asym"); QAT_LOG(DEBUG, "Creating QAT ASYM device %s\n", name); diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c index 549345b6fa..e15722ad66 100644 --- a/drivers/crypto/qat/qat_sym_pmd.c +++ b/drivers/crypto/qat/qat_sym_pmd.c @@ -139,6 +139,7 @@ static void qat_sym_stats_reset(struct rte_cryptodev *dev) static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) { struct qat_sym_dev_private *qat_private = dev->data->dev_private; + enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen; QAT_LOG(DEBUG, "Release sym qp %u on device %d", queue_pair_id, dev->data->dev_id); @@ -146,7 +147,7 @@ static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][queue_pair_id] = NULL; - return qat_qp_release((struct qat_qp **) + return qat_qp_release(qat_dev_gen, (struct qat_qp **) &(dev->data->queue_pairs[queue_pair_id])); } @@ -158,15 +159,33 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, int ret = 0; uint32_t i; struct qat_qp_config qat_qp_conf; + const struct qat_qp_hw_data *sym_hw_qps = NULL; + const struct qat_qp_hw_data *qp_hw_data = NULL; struct qat_qp **qp_addr = (struct qat_qp **)&(dev->data->queue_pairs[qp_id]); struct qat_sym_dev_private *qat_private = dev->data->dev_private; struct qat_pci_device *qat_dev = qat_private->qat_dev; - const struct qat_qp_hw_data *sym_hw_qps = - qat_gen_config[qat_private->qat_dev->qat_dev_gen] - .qp_hw_data[QAT_SERVICE_SYMMETRIC]; - const struct qat_qp_hw_data *qp_hw_data = sym_hw_qps + qp_id; + + if (qat_dev->qat_dev_gen == QAT_GEN4) { + int ring_pair = + qat_select_valid_queue(qat_dev, qp_id, + QAT_SERVICE_SYMMETRIC); + sym_hw_qps = + &qat_dev->qp_gen4_data[0][0]; + qp_hw_data = + &qat_dev->qp_gen4_data[ring_pair][0]; + if (ring_pair < 0) { + QAT_LOG(ERR, + "qp_id %u invalid for this device, no enough services allocated for GEN4 device", + qp_id); + return -EINVAL; + } + } else { + sym_hw_qps = qat_gen_config[qat_dev->qat_dev_gen] + .qp_hw_data[QAT_SERVICE_SYMMETRIC]; + qp_hw_data = sym_hw_qps + qp_id; + } /* If qp is already in use free ring memory and qp metadata. */ if (*qp_addr != NULL) { @@ -430,6 +449,10 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, capabilities = qat_gen3_sym_capabilities; capa_size = sizeof(qat_gen3_sym_capabilities); break; + case QAT_GEN4: + capabilities = NULL; + capa_size = 0; + break; default: QAT_LOG(DEBUG, "QAT gen %d capabilities unknown", diff --git a/drivers/crypto/qat/qat_sym_session.c b/drivers/crypto/qat/qat_sym_session.c index e22dd3600c..1070dcf4a9 100644 --- a/drivers/crypto/qat/qat_sym_session.c +++ b/drivers/crypto/qat/qat_sym_session.c @@ -550,6 +550,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev, return -EINVAL; } + memset(session, 0, sizeof(*session)); /* Set context descriptor physical address */ session->cd_paddr = session_paddr + offsetof(struct qat_sym_session, cd);