1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2018 Intel Corporation
5 #include <rte_common.h>
6 #include <rte_cycles.h>
8 #include <rte_malloc.h>
9 #include <rte_memzone.h>
11 #include <rte_bus_pci.h>
12 #include <rte_atomic.h>
13 #include <rte_prefetch.h>
16 #include "qat_device.h"
21 #include "adf_transport_access_macros.h"
22 #include "adf_transport_access_macros_gen4vf.h"
24 #define QAT_CQ_MAX_DEQ_RETRIES 10
26 #define ADF_MAX_DESC 4096
27 #define ADF_MIN_DESC 128
29 #define ADF_ARB_REG_SLOT 0x1000
30 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C
32 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
33 ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
34 (ADF_ARB_REG_SLOT * index), value)
37 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
38 [ADF_MAX_QPS_ON_ANY_SERVICE] = {
39 /* queue pairs which provide an asymmetric crypto service */
40 [QAT_SERVICE_ASYMMETRIC] = {
42 .service_type = QAT_SERVICE_ASYMMETRIC,
50 .service_type = QAT_SERVICE_ASYMMETRIC,
58 /* queue pairs which provide a symmetric crypto service */
59 [QAT_SERVICE_SYMMETRIC] = {
61 .service_type = QAT_SERVICE_SYMMETRIC,
69 .service_type = QAT_SERVICE_SYMMETRIC,
77 /* queue pairs which provide a compression service */
78 [QAT_SERVICE_COMPRESSION] = {
80 .service_type = QAT_SERVICE_COMPRESSION,
87 .service_type = QAT_SERVICE_COMPRESSION,
98 const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
99 [ADF_MAX_QPS_ON_ANY_SERVICE] = {
100 /* queue pairs which provide an asymmetric crypto service */
101 [QAT_SERVICE_ASYMMETRIC] = {
103 .service_type = QAT_SERVICE_ASYMMETRIC,
111 /* queue pairs which provide a symmetric crypto service */
112 [QAT_SERVICE_SYMMETRIC] = {
114 .service_type = QAT_SERVICE_SYMMETRIC,
122 /* queue pairs which provide a compression service */
123 [QAT_SERVICE_COMPRESSION] = {
125 .service_type = QAT_SERVICE_COMPRESSION,
135 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
136 uint32_t queue_size_bytes);
137 static void qat_queue_delete(struct qat_queue *queue);
138 static int qat_queue_create(struct qat_pci_device *qat_dev,
139 struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
140 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
141 uint32_t *queue_size_for_csr);
142 static void adf_configure_queues(struct qat_qp *queue,
143 enum qat_device_gen qat_dev_gen);
144 static void adf_queue_arb_enable(enum qat_device_gen qat_dev_gen,
145 struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
146 static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
147 struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
149 int qat_qps_per_service(struct qat_pci_device *qat_dev,
150 enum qat_service_type service)
152 int i = 0, count = 0, max_ops_per_srv = 0;
154 if (qat_dev->qat_dev_gen == QAT_GEN4) {
155 max_ops_per_srv = QAT_GEN4_BUNDLE_NUM;
156 for (i = 0, count = 0; i < max_ops_per_srv; i++)
157 if (qat_dev->qp_gen4_data[i][0].service_type == service)
160 const struct qat_qp_hw_data *sym_hw_qps =
161 qat_gen_config[qat_dev->qat_dev_gen]
162 .qp_hw_data[service];
164 max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE;
165 for (i = 0, count = 0; i < max_ops_per_srv; i++)
166 if (sym_hw_qps[i].service_type == service)
173 static const struct rte_memzone *
174 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
177 const struct rte_memzone *mz;
179 mz = rte_memzone_lookup(queue_name);
181 if (((size_t)queue_size <= mz->len) &&
182 ((socket_id == SOCKET_ID_ANY) ||
183 (socket_id == mz->socket_id))) {
184 QAT_LOG(DEBUG, "re-use memzone already "
185 "allocated for %s", queue_name);
189 QAT_LOG(ERR, "Incompatible memzone already "
190 "allocated %s, size %u, socket %d. "
191 "Requested size %u, socket %u",
192 queue_name, (uint32_t)mz->len,
193 mz->socket_id, queue_size, socket_id);
197 QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
198 queue_name, queue_size, socket_id);
199 return rte_memzone_reserve_aligned(queue_name, queue_size,
200 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
203 int qat_qp_setup(struct qat_pci_device *qat_dev,
204 struct qat_qp **qp_addr,
205 uint16_t queue_pair_id,
206 struct qat_qp_config *qat_qp_conf)
209 struct rte_pci_device *pci_dev =
210 qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
211 char op_cookie_pool_name[RTE_RING_NAMESIZE];
212 enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
215 QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
216 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
218 if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
219 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
220 QAT_LOG(ERR, "Can't create qp for %u descriptors",
221 qat_qp_conf->nb_descriptors);
225 if (pci_dev->mem_resource[0].addr == NULL) {
226 QAT_LOG(ERR, "Could not find VF config space "
227 "(UIO driver attached?).");
231 /* Allocate the queue pair data structure. */
232 qp = rte_zmalloc_socket("qat PMD qp metadata",
233 sizeof(*qp), RTE_CACHE_LINE_SIZE,
234 qat_qp_conf->socket_id);
236 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
239 qp->nb_descriptors = qat_qp_conf->nb_descriptors;
240 qp->op_cookies = rte_zmalloc_socket("qat PMD op cookie pointer",
241 qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
242 RTE_CACHE_LINE_SIZE, qat_qp_conf->socket_id);
243 if (qp->op_cookies == NULL) {
244 QAT_LOG(ERR, "Failed to alloc mem for cookie");
249 qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
250 qp->enqueued = qp->dequeued = 0;
252 if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
253 ADF_RING_DIR_TX) != 0) {
254 QAT_LOG(ERR, "Tx queue create failed "
255 "queue_pair_id=%u", queue_pair_id);
259 qp->max_inflights = ADF_MAX_INFLIGHTS(qp->tx_q.queue_size,
260 ADF_BYTES_TO_MSG_SIZE(qp->tx_q.msg_size));
262 if (qp->max_inflights < 2) {
263 QAT_LOG(ERR, "Invalid num inflights");
264 qat_queue_delete(&(qp->tx_q));
268 if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
269 ADF_RING_DIR_RX) != 0) {
270 QAT_LOG(ERR, "Rx queue create failed "
271 "queue_pair_id=%hu", queue_pair_id);
272 qat_queue_delete(&(qp->tx_q));
276 adf_configure_queues(qp, qat_dev_gen);
277 adf_queue_arb_enable(qat_dev_gen, &qp->tx_q, qp->mmap_bar_addr,
278 &qat_dev->arb_csr_lock);
280 snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
281 "%s%d_cookies_%s_qp%hu",
282 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
283 qat_qp_conf->service_str, queue_pair_id);
285 QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
286 qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
287 if (qp->op_cookie_pool == NULL)
288 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
290 qat_qp_conf->cookie_size, 64, 0,
291 NULL, NULL, NULL, NULL,
292 pci_dev->device.numa_node,
294 if (!qp->op_cookie_pool) {
295 QAT_LOG(ERR, "QAT PMD Cannot create"
300 for (i = 0; i < qp->nb_descriptors; i++) {
301 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
302 QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
305 memset(qp->op_cookies[i], 0, qat_qp_conf->cookie_size);
308 qp->qat_dev_gen = qat_dev->qat_dev_gen;
309 qp->service_type = qat_qp_conf->hw->service_type;
310 qp->qat_dev = qat_dev;
312 QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
313 queue_pair_id, op_cookie_pool_name);
319 if (qp->op_cookie_pool)
320 rte_mempool_free(qp->op_cookie_pool);
321 rte_free(qp->op_cookies);
327 int qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr)
329 struct qat_qp *qp = *qp_addr;
333 QAT_LOG(DEBUG, "qp already freed");
337 QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
338 qp->qat_dev->qat_dev_id);
340 /* Don't free memory if there are still responses to be processed */
341 if ((qp->enqueued - qp->dequeued) == 0) {
342 qat_queue_delete(&(qp->tx_q));
343 qat_queue_delete(&(qp->rx_q));
348 adf_queue_arb_disable(qat_dev_gen, &(qp->tx_q), qp->mmap_bar_addr,
349 &qp->qat_dev->arb_csr_lock);
351 for (i = 0; i < qp->nb_descriptors; i++)
352 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
354 if (qp->op_cookie_pool)
355 rte_mempool_free(qp->op_cookie_pool);
357 rte_free(qp->op_cookies);
364 static void qat_queue_delete(struct qat_queue *queue)
366 const struct rte_memzone *mz;
370 QAT_LOG(DEBUG, "Invalid queue");
373 QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
374 queue->hw_queue_number, queue->memz_name);
376 mz = rte_memzone_lookup(queue->memz_name);
378 /* Write an unused pattern to the queue memory. */
379 memset(queue->base_addr, 0x7F, queue->queue_size);
380 status = rte_memzone_free(mz);
382 QAT_LOG(ERR, "Error %d on freeing queue %s",
383 status, queue->memz_name);
385 QAT_LOG(DEBUG, "queue %s doesn't exist",
391 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
392 struct qat_qp_config *qp_conf, uint8_t dir)
396 const struct rte_memzone *qp_mz;
397 struct rte_pci_device *pci_dev =
398 qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
399 enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
401 uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
402 qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
403 uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
405 queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
406 queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
407 qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
409 if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
410 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
415 * Allocate a memzone for the queue - create a unique name.
417 snprintf(queue->memz_name, sizeof(queue->memz_name),
419 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
420 qp_conf->service_str, "qp_mem",
421 queue->hw_bundle_number, queue->hw_queue_number);
422 qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
423 pci_dev->device.numa_node);
425 QAT_LOG(ERR, "Failed to allocate ring memzone");
429 queue->base_addr = (char *)qp_mz->addr;
430 queue->base_phys_addr = qp_mz->iova;
431 if (qat_qp_check_queue_alignment(queue->base_phys_addr,
433 QAT_LOG(ERR, "Invalid alignment on queue create "
435 queue->base_phys_addr);
437 goto queue_create_err;
440 if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
441 &(queue->queue_size)) != 0) {
442 QAT_LOG(ERR, "Invalid num inflights");
444 goto queue_create_err;
447 queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
450 queue->msg_size = desc_size;
452 /* For fast calculation of cookie index, relies on msg_size being 2^n */
453 queue->trailz = __builtin_ctz(desc_size);
456 * Write an unused pattern to the queue memory.
458 memset(queue->base_addr, 0x7F, queue_size_bytes);
459 io_addr = pci_dev->mem_resource[0].addr;
461 if (qat_dev_gen == QAT_GEN4) {
462 queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr,
464 WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number,
465 queue->hw_queue_number, queue_base);
467 queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
469 WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
470 queue->hw_queue_number, queue_base);
473 QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
474 " nb msgs %u, msg_size %u, modulo mask %u",
476 queue->queue_size, queue_size_bytes,
477 qp_conf->nb_descriptors, desc_size,
483 rte_memzone_free(qp_mz);
488 qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id,
489 enum qat_service_type service_type)
491 if (qat_dev->qat_dev_gen == QAT_GEN4) {
492 int i = 0, valid_qps = 0;
494 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
495 if (qat_dev->qp_gen4_data[i][0].service_type ==
497 if (valid_qps == qp_id)
507 qat_read_qp_config(struct qat_pci_device *qat_dev)
510 enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
512 if (qat_dev_gen == QAT_GEN4) {
515 if (qat_query_svc(qat_dev, (uint8_t *)&svc))
517 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
518 struct qat_qp_hw_data *hw_data =
519 &qat_dev->qp_gen4_data[i][0];
520 uint8_t svc1 = (svc >> (3 * i)) & 0x7;
521 enum qat_service_type service_type = QAT_SERVICE_INVALID;
523 if (svc1 == QAT_SVC_SYM) {
524 service_type = QAT_SERVICE_SYMMETRIC;
526 "Discovered SYMMETRIC service on bundle %d",
528 } else if (svc1 == QAT_SVC_COMPRESSION) {
529 service_type = QAT_SERVICE_COMPRESSION;
531 "Discovered COPRESSION service on bundle %d",
533 } else if (svc1 == QAT_SVC_ASYM) {
534 service_type = QAT_SERVICE_ASYMMETRIC;
536 "Discovered ASYMMETRIC service on bundle %d",
540 "Unrecognized service on bundle %d",
545 memset(hw_data, 0, sizeof(*hw_data));
546 hw_data->service_type = service_type;
547 if (service_type == QAT_SERVICE_ASYMMETRIC) {
548 hw_data->tx_msg_size = 64;
549 hw_data->rx_msg_size = 32;
550 } else if (service_type == QAT_SERVICE_SYMMETRIC ||
552 QAT_SERVICE_COMPRESSION) {
553 hw_data->tx_msg_size = 128;
554 hw_data->rx_msg_size = 32;
556 hw_data->tx_ring_num = 0;
557 hw_data->rx_ring_num = 1;
558 hw_data->hw_bundle_num = i;
565 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
566 uint32_t queue_size_bytes)
568 if (((queue_size_bytes - 1) & phys_addr) != 0)
573 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
574 uint32_t *p_queue_size_for_csr)
576 uint8_t i = ADF_MIN_RING_SIZE;
578 for (; i <= ADF_MAX_RING_SIZE; i++)
579 if ((msg_size * msg_num) ==
580 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
581 *p_queue_size_for_csr = i;
584 QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
589 adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq,
590 void *base_addr, rte_spinlock_t *lock)
592 uint32_t arb_csr_offset = 0, value;
594 rte_spinlock_lock(lock);
595 if (qat_dev_gen == QAT_GEN4) {
596 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
597 (ADF_RING_BUNDLE_SIZE_GEN4 *
598 txq->hw_bundle_number);
599 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
602 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
604 txq->hw_bundle_number);
605 value = ADF_CSR_RD(base_addr,
608 value |= (0x01 << txq->hw_queue_number);
609 ADF_CSR_WR(base_addr, arb_csr_offset, value);
610 rte_spinlock_unlock(lock);
613 static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
614 struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock)
616 uint32_t arb_csr_offset = 0, value;
618 rte_spinlock_lock(lock);
619 if (qat_dev_gen == QAT_GEN4) {
620 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
621 (ADF_RING_BUNDLE_SIZE_GEN4 *
622 txq->hw_bundle_number);
623 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
626 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
628 txq->hw_bundle_number);
629 value = ADF_CSR_RD(base_addr,
632 value &= ~(0x01 << txq->hw_queue_number);
633 ADF_CSR_WR(base_addr, arb_csr_offset, value);
634 rte_spinlock_unlock(lock);
637 static void adf_configure_queues(struct qat_qp *qp,
638 enum qat_device_gen qat_dev_gen)
640 uint32_t q_tx_config, q_resp_config;
641 struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q;
643 q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size);
644 q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size,
645 ADF_RING_NEAR_WATERMARK_512,
646 ADF_RING_NEAR_WATERMARK_0);
648 if (qat_dev_gen == QAT_GEN4) {
649 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
650 q_tx->hw_bundle_number, q_tx->hw_queue_number,
652 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
653 q_rx->hw_bundle_number, q_rx->hw_queue_number,
656 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
657 q_tx->hw_bundle_number, q_tx->hw_queue_number,
659 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
660 q_rx->hw_bundle_number, q_rx->hw_queue_number,
665 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
667 return data & modulo_mask;
671 txq_write_tail(enum qat_device_gen qat_dev_gen,
672 struct qat_qp *qp, struct qat_queue *q) {
674 if (qat_dev_gen == QAT_GEN4) {
675 WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr,
676 q->hw_bundle_number, q->hw_queue_number, q->tail);
678 WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
679 q->hw_queue_number, q->tail);
684 void rxq_free_desc(enum qat_device_gen qat_dev_gen, struct qat_qp *qp,
687 uint32_t old_head, new_head;
690 old_head = q->csr_head;
692 max_head = qp->nb_descriptors * q->msg_size;
694 /* write out free descriptors */
695 void *cur_desc = (uint8_t *)q->base_addr + old_head;
697 if (new_head < old_head) {
698 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
699 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
701 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
703 q->nb_processed_responses = 0;
704 q->csr_head = new_head;
706 /* write current head to CSR */
707 if (qat_dev_gen == QAT_GEN4) {
708 WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr,
709 q->hw_bundle_number, q->hw_queue_number, new_head);
711 WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
712 q->hw_queue_number, new_head);
718 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
720 register struct qat_queue *queue;
721 struct qat_qp *tmp_qp = (struct qat_qp *)qp;
722 register uint32_t nb_ops_sent = 0;
723 register int ret = -1;
724 uint16_t nb_ops_possible = nb_ops;
725 register uint8_t *base_addr;
726 register uint32_t tail;
728 if (unlikely(nb_ops == 0))
731 /* read params used a lot in main loop into registers */
732 queue = &(tmp_qp->tx_q);
733 base_addr = (uint8_t *)queue->base_addr;
736 /* Find how many can actually fit on the ring */
738 /* dequeued can only be written by one thread, but it may not
739 * be this thread. As it's 4-byte aligned it will be read
740 * atomically here by any Intel CPU.
741 * enqueued can wrap before dequeued, but cannot
742 * lap it as var size of enq/deq (uint32_t) > var size of
743 * max_inflights (uint16_t). In reality inflights is never
744 * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
745 * On wrapping, the calculation still returns the correct
746 * positive value as all three vars are unsigned.
749 tmp_qp->enqueued - tmp_qp->dequeued;
751 if ((inflights + nb_ops) > tmp_qp->max_inflights) {
752 nb_ops_possible = tmp_qp->max_inflights - inflights;
753 if (nb_ops_possible == 0)
756 /* QAT has plenty of work queued already, so don't waste cycles
757 * enqueueing, wait til the application has gathered a bigger
758 * burst or some completed ops have been dequeued
760 if (tmp_qp->min_enq_burst_threshold && inflights >
761 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
762 tmp_qp->min_enq_burst_threshold) {
763 tmp_qp->stats.threshold_hit_count++;
769 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
770 qat_sym_preprocess_requests(ops, nb_ops_possible);
773 while (nb_ops_sent != nb_ops_possible) {
774 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
776 ret = qat_sym_build_request(*ops, base_addr + tail,
777 tmp_qp->op_cookies[tail >> queue->trailz],
778 tmp_qp->qat_dev_gen);
780 } else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
781 ret = qat_comp_build_request(*ops, base_addr + tail,
782 tmp_qp->op_cookies[tail >> queue->trailz],
783 tmp_qp->qat_dev_gen);
784 } else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
785 #ifdef BUILD_QAT_ASYM
786 ret = qat_asym_build_request(*ops, base_addr + tail,
787 tmp_qp->op_cookies[tail >> queue->trailz],
788 tmp_qp->qat_dev_gen);
792 tmp_qp->stats.enqueue_err_count++;
793 /* This message cannot be enqueued */
794 if (nb_ops_sent == 0)
799 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
805 tmp_qp->enqueued += nb_ops_sent;
806 tmp_qp->stats.enqueued_count += nb_ops_sent;
807 txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
811 /* Use this for compression only - but keep consistent with above common
812 * function as much as possible.
815 qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
817 register struct qat_queue *queue;
818 struct qat_qp *tmp_qp = (struct qat_qp *)qp;
819 register uint32_t nb_ops_sent = 0;
820 register int nb_desc_to_build;
821 uint16_t nb_ops_possible = nb_ops;
822 register uint8_t *base_addr;
823 register uint32_t tail;
825 int descriptors_built, total_descriptors_built = 0;
826 int nb_remaining_descriptors;
829 if (unlikely(nb_ops == 0))
832 /* read params used a lot in main loop into registers */
833 queue = &(tmp_qp->tx_q);
834 base_addr = (uint8_t *)queue->base_addr;
837 /* Find how many can actually fit on the ring */
839 /* dequeued can only be written by one thread, but it may not
840 * be this thread. As it's 4-byte aligned it will be read
841 * atomically here by any Intel CPU.
842 * enqueued can wrap before dequeued, but cannot
843 * lap it as var size of enq/deq (uint32_t) > var size of
844 * max_inflights (uint16_t). In reality inflights is never
845 * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
846 * On wrapping, the calculation still returns the correct
847 * positive value as all three vars are unsigned.
850 tmp_qp->enqueued - tmp_qp->dequeued;
852 /* Find how many can actually fit on the ring */
853 overflow = (inflights + nb_ops) - tmp_qp->max_inflights;
855 nb_ops_possible = nb_ops - overflow;
856 if (nb_ops_possible == 0)
860 /* QAT has plenty of work queued already, so don't waste cycles
861 * enqueueing, wait til the application has gathered a bigger
862 * burst or some completed ops have been dequeued
864 if (tmp_qp->min_enq_burst_threshold && inflights >
865 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
866 tmp_qp->min_enq_burst_threshold) {
867 tmp_qp->stats.threshold_hit_count++;
872 /* At this point nb_ops_possible is assuming a 1:1 mapping
873 * between ops and descriptors.
874 * Fewer may be sent if some ops have to be split.
875 * nb_ops_possible is <= burst size.
876 * Find out how many spaces are actually available on the qp in case
879 nb_remaining_descriptors = nb_ops_possible
880 + ((overflow >= 0) ? 0 : overflow * (-1));
881 QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d",
882 nb_ops, nb_remaining_descriptors);
884 while (nb_ops_sent != nb_ops_possible &&
885 nb_remaining_descriptors > 0) {
886 struct qat_comp_op_cookie *cookie =
887 tmp_qp->op_cookies[tail >> queue->trailz];
889 descriptors_built = 0;
891 QAT_DP_LOG(DEBUG, "--- data length: %u",
892 ((struct rte_comp_op *)*ops)->src.length);
894 nb_desc_to_build = qat_comp_build_request(*ops,
895 base_addr + tail, cookie, tmp_qp->qat_dev_gen);
896 QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, "
897 "%d ops sent, %d descriptors needed",
898 total_descriptors_built, nb_remaining_descriptors,
899 nb_ops_sent, nb_desc_to_build);
901 if (unlikely(nb_desc_to_build < 0)) {
902 /* this message cannot be enqueued */
903 tmp_qp->stats.enqueue_err_count++;
904 if (nb_ops_sent == 0)
907 } else if (unlikely(nb_desc_to_build > 1)) {
908 /* this op is too big and must be split - get more
909 * descriptors and retry
912 QAT_DP_LOG(DEBUG, "Build %d descriptors for this op",
915 nb_remaining_descriptors -= nb_desc_to_build;
916 if (nb_remaining_descriptors >= 0) {
917 /* There are enough remaining descriptors
920 int ret2 = qat_comp_build_multiple_requests(
924 if (unlikely(ret2 < 1)) {
926 "Failed to build (%d) descriptors, status %d",
927 nb_desc_to_build, ret2);
929 qat_comp_free_split_op_memzones(cookie,
930 nb_desc_to_build - 1);
932 tmp_qp->stats.enqueue_err_count++;
934 /* This message cannot be enqueued */
935 if (nb_ops_sent == 0)
939 descriptors_built = ret2;
940 total_descriptors_built +=
942 nb_remaining_descriptors -=
945 "Multiple descriptors (%d) built ok",
949 QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) "
950 "exceeds number of available descriptors (%d)",
952 nb_remaining_descriptors +
955 qat_comp_free_split_op_memzones(cookie,
956 nb_desc_to_build - 1);
958 /* Not enough extra descriptors */
959 if (nb_ops_sent == 0)
964 descriptors_built = 1;
965 total_descriptors_built++;
966 nb_remaining_descriptors--;
967 QAT_DP_LOG(DEBUG, "Single descriptor built ok");
970 tail = adf_modulo(tail + (queue->msg_size * descriptors_built),
978 tmp_qp->enqueued += total_descriptors_built;
979 tmp_qp->stats.enqueued_count += nb_ops_sent;
980 txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
985 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
987 struct qat_queue *rx_queue;
988 struct qat_qp *tmp_qp = (struct qat_qp *)qp;
990 uint32_t op_resp_counter = 0, fw_resp_counter = 0;
994 rx_queue = &(tmp_qp->rx_q);
995 head = rx_queue->head;
996 resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
998 while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
999 op_resp_counter != nb_ops) {
1001 nb_fw_responses = 1;
1003 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
1004 qat_sym_process_response(ops, resp_msg,
1005 tmp_qp->op_cookies[head >> rx_queue->trailz]);
1006 else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
1007 nb_fw_responses = qat_comp_process_response(
1009 tmp_qp->op_cookies[head >> rx_queue->trailz],
1010 &tmp_qp->stats.dequeue_err_count);
1011 #ifdef BUILD_QAT_ASYM
1012 else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
1013 qat_asym_process_response(ops, resp_msg,
1014 tmp_qp->op_cookies[head >> rx_queue->trailz]);
1017 head = adf_modulo(head + rx_queue->msg_size,
1018 rx_queue->modulo_mask);
1020 resp_msg = (uint8_t *)rx_queue->base_addr + head;
1022 if (nb_fw_responses) {
1023 /* only move on to next op if one was ready to return
1030 /* A compression op may be broken up into multiple fw requests.
1031 * Only count fw responses as complete once ALL the responses
1032 * associated with an op have been processed, as the cookie
1033 * data from the first response must be available until
1034 * finished with all firmware responses.
1036 fw_resp_counter += nb_fw_responses;
1038 rx_queue->nb_processed_responses++;
1041 tmp_qp->dequeued += fw_resp_counter;
1042 tmp_qp->stats.dequeued_count += op_resp_counter;
1044 rx_queue->head = head;
1045 if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
1046 rxq_free_desc(tmp_qp->qat_dev_gen, tmp_qp, rx_queue);
1048 QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u",
1049 op_resp_counter, fw_resp_counter);
1051 return op_resp_counter;
1054 /* This is almost same as dequeue_op_burst, without the atomic, without stats
1055 * and without the op. Dequeues one response.
1058 qat_cq_dequeue_response(struct qat_qp *qp, void *out_data)
1061 uint8_t retries = 0;
1062 struct qat_queue *queue = &(qp->rx_q);
1063 struct icp_qat_fw_comn_resp *resp_msg = (struct icp_qat_fw_comn_resp *)
1064 ((uint8_t *)queue->base_addr + queue->head);
1066 while (retries++ < QAT_CQ_MAX_DEQ_RETRIES &&
1067 *(uint32_t *)resp_msg == ADF_RING_EMPTY_SIG) {
1068 /* loop waiting for response until we reach the timeout */
1072 if (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG) {
1073 /* response received */
1076 /* check status flag */
1077 if (ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
1078 resp_msg->comn_hdr.comn_status) ==
1079 ICP_QAT_FW_COMN_STATUS_FLAG_OK) {
1081 memcpy(out_data, resp_msg, queue->msg_size);
1083 memset(out_data, 0, queue->msg_size);
1086 queue->head = adf_modulo(queue->head + queue->msg_size,
1087 queue->modulo_mask);
1088 rxq_free_desc(qp->qat_dev_gen, qp, queue);
1094 /* Sends a NULL message and extracts QAT fw version from the response.
1095 * Used to determine detailed capabilities based on the fw version number.
1096 * This assumes that there are no inflight messages, i.e. assumes there's space
1097 * on the qp, one message is sent and only one response collected.
1098 * Returns fw version number or 0 for unknown version or a negative error code.
1101 qat_cq_get_fw_version(struct qat_qp *qp)
1103 struct qat_queue *queue = &(qp->tx_q);
1104 uint8_t *base_addr = (uint8_t *)queue->base_addr;
1105 struct icp_qat_fw_comn_req null_msg;
1106 struct icp_qat_fw_comn_resp response;
1108 /* prepare the NULL request */
1109 memset(&null_msg, 0, sizeof(null_msg));
1110 null_msg.comn_hdr.hdr_flags =
1111 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
1112 null_msg.comn_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
1113 null_msg.comn_hdr.service_cmd_id = ICP_QAT_FW_NULL_REQ_SERV_ID;
1115 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
1116 QAT_DP_HEXDUMP_LOG(DEBUG, "NULL request", &null_msg, sizeof(null_msg));
1119 /* send the NULL request */
1120 memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg));
1121 queue->tail = adf_modulo(queue->tail + queue->msg_size,
1122 queue->modulo_mask);
1123 txq_write_tail(qp->qat_dev_gen, qp, queue);
1125 /* receive a response */
1126 if (qat_cq_dequeue_response(qp, &response)) {
1128 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
1129 QAT_DP_HEXDUMP_LOG(DEBUG, "NULL response:", &response,
1132 /* if LW0 bit 24 is set - then the fw version was returned */
1133 if (QAT_FIELD_GET(response.comn_hdr.hdr_flags,
1134 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_BITPOS,
1135 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_MASK))
1136 return response.resrvd[0]; /* return LW4 */
1138 return 0; /* not set - we don't know fw version */
1141 QAT_LOG(ERR, "No response received");
1146 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
1147 void *op_cookie __rte_unused,
1148 uint64_t *dequeue_err_count __rte_unused)