crypto/qat: enable Rx head writes coalescing

[dpdk.git] / drivers / crypto / qat / qat_crypto.c

diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c
index 62ee175..1656e0f 100644 (file)
--- a/drivers/crypto/qat/qat_crypto.c
+++ b/drivers/crypto/qat/qat_crypto.c
@@ -51,7 +51,6 @@
 #include <rte_eal.h>
 #include <rte_per_lcore.h>
 #include <rte_lcore.h>
-#include <rte_atomic.h>
 #include <rte_branch_prediction.h>
 #include <rte_mempool.h>
 #include <rte_mbuf.h>
@@ -945,10 +944,10 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
        tail = queue->tail;
 
        /* Find how many can actually fit on the ring */
-       overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops)
-                               - queue->max_inflights;
+       tmp_qp->inflights16 += nb_ops;
+       overflow = tmp_qp->inflights16 - queue->max_inflights;
        if (overflow > 0) {
-               rte_atomic16_sub(&tmp_qp->inflights16, overflow);
+               tmp_qp->inflights16 -= overflow;
                nb_ops_possible = nb_ops - overflow;
                if (nb_ops_possible == 0)
                        return 0;
@@ -963,8 +962,7 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
                         * This message cannot be enqueued,
                         * decrease number of ops that wasn't sent
                         */
-                       rte_atomic16_sub(&tmp_qp->inflights16,
-                                       nb_ops_possible - nb_ops_sent);
+                       tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
                        if (nb_ops_sent == 0)
                                return 0;
                        goto kick_tail;
@@ -982,6 +980,33 @@ kick_tail:
        return nb_ops_sent;
 }
 
+static inline
+void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
+{
+       uint32_t old_head, new_head;
+       uint32_t max_head;
+
+       old_head = q->csr_head;
+       new_head = q->head;
+       max_head = qp->nb_descriptors * q->msg_size;
+
+       /* write out free descriptors */
+       void *cur_desc = (uint8_t *)q->base_addr + old_head;
+
+       if (new_head < old_head) {
+               memset(cur_desc, ADF_RING_EMPTY_SIG, max_head - old_head);
+               memset(q->base_addr, ADF_RING_EMPTY_SIG, new_head);
+       } else {
+               memset(cur_desc, ADF_RING_EMPTY_SIG, new_head - old_head);
+       }
+       q->nb_processed_responses = 0;
+       q->csr_head = new_head;
+
+       /* write current head to CSR */
+       WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
+                           q->hw_queue_number, new_head);
+}
+
 uint16_t
 qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
                uint16_t nb_ops)
@@ -991,10 +1016,12 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
        uint32_t msg_counter = 0;
        struct rte_crypto_op *rx_op;
        struct icp_qat_fw_comn_resp *resp_msg;
+       uint32_t head;
 
        queue = &(tmp_qp->rx_q);
+       head = queue->head;
        resp_msg = (struct icp_qat_fw_comn_resp *)
-                       ((uint8_t *)queue->base_addr + queue->head);
+                       ((uint8_t *)queue->base_addr + head);
 
        while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
                        msg_counter != nb_ops) {
@@ -1021,23 +1048,21 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
                        rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
                }
 
-               *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG;
-               queue->head = adf_modulo(queue->head +
-                               queue->msg_size,
-                               ADF_RING_SIZE_MODULO(queue->queue_size));
+               head = adf_modulo(head + queue->msg_size, queue->modulo);
                resp_msg = (struct icp_qat_fw_comn_resp *)
-                                       ((uint8_t *)queue->base_addr +
-                                                       queue->head);
+                               ((uint8_t *)queue->base_addr + head);
                *ops = rx_op;
                ops++;
                msg_counter++;
        }
        if (msg_counter > 0) {
-               WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr,
-                                       queue->hw_bundle_number,
-                                       queue->hw_queue_number, queue->head);
-               rte_atomic16_sub(&tmp_qp->inflights16, msg_counter);
+               queue->head = head;
                tmp_qp->stats.dequeued_count += msg_counter;
+               queue->nb_processed_responses += msg_counter;
+               tmp_qp->inflights16 -= msg_counter;
+
+               if (queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
+                       rxq_free_desc(tmp_qp, queue);
        }
        return msg_counter;
 }