crypto/qat: add minimum enqueue threshold

This patch adds minimum enqueue threshold to Intel
QuickAssist Technology PMD.
It is an optimisation, configured by a command line option,
which can be used to reduce MMIO write occurrences.

Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>

diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst
index 3a4a189..12b8e79 100644 (file)
--- a/doc/guides/cryptodevs/qat.rst
+++ b/doc/guides/cryptodevs/qat.rst
@@ -243,6 +243,29 @@ allocated while for GEN1 devices, 12 buffers are allocated, plus 1472 bytes over
        larger than the input size).
 
 
+Running QAT PMD with minimum threshold for burst size
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If only a small number or packets can be enqueued. Each enqueue causes an expensive MMIO write.
+These MMIO write occurrences can be optimised by setting any of the following parameters:
+
+- qat_sym_enq_threshold
+- qat_asym_enq_threshold
+- qat_comp_enq_threshold
+
+When any of these parameters is set rte_cryptodev_enqueue_burst function will
+return 0 (thereby avoiding an MMIO) if the device is congested and number of packets
+possible to enqueue is smaller.
+To use this feature the user must set the parameter on process start as a device additional parameter::
+
+  -w 03:01.1,qat_sym_enq_threshold=32,qat_comp_enq_threshold=16
+
+All parameters can be used with the same device regardless of order. Parameters are separated
+by comma. When the same parameter is used more than once first occurrence of the parameter
+is used.
+Maximum threshold that can be set is 32.
+
+
 Device and driver naming
 ~~~~~~~~~~~~~~~~~~~~~~~~
 

diff --git a/drivers/common/qat/qat_common.c b/drivers/common/qat/qat_common.c
index 4753866..5343a14 100644 (file)
--- a/drivers/common/qat/qat_common.c
+++ b/drivers/common/qat/qat_common.c
@@ -94,6 +94,9 @@ void qat_stats_get(struct qat_pci_device *dev,
                stats->dequeued_count += qp[i]->stats.dequeued_count;
                stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
                stats->dequeue_err_count += qp[i]->stats.dequeue_err_count;
+               stats->threshold_hit_count += qp[i]->stats.threshold_hit_count;
+               QAT_LOG(DEBUG, "Threshold was used for qp %d %"PRIu64" times",
+                               i, stats->threshold_hit_count);
        }
 }
 

diff --git a/drivers/common/qat/qat_common.h b/drivers/common/qat/qat_common.h
index de9a3ba..cf840fe 100644 (file)
--- a/drivers/common/qat/qat_common.h
+++ b/drivers/common/qat/qat_common.h
@@ -61,6 +61,9 @@ struct qat_common_stats {
        /**< Total error count on operations enqueued */
        uint64_t dequeue_err_count;
        /**< Total error count on operations dequeued */
+       uint64_t threshold_hit_count;
+       /**< Total number of times min qp threshold condition was fulfilled */
+
 };
 
 struct qat_pci_device;

diff --git a/drivers/common/qat/qat_device.c b/drivers/common/qat/qat_device.c
index 2a1cf3e..2b41d9a 100644 (file)
--- a/drivers/common/qat/qat_device.c
+++ b/drivers/common/qat/qat_device.c
@@ -3,6 +3,8 @@
  */
 
 #include <rte_string_fns.h>
+#include <rte_devargs.h>
+#include <ctype.h>
 
 #include "qat_device.h"
 #include "adf_transport_access_macros.h"
@@ -105,12 +107,71 @@ qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
        return qat_pci_get_named_dev(name);
 }
 
+static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
+               *qat_dev_cmd_param)
+{
+       int i = 0;
+       const char *param;
+
+       while (1) {
+               char value_str[4] = { };
+
+               param = qat_dev_cmd_param[i].name;
+               if (param == NULL)
+                       return;
+               long value = 0;
+               const char *arg = strstr(str, param);
+               const char *arg2 = NULL;
+
+               if (arg) {
+                       arg2 = arg + strlen(param);
+                       if (*arg2 != '=') {
+                               QAT_LOG(DEBUG, "parsing error '=' sign"
+                                               " should immediately follow %s",
+                                               param);
+                               arg2 = NULL;
+                       } else
+                               arg2++;
+               } else {
+                       QAT_LOG(DEBUG, "%s not provided", param);
+               }
+               if (arg2) {
+                       int iter = 0;
+                       while (iter < 2) {
+                               if (!isdigit(*(arg2 + iter)))
+                                       break;
+                               iter++;
+                       }
+                       if (!iter) {
+                               QAT_LOG(DEBUG, "parsing error %s"
+                                              " no number provided",
+                                              param);
+                       } else {
+                               memcpy(value_str, arg2, iter);
+                               value = strtol(value_str, NULL, 10);
+                               if (value > MAX_QP_THRESHOLD_SIZE) {
+                                       QAT_LOG(DEBUG, "Exceeded max size of"
+                                               " threshold, setting to %d",
+                                               MAX_QP_THRESHOLD_SIZE);
+                                       value = MAX_QP_THRESHOLD_SIZE;
+                               }
+                               QAT_LOG(DEBUG, "parsing %s = %ld",
+                                               param, value);
+                       }
+               }
+               qat_dev_cmd_param[i].val = value;
+               i++;
+       }
+}
+
 struct qat_pci_device *
-qat_pci_device_allocate(struct rte_pci_device *pci_dev)
+qat_pci_device_allocate(struct rte_pci_device *pci_dev,
+               struct qat_dev_cmd_param *qat_dev_cmd_param)
 {
        struct qat_pci_device *qat_dev;
        uint8_t qat_dev_id;
        char name[QAT_DEV_NAME_MAX_LEN];
+       struct rte_devargs *devargs = pci_dev->device.devargs;
 
        rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
        snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
@@ -148,6 +209,9 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev)
                return NULL;
        }
 
+       if (devargs && devargs->drv_str)
+               qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
+
        rte_spinlock_init(&qat_dev->arb_csr_lock);
 
        qat_dev->attached = QAT_ATTACHED;
@@ -199,37 +263,45 @@ qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
                struct rte_pci_device *pci_dev)
 {
-       int ret = 0;
+       int sym_ret = 0, asym_ret = 0, comp_ret = 0;
        int num_pmds_created = 0;
        struct qat_pci_device *qat_pci_dev;
+       struct qat_dev_cmd_param qat_dev_cmd_param[] = {
+                       { SYM_ENQ_THRESHOLD_NAME, 0 },
+                       { ASYM_ENQ_THRESHOLD_NAME, 0 },
+                       { COMP_ENQ_THRESHOLD_NAME, 0 },
+                       { NULL, 0 },
+       };
 
        QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
                        pci_dev->addr.bus,
                        pci_dev->addr.devid,
                        pci_dev->addr.function);
 
-       qat_pci_dev = qat_pci_device_allocate(pci_dev);
+       qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param);
        if (qat_pci_dev == NULL)
                return -ENODEV;
 
-       ret = qat_sym_dev_create(qat_pci_dev);
-       if (ret == 0)
+       sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param);
+       if (sym_ret == 0) {
                num_pmds_created++;
+
+       }
        else
                QAT_LOG(WARNING,
                                "Failed to create QAT SYM PMD on device %s",
                                qat_pci_dev->name);
 
-       ret = qat_comp_dev_create(qat_pci_dev);
-       if (ret == 0)
+       comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param);
+       if (comp_ret == 0)
                num_pmds_created++;
        else
                QAT_LOG(WARNING,
                                "Failed to create QAT COMP PMD on device %s",
                                qat_pci_dev->name);
 
-       ret = qat_asym_dev_create(qat_pci_dev);
-       if (ret == 0)
+       asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param);
+       if (asym_ret == 0)
                num_pmds_created++;
        else
                QAT_LOG(WARNING,
@@ -264,13 +336,15 @@ static struct rte_pci_driver rte_qat_pmd = {
 };
 
 __rte_weak int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+               struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 {
        return 0;
 }
 
 __rte_weak int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+               struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 {
        return 0;
 }
@@ -288,7 +362,8 @@ qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 }
 
 __rte_weak int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused)
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+               struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 {
        return 0;
 }

diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h
index 131375e..09a4c55 100644 (file)
--- a/drivers/common/qat/qat_device.h
+++ b/drivers/common/qat/qat_device.h
@@ -16,6 +16,16 @@
 
 #define QAT_DEV_NAME_MAX_LEN   64
 
+#define SYM_ENQ_THRESHOLD_NAME "qat_sym_enq_threshold"
+#define ASYM_ENQ_THRESHOLD_NAME "qat_asym_enq_threshold"
+#define COMP_ENQ_THRESHOLD_NAME "qat_comp_enq_threshold"
+#define MAX_QP_THRESHOLD_SIZE  32
+
+struct qat_dev_cmd_param {
+       const char *name;
+       uint16_t val;
+};
+
 enum qat_comp_num_im_buffers {
        QAT_NUM_INTERM_BUFS_GEN1 = 12,
        QAT_NUM_INTERM_BUFS_GEN2 = 20,
@@ -94,7 +104,8 @@ struct qat_gen_hw_data {
 extern struct qat_gen_hw_data qat_gen_config[];
 
 struct qat_pci_device *
-qat_pci_device_allocate(struct rte_pci_device *pci_dev);
+qat_pci_device_allocate(struct rte_pci_device *pci_dev,
+               struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_pci_device_release(struct rte_pci_device *pci_dev);
@@ -104,10 +115,12 @@ qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev);
 
 /* declaration needed for weak functions */
 int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+               struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+               struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
@@ -116,7 +129,8 @@ int
 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);
 
 int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused);
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
+               struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused);

diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c
index 30cdc61..9958789 100644 (file)
--- a/drivers/common/qat/qat_qp.c
+++ b/drivers/common/qat/qat_qp.c
@@ -608,8 +608,19 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
                        if (nb_ops_possible == 0)
                                return 0;
                }
+               /* QAT has plenty of work queued already, so don't waste cycles
+                * enqueueing, wait til the application has gathered a bigger
+                * burst or some completed ops have been dequeued
+                */
+               if (tmp_qp->min_enq_burst_threshold && inflights >
+                               QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
+                               tmp_qp->min_enq_burst_threshold) {
+                       tmp_qp->stats.threshold_hit_count++;
+                       return 0;
+               }
        }
 
+
        while (nb_ops_sent != nb_ops_possible) {
                ret = tmp_qp->build_request(*ops, base_addr + tail,
                                tmp_qp->op_cookies[tail / queue->msg_size],

diff --git a/drivers/common/qat/qat_qp.h b/drivers/common/qat/qat_qp.h
index 8b9ab79..0b95ea3 100644 (file)
--- a/drivers/common/qat/qat_qp.h
+++ b/drivers/common/qat/qat_qp.h
@@ -12,6 +12,8 @@ struct qat_pci_device;
 #define QAT_CSR_HEAD_WRITE_THRESH 32U
 /* number of requests to accumulate before writing head CSR */
 
+#define QAT_QP_MIN_INFL_THRESHOLD      256
+
 typedef int (*build_request_t)(void *op,
                uint8_t *req, void *op_cookie,
                enum qat_device_gen qat_dev_gen);
@@ -77,6 +79,7 @@ struct qat_qp {
        uint32_t enqueued;
        uint32_t dequeued __rte_aligned(4);
        uint16_t max_inflights;
+       uint16_t min_enq_burst_threshold;
 } __rte_cache_aligned;
 
 extern const struct qat_qp_hw_data qat_gen1_qps[][ADF_MAX_QPS_ON_ANY_SERVICE];

diff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c
index 05b7dfe..7d4fdf1 100644 (file)
--- a/drivers/compress/qat/qat_comp_pmd.c
+++ b/drivers/compress/qat/qat_comp_pmd.c
@@ -139,6 +139,7 @@ qat_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
                                                                = *qp_addr;
 
        qp = (struct qat_qp *)*qp_addr;
+       qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
 
        for (i = 0; i < qp->nb_descriptors; i++) {
 
@@ -660,8 +661,10 @@ static const struct rte_driver compdev_qat_driver = {
        .alias = qat_comp_drv_name
 };
 int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev,
+               struct qat_dev_cmd_param *qat_dev_cmd_param)
 {
+       int i = 0;
        if (qat_pci_dev->qat_dev_gen == QAT_GEN3) {
                QAT_LOG(ERR, "Compression PMD not supported on QAT c4xxx");
                return 0;
@@ -719,6 +722,15 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev)
                break;
        }
 
+       while (1) {
+               if (qat_dev_cmd_param[i].name == NULL)
+                       break;
+               if (!strcmp(qat_dev_cmd_param[i].name, COMP_ENQ_THRESHOLD_NAME))
+                       comp_dev->min_enq_burst_threshold =
+                                       qat_dev_cmd_param[i].val;
+               i++;
+       }
+
        QAT_LOG(DEBUG,
                    "Created QAT COMP device %s as compressdev instance %d",
                        name, compressdev->data->dev_id);

diff --git a/drivers/compress/qat/qat_comp_pmd.h b/drivers/compress/qat/qat_comp_pmd.h
index 6979de1..5c7fa9f 100644 (file)
--- a/drivers/compress/qat/qat_comp_pmd.h
+++ b/drivers/compress/qat/qat_comp_pmd.h
@@ -32,10 +32,12 @@ struct qat_comp_dev_private {
        /**< The device's pool for qat_comp_xforms */
        struct rte_mempool *streampool;
        /**< The device's pool for qat_comp_streams */
+       uint16_t min_enq_burst_threshold;
 };
 
 int
-qat_comp_dev_create(struct qat_pci_device *qat_pci_dev);
+qat_comp_dev_create(struct qat_pci_device *qat_pci_dev,
+               struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev);

diff --git a/drivers/crypto/qat/qat_asym_pmd.c b/drivers/crypto/qat/qat_asym_pmd.c
index c8a52b6..cdf10de 100644 (file)
--- a/drivers/crypto/qat/qat_asym_pmd.c
+++ b/drivers/crypto/qat/qat_asym_pmd.c
@@ -160,6 +160,7 @@ static int qat_asym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
                                                        = *qp_addr;
 
        qp = (struct qat_qp *)*qp_addr;
+       qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
 
        for (i = 0; i < qp->nb_descriptors; i++) {
                int j;
@@ -235,8 +236,10 @@ static const struct rte_driver cryptodev_qat_asym_driver = {
 };
 
 int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev)
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
+               struct qat_dev_cmd_param *qat_dev_cmd_param)
 {
+       int i = 0;
        struct rte_cryptodev_pmd_init_params init_params = {
                        .name = "",
                        .socket_id = qat_pci_dev->pci_dev->device.numa_node,
@@ -281,6 +284,15 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev)
        internals->asym_dev_id = cryptodev->data->dev_id;
        internals->qat_dev_capabilities = qat_gen1_asym_capabilities;
 
+       while (1) {
+               if (qat_dev_cmd_param[i].name == NULL)
+                       break;
+               if (!strcmp(qat_dev_cmd_param[i].name, ASYM_ENQ_THRESHOLD_NAME))
+                       internals->min_enq_burst_threshold =
+                                       qat_dev_cmd_param[i].val;
+               i++;
+       }
+
        QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
                        cryptodev->data->name, internals->asym_dev_id);
        return 0;

diff --git a/drivers/crypto/qat/qat_asym_pmd.h b/drivers/crypto/qat/qat_asym_pmd.h
index 895d0f6..0535bc6 100644 (file)
--- a/drivers/crypto/qat/qat_asym_pmd.h
+++ b/drivers/crypto/qat/qat_asym_pmd.h
@@ -26,6 +26,7 @@ struct qat_asym_dev_private {
        /**< Device instance for this rte_cryptodev */
        const struct rte_cryptodev_capabilities *qat_dev_capabilities;
        /* QAT device asymmetric crypto capabilities */
+       uint16_t min_enq_burst_threshold;
 };
 
 uint16_t
@@ -42,7 +43,8 @@ int qat_asym_session_configure(struct rte_cryptodev *dev,
                struct rte_mempool *mempool);
 
 int
-qat_asym_dev_create(struct qat_pci_device *qat_pci_dev);
+qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
+               struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev);

diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c
index 71f21ce..666ede7 100644 (file)
--- a/drivers/crypto/qat/qat_sym_pmd.c
+++ b/drivers/crypto/qat/qat_sym_pmd.c
@@ -169,6 +169,7 @@ static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
                                                        = *qp_addr;
 
        qp = (struct qat_qp *)*qp_addr;
+       qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
 
        for (i = 0; i < qp->nb_descriptors; i++) {
 
@@ -237,8 +238,10 @@ static const struct rte_driver cryptodev_qat_sym_driver = {
 };
 
 int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
+               struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 {
+       int i = 0;
        struct rte_cryptodev_pmd_init_params init_params = {
                        .name = "",
                        .socket_id = qat_pci_dev->pci_dev->device.numa_node,
@@ -302,6 +305,15 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev)
                break;
        }
 
+       while (1) {
+               if (qat_dev_cmd_param[i].name == NULL)
+                       break;
+               if (!strcmp(qat_dev_cmd_param[i].name, SYM_ENQ_THRESHOLD_NAME))
+                       internals->min_enq_burst_threshold =
+                                       qat_dev_cmd_param[i].val;
+               i++;
+       }
+
        QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
                        cryptodev->data->name, internals->sym_dev_id);
        return 0;

diff --git a/drivers/crypto/qat/qat_sym_pmd.h b/drivers/crypto/qat/qat_sym_pmd.h
index 7ddaf45..a32c25a 100644 (file)
--- a/drivers/crypto/qat/qat_sym_pmd.h
+++ b/drivers/crypto/qat/qat_sym_pmd.h
@@ -28,10 +28,12 @@ struct qat_sym_dev_private {
        /**< Device instance for this rte_cryptodev */
        const struct rte_cryptodev_capabilities *qat_dev_capabilities;
        /* QAT device symmetric crypto capabilities */
+       uint16_t min_enq_burst_threshold;
 };
 
 int
-qat_sym_dev_create(struct qat_pci_device *qat_pci_dev);
+qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
+               struct qat_dev_cmd_param *qat_dev_cmd_param);
 
 int
 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev);