drivers/common/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_cycles.h>
   7 #include <rte_dev.h>
   8 #include <rte_malloc.h>
   9 #include <rte_memzone.h>
  10 #include <rte_pci.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_atomic.h>
  13 #include <rte_prefetch.h>
  14
  15 #include "qat_logs.h"
  16 #include "qat_device.h"
  17 #include "qat_qp.h"
  18 #include "qat_sym.h"
  19 #include "qat_asym.h"
  20 #include "qat_comp.h"
  21 #include "adf_transport_access_macros.h"
  22
  23 #define QAT_CQ_MAX_DEQ_RETRIES 10
  24
  25 #define ADF_MAX_DESC                            4096
  26 #define ADF_MIN_DESC                            128
  27
  28 #define ADF_ARB_REG_SLOT                        0x1000
  29 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  30
  31 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  32         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  33         (ADF_ARB_REG_SLOT * index), value)
  34
  35 __extension__
  36 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
  37                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  38         /* queue pairs which provide an asymmetric crypto service */
  39         [QAT_SERVICE_ASYMMETRIC] = {
  40                 {
  41                         .service_type = QAT_SERVICE_ASYMMETRIC,
  42                         .hw_bundle_num = 0,
  43                         .tx_ring_num = 0,
  44                         .rx_ring_num = 8,
  45                         .tx_msg_size = 64,
  46                         .rx_msg_size = 32,
  47
  48                 }, {
  49                         .service_type = QAT_SERVICE_ASYMMETRIC,
  50                         .hw_bundle_num = 0,
  51                         .tx_ring_num = 1,
  52                         .rx_ring_num = 9,
  53                         .tx_msg_size = 64,
  54                         .rx_msg_size = 32,
  55                 }
  56         },
  57         /* queue pairs which provide a symmetric crypto service */
  58         [QAT_SERVICE_SYMMETRIC] = {
  59                 {
  60                         .service_type = QAT_SERVICE_SYMMETRIC,
  61                         .hw_bundle_num = 0,
  62                         .tx_ring_num = 2,
  63                         .rx_ring_num = 10,
  64                         .tx_msg_size = 128,
  65                         .rx_msg_size = 32,
  66                 },
  67                 {
  68                         .service_type = QAT_SERVICE_SYMMETRIC,
  69                         .hw_bundle_num = 0,
  70                         .tx_ring_num = 3,
  71                         .rx_ring_num = 11,
  72                         .tx_msg_size = 128,
  73                         .rx_msg_size = 32,
  74                 }
  75         },
  76         /* queue pairs which provide a compression service */
  77         [QAT_SERVICE_COMPRESSION] = {
  78                 {
  79                         .service_type = QAT_SERVICE_COMPRESSION,
  80                         .hw_bundle_num = 0,
  81                         .tx_ring_num = 6,
  82                         .rx_ring_num = 14,
  83                         .tx_msg_size = 128,
  84                         .rx_msg_size = 32,
  85                 }, {
  86                         .service_type = QAT_SERVICE_COMPRESSION,
  87                         .hw_bundle_num = 0,
  88                         .tx_ring_num = 7,
  89                         .rx_ring_num = 15,
  90                         .tx_msg_size = 128,
  91                         .rx_msg_size = 32,
  92                 }
  93         }
  94 };
  95
  96 __extension__
  97 const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
  98                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  99         /* queue pairs which provide an asymmetric crypto service */
 100         [QAT_SERVICE_ASYMMETRIC] = {
 101                 {
 102                         .service_type = QAT_SERVICE_ASYMMETRIC,
 103                         .hw_bundle_num = 0,
 104                         .tx_ring_num = 0,
 105                         .rx_ring_num = 4,
 106                         .tx_msg_size = 64,
 107                         .rx_msg_size = 32,
 108                 }
 109         },
 110         /* queue pairs which provide a symmetric crypto service */
 111         [QAT_SERVICE_SYMMETRIC] = {
 112                 {
 113                         .service_type = QAT_SERVICE_SYMMETRIC,
 114                         .hw_bundle_num = 0,
 115                         .tx_ring_num = 1,
 116                         .rx_ring_num = 5,
 117                         .tx_msg_size = 128,
 118                         .rx_msg_size = 32,
 119                 }
 120         },
 121         /* queue pairs which provide a compression service */
 122         [QAT_SERVICE_COMPRESSION] = {
 123                 {
 124                         .service_type = QAT_SERVICE_COMPRESSION,
 125                         .hw_bundle_num = 0,
 126                         .tx_ring_num = 3,
 127                         .rx_ring_num = 7,
 128                         .tx_msg_size = 128,
 129                         .rx_msg_size = 32,
 130                 }
 131         }
 132 };
 133
 134 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 135         uint32_t queue_size_bytes);
 136 static void qat_queue_delete(struct qat_queue *queue);
 137 static int qat_queue_create(struct qat_pci_device *qat_dev,
 138         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
 139 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 140         uint32_t *queue_size_for_csr);
 141 static void adf_configure_queues(struct qat_qp *queue);
 142 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 143         rte_spinlock_t *lock);
 144 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 145         rte_spinlock_t *lock);
 146
 147
 148 int qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data,
 149                 enum qat_service_type service)
 150 {
 151         int i, count;
 152
 153         for (i = 0, count = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++)
 154                 if (qp_hw_data[i].service_type == service)
 155                         count++;
 156         return count;
 157 }
 158
 159 static const struct rte_memzone *
 160 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 161                         int socket_id)
 162 {
 163         const struct rte_memzone *mz;
 164
 165         mz = rte_memzone_lookup(queue_name);
 166         if (mz != 0) {
 167                 if (((size_t)queue_size <= mz->len) &&
 168                                 ((socket_id == SOCKET_ID_ANY) ||
 169                                         (socket_id == mz->socket_id))) {
 170                         QAT_LOG(DEBUG, "re-use memzone already "
 171                                         "allocated for %s", queue_name);
 172                         return mz;
 173                 }
 174
 175                 QAT_LOG(ERR, "Incompatible memzone already "
 176                                 "allocated %s, size %u, socket %d. "
 177                                 "Requested size %u, socket %u",
 178                                 queue_name, (uint32_t)mz->len,
 179                                 mz->socket_id, queue_size, socket_id);
 180                 return NULL;
 181         }
 182
 183         QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
 184                                         queue_name, queue_size, socket_id);
 185         return rte_memzone_reserve_aligned(queue_name, queue_size,
 186                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 187 }
 188
 189 int qat_qp_setup(struct qat_pci_device *qat_dev,
 190                 struct qat_qp **qp_addr,
 191                 uint16_t queue_pair_id,
 192                 struct qat_qp_config *qat_qp_conf)
 193
 194 {
 195         struct qat_qp *qp;
 196         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 197         char op_cookie_pool_name[RTE_RING_NAMESIZE];
 198         uint32_t i;
 199
 200         QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
 201                 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
 202
 203         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
 204                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
 205                 QAT_LOG(ERR, "Can't create qp for %u descriptors",
 206                                 qat_qp_conf->nb_descriptors);
 207                 return -EINVAL;
 208         }
 209
 210         if (pci_dev->mem_resource[0].addr == NULL) {
 211                 QAT_LOG(ERR, "Could not find VF config space "
 212                                 "(UIO driver attached?).");
 213                 return -EINVAL;
 214         }
 215
 216         /* Allocate the queue pair data structure. */
 217         qp = rte_zmalloc_socket("qat PMD qp metadata",
 218                                 sizeof(*qp), RTE_CACHE_LINE_SIZE,
 219                                 qat_qp_conf->socket_id);
 220         if (qp == NULL) {
 221                 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
 222                 return -ENOMEM;
 223         }
 224         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
 225         qp->op_cookies = rte_zmalloc_socket("qat PMD op cookie pointer",
 226                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 227                         RTE_CACHE_LINE_SIZE, qat_qp_conf->socket_id);
 228         if (qp->op_cookies == NULL) {
 229                 QAT_LOG(ERR, "Failed to alloc mem for cookie");
 230                 rte_free(qp);
 231                 return -ENOMEM;
 232         }
 233
 234         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 235         qp->enqueued = qp->dequeued = 0;
 236
 237         if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
 238                                         ADF_RING_DIR_TX) != 0) {
 239                 QAT_LOG(ERR, "Tx queue create failed "
 240                                 "queue_pair_id=%u", queue_pair_id);
 241                 goto create_err;
 242         }
 243
 244         qp->max_inflights = ADF_MAX_INFLIGHTS(qp->tx_q.queue_size,
 245                                 ADF_BYTES_TO_MSG_SIZE(qp->tx_q.msg_size));
 246
 247         if (qp->max_inflights < 2) {
 248                 QAT_LOG(ERR, "Invalid num inflights");
 249                 qat_queue_delete(&(qp->tx_q));
 250                 goto create_err;
 251         }
 252
 253         if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
 254                                         ADF_RING_DIR_RX) != 0) {
 255                 QAT_LOG(ERR, "Rx queue create failed "
 256                                 "queue_pair_id=%hu", queue_pair_id);
 257                 qat_queue_delete(&(qp->tx_q));
 258                 goto create_err;
 259         }
 260
 261         adf_configure_queues(qp);
 262         adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
 263                                         &qat_dev->arb_csr_lock);
 264
 265         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
 266                                         "%s%d_cookies_%s_qp%hu",
 267                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 268                 qat_qp_conf->service_str, queue_pair_id);
 269
 270         QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
 271         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 272         if (qp->op_cookie_pool == NULL)
 273                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 274                                 qp->nb_descriptors,
 275                                 qat_qp_conf->cookie_size, 64, 0,
 276                                 NULL, NULL, NULL, NULL,
 277                                 qat_dev->pci_dev->device.numa_node,
 278                                 0);
 279         if (!qp->op_cookie_pool) {
 280                 QAT_LOG(ERR, "QAT PMD Cannot create"
 281                                 " op mempool");
 282                 goto create_err;
 283         }
 284
 285         for (i = 0; i < qp->nb_descriptors; i++) {
 286                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 287                         QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
 288                         goto create_err;
 289                 }
 290                 memset(qp->op_cookies[i], 0, qat_qp_conf->cookie_size);
 291         }
 292
 293         qp->qat_dev_gen = qat_dev->qat_dev_gen;
 294         qp->build_request = qat_qp_conf->build_request;
 295         qp->service_type = qat_qp_conf->hw->service_type;
 296         qp->qat_dev = qat_dev;
 297
 298         QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
 299                         queue_pair_id, op_cookie_pool_name);
 300
 301         *qp_addr = qp;
 302         return 0;
 303
 304 create_err:
 305         if (qp->op_cookie_pool)
 306                 rte_mempool_free(qp->op_cookie_pool);
 307         rte_free(qp->op_cookies);
 308         rte_free(qp);
 309         return -EFAULT;
 310 }
 311
 312 int qat_qp_release(struct qat_qp **qp_addr)
 313 {
 314         struct qat_qp *qp = *qp_addr;
 315         uint32_t i;
 316
 317         if (qp == NULL) {
 318                 QAT_LOG(DEBUG, "qp already freed");
 319                 return 0;
 320         }
 321
 322         QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
 323                                 qp->qat_dev->qat_dev_id);
 324
 325         /* Don't free memory if there are still responses to be processed */
 326         if ((qp->enqueued - qp->dequeued) == 0) {
 327                 qat_queue_delete(&(qp->tx_q));
 328                 qat_queue_delete(&(qp->rx_q));
 329         } else {
 330                 return -EAGAIN;
 331         }
 332
 333         adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
 334                                         &qp->qat_dev->arb_csr_lock);
 335
 336         for (i = 0; i < qp->nb_descriptors; i++)
 337                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 338
 339         if (qp->op_cookie_pool)
 340                 rte_mempool_free(qp->op_cookie_pool);
 341
 342         rte_free(qp->op_cookies);
 343         rte_free(qp);
 344         *qp_addr = NULL;
 345         return 0;
 346 }
 347
 348
 349 static void qat_queue_delete(struct qat_queue *queue)
 350 {
 351         const struct rte_memzone *mz;
 352         int status = 0;
 353
 354         if (queue == NULL) {
 355                 QAT_LOG(DEBUG, "Invalid queue");
 356                 return;
 357         }
 358         QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
 359                         queue->hw_queue_number, queue->memz_name);
 360
 361         mz = rte_memzone_lookup(queue->memz_name);
 362         if (mz != NULL) {
 363                 /* Write an unused pattern to the queue memory. */
 364                 memset(queue->base_addr, 0x7F, queue->queue_size);
 365                 status = rte_memzone_free(mz);
 366                 if (status != 0)
 367                         QAT_LOG(ERR, "Error %d on freeing queue %s",
 368                                         status, queue->memz_name);
 369         } else {
 370                 QAT_LOG(DEBUG, "queue %s doesn't exist",
 371                                 queue->memz_name);
 372         }
 373 }
 374
 375 static int
 376 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 377                 struct qat_qp_config *qp_conf, uint8_t dir)
 378 {
 379         uint64_t queue_base;
 380         void *io_addr;
 381         const struct rte_memzone *qp_mz;
 382         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 383         int ret = 0;
 384         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
 385                         qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
 386         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
 387
 388         queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
 389         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
 390                         qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
 391
 392         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 393                 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
 394                 return -EINVAL;
 395         }
 396
 397         /*
 398          * Allocate a memzone for the queue - create a unique name.
 399          */
 400         snprintf(queue->memz_name, sizeof(queue->memz_name),
 401                         "%s_%d_%s_%s_%d_%d",
 402                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 403                 qp_conf->service_str, "qp_mem",
 404                 queue->hw_bundle_number, queue->hw_queue_number);
 405         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 406                         qat_dev->pci_dev->device.numa_node);
 407         if (qp_mz == NULL) {
 408                 QAT_LOG(ERR, "Failed to allocate ring memzone");
 409                 return -ENOMEM;
 410         }
 411
 412         queue->base_addr = (char *)qp_mz->addr;
 413         queue->base_phys_addr = qp_mz->iova;
 414         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 415                         queue_size_bytes)) {
 416                 QAT_LOG(ERR, "Invalid alignment on queue create "
 417                                         " 0x%"PRIx64"\n",
 418                                         queue->base_phys_addr);
 419                 ret = -EFAULT;
 420                 goto queue_create_err;
 421         }
 422
 423         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
 424                         &(queue->queue_size)) != 0) {
 425                 QAT_LOG(ERR, "Invalid num inflights");
 426                 ret = -EINVAL;
 427                 goto queue_create_err;
 428         }
 429
 430         queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
 431         queue->head = 0;
 432         queue->tail = 0;
 433         queue->msg_size = desc_size;
 434
 435         /* For fast calculation of cookie index, relies on msg_size being 2^n */
 436         queue->trailz = __builtin_ctz(desc_size);
 437
 438         /*
 439          * Write an unused pattern to the queue memory.
 440          */
 441         memset(queue->base_addr, 0x7F, queue_size_bytes);
 442
 443         queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 444                                         queue->queue_size);
 445
 446         io_addr = pci_dev->mem_resource[0].addr;
 447
 448         WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 449                         queue->hw_queue_number, queue_base);
 450
 451         QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
 452                 " nb msgs %u, msg_size %u, modulo mask %u",
 453                         queue->memz_name,
 454                         queue->queue_size, queue_size_bytes,
 455                         qp_conf->nb_descriptors, desc_size,
 456                         queue->modulo_mask);
 457
 458         return 0;
 459
 460 queue_create_err:
 461         rte_memzone_free(qp_mz);
 462         return ret;
 463 }
 464
 465 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 466                                         uint32_t queue_size_bytes)
 467 {
 468         if (((queue_size_bytes - 1) & phys_addr) != 0)
 469                 return -EINVAL;
 470         return 0;
 471 }
 472
 473 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 474         uint32_t *p_queue_size_for_csr)
 475 {
 476         uint8_t i = ADF_MIN_RING_SIZE;
 477
 478         for (; i <= ADF_MAX_RING_SIZE; i++)
 479                 if ((msg_size * msg_num) ==
 480                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 481                         *p_queue_size_for_csr = i;
 482                         return 0;
 483                 }
 484         QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 485         return -EINVAL;
 486 }
 487
 488 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 489                                         rte_spinlock_t *lock)
 490 {
 491         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 492                                         (ADF_ARB_REG_SLOT *
 493                                                         txq->hw_bundle_number);
 494         uint32_t value;
 495
 496         rte_spinlock_lock(lock);
 497         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 498         value |= (0x01 << txq->hw_queue_number);
 499         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 500         rte_spinlock_unlock(lock);
 501 }
 502
 503 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 504                                         rte_spinlock_t *lock)
 505 {
 506         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 507                                         (ADF_ARB_REG_SLOT *
 508                                                         txq->hw_bundle_number);
 509         uint32_t value;
 510
 511         rte_spinlock_lock(lock);
 512         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 513         value &= ~(0x01 << txq->hw_queue_number);
 514         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 515         rte_spinlock_unlock(lock);
 516 }
 517
 518 static void adf_configure_queues(struct qat_qp *qp)
 519 {
 520         uint32_t queue_config;
 521         struct qat_queue *queue = &qp->tx_q;
 522
 523         queue_config = BUILD_RING_CONFIG(queue->queue_size);
 524
 525         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 526                         queue->hw_queue_number, queue_config);
 527
 528         queue = &qp->rx_q;
 529         queue_config =
 530                         BUILD_RESP_RING_CONFIG(queue->queue_size,
 531                                         ADF_RING_NEAR_WATERMARK_512,
 532                                         ADF_RING_NEAR_WATERMARK_0);
 533
 534         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 535                         queue->hw_queue_number, queue_config);
 536 }
 537
 538 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
 539 {
 540         return data & modulo_mask;
 541 }
 542
 543 static inline void
 544 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
 545         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 546                         q->hw_queue_number, q->tail);
 547         q->csr_tail = q->tail;
 548 }
 549
 550 static inline
 551 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
 552 {
 553         uint32_t old_head, new_head;
 554         uint32_t max_head;
 555
 556         old_head = q->csr_head;
 557         new_head = q->head;
 558         max_head = qp->nb_descriptors * q->msg_size;
 559
 560         /* write out free descriptors */
 561         void *cur_desc = (uint8_t *)q->base_addr + old_head;
 562
 563         if (new_head < old_head) {
 564                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
 565                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
 566         } else {
 567                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
 568         }
 569         q->nb_processed_responses = 0;
 570         q->csr_head = new_head;
 571
 572         /* write current head to CSR */
 573         WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
 574                             q->hw_queue_number, new_head);
 575 }
 576
 577 uint16_t
 578 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 579 {
 580         register struct qat_queue *queue;
 581         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 582         register uint32_t nb_ops_sent = 0;
 583         register int ret;
 584         uint16_t nb_ops_possible = nb_ops;
 585         register uint8_t *base_addr;
 586         register uint32_t tail;
 587
 588         if (unlikely(nb_ops == 0))
 589                 return 0;
 590
 591         /* read params used a lot in main loop into registers */
 592         queue = &(tmp_qp->tx_q);
 593         base_addr = (uint8_t *)queue->base_addr;
 594         tail = queue->tail;
 595
 596         /* Find how many can actually fit on the ring */
 597         {
 598                 /* dequeued can only be written by one thread, but it may not
 599                  * be this thread. As it's 4-byte aligned it will be read
 600                  * atomically here by any Intel CPU.
 601                  * enqueued can wrap before dequeued, but cannot
 602                  * lap it as var size of enq/deq (uint32_t) > var size of
 603                  * max_inflights (uint16_t). In reality inflights is never
 604                  * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
 605                  * On wrapping, the calculation still returns the correct
 606                  * positive value as all three vars are unsigned.
 607                  */
 608                 uint32_t inflights =
 609                         tmp_qp->enqueued - tmp_qp->dequeued;
 610
 611                 if ((inflights + nb_ops) > tmp_qp->max_inflights) {
 612                         nb_ops_possible = tmp_qp->max_inflights - inflights;
 613                         if (nb_ops_possible == 0)
 614                                 return 0;
 615                 }
 616                 /* QAT has plenty of work queued already, so don't waste cycles
 617                  * enqueueing, wait til the application has gathered a bigger
 618                  * burst or some completed ops have been dequeued
 619                  */
 620                 if (tmp_qp->min_enq_burst_threshold && inflights >
 621                                 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
 622                                 tmp_qp->min_enq_burst_threshold) {
 623                         tmp_qp->stats.threshold_hit_count++;
 624                         return 0;
 625                 }
 626         }
 627
 628
 629         while (nb_ops_sent != nb_ops_possible) {
 630                 ret = tmp_qp->build_request(*ops, base_addr + tail,
 631                                 tmp_qp->op_cookies[tail >> queue->trailz],
 632                                 tmp_qp->qat_dev_gen);
 633                 if (ret != 0) {
 634                         tmp_qp->stats.enqueue_err_count++;
 635                         /* This message cannot be enqueued */
 636                         if (nb_ops_sent == 0)
 637                                 return 0;
 638                         goto kick_tail;
 639                 }
 640
 641                 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
 642                 ops++;
 643                 nb_ops_sent++;
 644         }
 645 kick_tail:
 646         queue->tail = tail;
 647         tmp_qp->enqueued += nb_ops_sent;
 648         tmp_qp->stats.enqueued_count += nb_ops_sent;
 649         txq_write_tail(tmp_qp, queue);
 650         return nb_ops_sent;
 651 }
 652
 653 /* Use this for compression only - but keep consistent with above common
 654  * function as much as possible.
 655  */
 656 uint16_t
 657 qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 658 {
 659         register struct qat_queue *queue;
 660         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 661         register uint32_t nb_ops_sent = 0;
 662         register int nb_desc_to_build;
 663         uint16_t nb_ops_possible = nb_ops;
 664         register uint8_t *base_addr;
 665         register uint32_t tail;
 666
 667         int descriptors_built, total_descriptors_built = 0;
 668         int nb_remaining_descriptors;
 669         int overflow = 0;
 670
 671         if (unlikely(nb_ops == 0))
 672                 return 0;
 673
 674         /* read params used a lot in main loop into registers */
 675         queue = &(tmp_qp->tx_q);
 676         base_addr = (uint8_t *)queue->base_addr;
 677         tail = queue->tail;
 678
 679         /* Find how many can actually fit on the ring */
 680         {
 681                 /* dequeued can only be written by one thread, but it may not
 682                  * be this thread. As it's 4-byte aligned it will be read
 683                  * atomically here by any Intel CPU.
 684                  * enqueued can wrap before dequeued, but cannot
 685                  * lap it as var size of enq/deq (uint32_t) > var size of
 686                  * max_inflights (uint16_t). In reality inflights is never
 687                  * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
 688                  * On wrapping, the calculation still returns the correct
 689                  * positive value as all three vars are unsigned.
 690                  */
 691                 uint32_t inflights =
 692                         tmp_qp->enqueued - tmp_qp->dequeued;
 693
 694                 /* Find how many can actually fit on the ring */
 695                 overflow = (inflights + nb_ops) - tmp_qp->max_inflights;
 696                 if (overflow > 0) {
 697                         nb_ops_possible = nb_ops - overflow;
 698                         if (nb_ops_possible == 0)
 699                                 return 0;
 700                 }
 701
 702                 /* QAT has plenty of work queued already, so don't waste cycles
 703                  * enqueueing, wait til the application has gathered a bigger
 704                  * burst or some completed ops have been dequeued
 705                  */
 706                 if (tmp_qp->min_enq_burst_threshold && inflights >
 707                                 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
 708                                 tmp_qp->min_enq_burst_threshold) {
 709                         tmp_qp->stats.threshold_hit_count++;
 710                         return 0;
 711                 }
 712         }
 713
 714         /* At this point nb_ops_possible is assuming a 1:1 mapping
 715          * between ops and descriptors.
 716          * Fewer may be sent if some ops have to be split.
 717          * nb_ops_possible is <= burst size.
 718          * Find out how many spaces are actually available on the qp in case
 719          * more are needed.
 720          */
 721         nb_remaining_descriptors = nb_ops_possible
 722                          + ((overflow >= 0) ? 0 : overflow * (-1));
 723         QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d",
 724                         nb_ops, nb_remaining_descriptors);
 725
 726         while (nb_ops_sent != nb_ops_possible &&
 727                                 nb_remaining_descriptors > 0) {
 728                 struct qat_comp_op_cookie *cookie =
 729                                 tmp_qp->op_cookies[tail >> queue->trailz];
 730
 731                 descriptors_built = 0;
 732
 733                 QAT_DP_LOG(DEBUG, "--- data length: %u",
 734                            ((struct rte_comp_op *)*ops)->src.length);
 735
 736                 nb_desc_to_build = qat_comp_build_request(*ops,
 737                                 base_addr + tail, cookie, tmp_qp->qat_dev_gen);
 738                 QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, "
 739                         "%d ops sent, %d descriptors needed",
 740                         total_descriptors_built, nb_remaining_descriptors,
 741                         nb_ops_sent, nb_desc_to_build);
 742
 743                 if (unlikely(nb_desc_to_build < 0)) {
 744                         /* this message cannot be enqueued */
 745                         tmp_qp->stats.enqueue_err_count++;
 746                         if (nb_ops_sent == 0)
 747                                 return 0;
 748                         goto kick_tail;
 749                 } else if (unlikely(nb_desc_to_build > 1)) {
 750                         /* this op is too big and must be split - get more
 751                          * descriptors and retry
 752                          */
 753
 754                         QAT_DP_LOG(DEBUG, "Build %d descriptors for this op",
 755                                         nb_desc_to_build);
 756
 757                         nb_remaining_descriptors -= nb_desc_to_build;
 758                         if (nb_remaining_descriptors >= 0) {
 759                                 /* There are enough remaining descriptors
 760                                  * so retry
 761                                  */
 762                                 int ret2 = qat_comp_build_multiple_requests(
 763                                                 *ops, tmp_qp, tail,
 764                                                 nb_desc_to_build);
 765
 766                                 if (unlikely(ret2 < 1)) {
 767                                         QAT_DP_LOG(DEBUG,
 768                                                         "Failed to build (%d) descriptors, status %d",
 769                                                         nb_desc_to_build, ret2);
 770
 771                                         qat_comp_free_split_op_memzones(cookie,
 772                                                         nb_desc_to_build - 1);
 773
 774                                         tmp_qp->stats.enqueue_err_count++;
 775
 776                                         /* This message cannot be enqueued */
 777                                         if (nb_ops_sent == 0)
 778                                                 return 0;
 779                                         goto kick_tail;
 780                                 } else {
 781                                         descriptors_built = ret2;
 782                                         total_descriptors_built +=
 783                                                         descriptors_built;
 784                                         nb_remaining_descriptors -=
 785                                                         descriptors_built;
 786                                         QAT_DP_LOG(DEBUG,
 787                                                         "Multiple descriptors (%d) built ok",
 788                                                         descriptors_built);
 789                                 }
 790                         } else {
 791                                 QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) "
 792                                                 "exceeds number of available descriptors (%d)",
 793                                                 nb_desc_to_build,
 794                                                 nb_remaining_descriptors +
 795                                                         nb_desc_to_build);
 796
 797                                 qat_comp_free_split_op_memzones(cookie,
 798                                                 nb_desc_to_build - 1);
 799
 800                                 /* Not enough extra descriptors */
 801                                 if (nb_ops_sent == 0)
 802                                         return 0;
 803                                 goto kick_tail;
 804                         }
 805                 } else {
 806                         descriptors_built = 1;
 807                         total_descriptors_built++;
 808                         nb_remaining_descriptors--;
 809                         QAT_DP_LOG(DEBUG, "Single descriptor built ok");
 810                 }
 811
 812                 tail = adf_modulo(tail + (queue->msg_size * descriptors_built),
 813                                   queue->modulo_mask);
 814                 ops++;
 815                 nb_ops_sent++;
 816         }
 817
 818 kick_tail:
 819         queue->tail = tail;
 820         tmp_qp->enqueued += total_descriptors_built;
 821         tmp_qp->stats.enqueued_count += nb_ops_sent;
 822         txq_write_tail(tmp_qp, queue);
 823         return nb_ops_sent;
 824 }
 825
 826 uint16_t
 827 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 828 {
 829         struct qat_queue *rx_queue;
 830         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 831         uint32_t head;
 832         uint32_t op_resp_counter = 0, fw_resp_counter = 0;
 833         uint8_t *resp_msg;
 834         int nb_fw_responses;
 835
 836         rx_queue = &(tmp_qp->rx_q);
 837         head = rx_queue->head;
 838         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
 839
 840         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 841                         op_resp_counter != nb_ops) {
 842
 843                 nb_fw_responses = 1;
 844
 845                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 846                         qat_sym_process_response(ops, resp_msg);
 847                 else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
 848                         nb_fw_responses = qat_comp_process_response(
 849                                 ops, resp_msg,
 850                                 tmp_qp->op_cookies[head >> rx_queue->trailz],
 851                                 &tmp_qp->stats.dequeue_err_count);
 852 #ifdef BUILD_QAT_ASYM
 853                 else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
 854                         qat_asym_process_response(ops, resp_msg,
 855                                 tmp_qp->op_cookies[head >> rx_queue->trailz]);
 856 #endif
 857
 858                 head = adf_modulo(head + rx_queue->msg_size,
 859                                   rx_queue->modulo_mask);
 860
 861                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
 862
 863                 if (nb_fw_responses) {
 864                         /* only move on to next op if one was ready to return
 865                          * to API
 866                          */
 867                         ops++;
 868                         op_resp_counter++;
 869                 }
 870
 871                  /* A compression op may be broken up into multiple fw requests.
 872                   * Only count fw responses as complete once ALL the responses
 873                   * associated with an op have been processed, as the cookie
 874                   * data from the first response must be available until
 875                   * finished with all firmware responses.
 876                   */
 877                 fw_resp_counter += nb_fw_responses;
 878
 879                 rx_queue->nb_processed_responses++;
 880         }
 881
 882         tmp_qp->dequeued += fw_resp_counter;
 883         tmp_qp->stats.dequeued_count += op_resp_counter;
 884
 885         rx_queue->head = head;
 886         if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
 887                 rxq_free_desc(tmp_qp, rx_queue);
 888
 889         QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u",
 890                         op_resp_counter, fw_resp_counter);
 891
 892         return op_resp_counter;
 893 }
 894
 895 /* This is almost same as dequeue_op_burst, without the atomic, without stats
 896  * and without the op. Dequeues one response.
 897  */
 898 static uint8_t
 899 qat_cq_dequeue_response(struct qat_qp *qp, void *out_data)
 900 {
 901         uint8_t result = 0;
 902         uint8_t retries = 0;
 903         struct qat_queue *queue = &(qp->rx_q);
 904         struct icp_qat_fw_comn_resp *resp_msg = (struct icp_qat_fw_comn_resp *)
 905                         ((uint8_t *)queue->base_addr + queue->head);
 906
 907         while (retries++ < QAT_CQ_MAX_DEQ_RETRIES &&
 908                         *(uint32_t *)resp_msg == ADF_RING_EMPTY_SIG) {
 909                 /* loop waiting for response until we reach the timeout */
 910                 rte_delay_ms(20);
 911         }
 912
 913         if (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG) {
 914                 /* response received */
 915                 result = 1;
 916
 917                 /* check status flag */
 918                 if (ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
 919                                 resp_msg->comn_hdr.comn_status) ==
 920                                 ICP_QAT_FW_COMN_STATUS_FLAG_OK) {
 921                         /* success */
 922                         memcpy(out_data, resp_msg, queue->msg_size);
 923                 } else {
 924                         memset(out_data, 0, queue->msg_size);
 925                 }
 926
 927                 queue->head = adf_modulo(queue->head + queue->msg_size,
 928                                 queue->modulo_mask);
 929                 rxq_free_desc(qp, queue);
 930         }
 931
 932         return result;
 933 }
 934
 935 /* Sends a NULL message and extracts QAT fw version from the response.
 936  * Used to determine detailed capabilities based on the fw version number.
 937  * This assumes that there are no inflight messages, i.e. assumes there's space
 938  * on the qp, one message is sent and only one response collected.
 939  * Returns fw version number or 0 for unknown version or a negative error code.
 940  */
 941 int
 942 qat_cq_get_fw_version(struct qat_qp *qp)
 943 {
 944         struct qat_queue *queue = &(qp->tx_q);
 945         uint8_t *base_addr = (uint8_t *)queue->base_addr;
 946         struct icp_qat_fw_comn_req null_msg;
 947         struct icp_qat_fw_comn_resp response;
 948
 949         /* prepare the NULL request */
 950         memset(&null_msg, 0, sizeof(null_msg));
 951         null_msg.comn_hdr.hdr_flags =
 952                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
 953         null_msg.comn_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
 954         null_msg.comn_hdr.service_cmd_id = ICP_QAT_FW_NULL_REQ_SERV_ID;
 955
 956 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 957         QAT_DP_HEXDUMP_LOG(DEBUG, "NULL request", &null_msg, sizeof(null_msg));
 958 #endif
 959
 960         /* send the NULL request */
 961         memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg));
 962         queue->tail = adf_modulo(queue->tail + queue->msg_size,
 963                         queue->modulo_mask);
 964         txq_write_tail(qp, queue);
 965
 966         /* receive a response */
 967         if (qat_cq_dequeue_response(qp, &response)) {
 968
 969 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 970                 QAT_DP_HEXDUMP_LOG(DEBUG, "NULL response:", &response,
 971                                 sizeof(response));
 972 #endif
 973                 /* if LW0 bit 24 is set - then the fw version was returned */
 974                 if (QAT_FIELD_GET(response.comn_hdr.hdr_flags,
 975                                 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_BITPOS,
 976                                 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_MASK))
 977                         return response.resrvd[0]; /* return LW4 */
 978                 else
 979                         return 0; /* not set - we don't know fw version */
 980         }
 981
 982         QAT_LOG(ERR, "No response received");
 983         return -EINVAL;
 984 }
 985
 986 __rte_weak int
 987 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
 988                           void *op_cookie __rte_unused,
 989                           uint64_t *dequeue_err_count __rte_unused)
 990 {
 991         return  0;
 992 }