drivers/common/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_dev.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_atomic.h>
  12 #include <rte_prefetch.h>
  13
  14 #include "qat_logs.h"
  15 #include "qat_device.h"
  16 #include "qat_qp.h"
  17 #include "qat_sym.h"
  18 #include "adf_transport_access_macros.h"
  19
  20
  21 #define ADF_MAX_DESC                            4096
  22 #define ADF_MIN_DESC                            128
  23
  24 #define ADF_ARB_REG_SLOT                        0x1000
  25 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  26
  27 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  28         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  29         (ADF_ARB_REG_SLOT * index), value)
  30
  31 __extension__
  32 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
  33                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  34         /* queue pairs which provide an asymmetric crypto service */
  35         [QAT_SERVICE_ASYMMETRIC] = {
  36                 {
  37                         .service_type = QAT_SERVICE_ASYMMETRIC,
  38                         .hw_bundle_num = 0,
  39                         .tx_ring_num = 0,
  40                         .rx_ring_num = 8,
  41                         .tx_msg_size = 64,
  42                         .rx_msg_size = 32,
  43
  44                 }, {
  45                         .service_type = QAT_SERVICE_ASYMMETRIC,
  46                         .hw_bundle_num = 0,
  47                         .tx_ring_num = 1,
  48                         .rx_ring_num = 9,
  49                         .tx_msg_size = 64,
  50                         .rx_msg_size = 32,
  51                 }
  52         },
  53         /* queue pairs which provide a symmetric crypto service */
  54         [QAT_SERVICE_SYMMETRIC] = {
  55                 {
  56                         .service_type = QAT_SERVICE_SYMMETRIC,
  57                         .hw_bundle_num = 0,
  58                         .tx_ring_num = 2,
  59                         .rx_ring_num = 10,
  60                         .tx_msg_size = 128,
  61                         .rx_msg_size = 32,
  62                 },
  63                 {
  64                         .service_type = QAT_SERVICE_SYMMETRIC,
  65                         .hw_bundle_num = 0,
  66                         .tx_ring_num = 3,
  67                         .rx_ring_num = 11,
  68                         .tx_msg_size = 128,
  69                         .rx_msg_size = 32,
  70                 }
  71         },
  72         /* queue pairs which provide a compression service */
  73         [QAT_SERVICE_COMPRESSION] = {
  74                 {
  75                         .service_type = QAT_SERVICE_COMPRESSION,
  76                         .hw_bundle_num = 0,
  77                         .tx_ring_num = 6,
  78                         .rx_ring_num = 14,
  79                         .tx_msg_size = 128,
  80                         .rx_msg_size = 32,
  81                 }, {
  82                         .service_type = QAT_SERVICE_COMPRESSION,
  83                         .hw_bundle_num = 0,
  84                         .tx_ring_num = 7,
  85                         .rx_ring_num = 15,
  86                         .tx_msg_size = 128,
  87                         .rx_msg_size = 32,
  88                 }
  89         }
  90 };
  91
  92 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
  93         uint32_t queue_size_bytes);
  94 static void qat_queue_delete(struct qat_queue *queue);
  95 static int qat_queue_create(struct qat_pci_device *qat_dev,
  96         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
  97 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
  98         uint32_t *queue_size_for_csr);
  99 static void adf_configure_queues(struct qat_qp *queue);
 100 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 101         rte_spinlock_t *lock);
 102 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 103         rte_spinlock_t *lock);
 104
 105
 106 int qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data,
 107                 enum qat_service_type service)
 108 {
 109         int i, count;
 110
 111         for (i = 0, count = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++)
 112                 if (qp_hw_data[i].service_type == service)
 113                         count++;
 114         return count;
 115 }
 116
 117 static const struct rte_memzone *
 118 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 119                         int socket_id)
 120 {
 121         const struct rte_memzone *mz;
 122
 123         mz = rte_memzone_lookup(queue_name);
 124         if (mz != 0) {
 125                 if (((size_t)queue_size <= mz->len) &&
 126                                 ((socket_id == SOCKET_ID_ANY) ||
 127                                         (socket_id == mz->socket_id))) {
 128                         QAT_LOG(DEBUG, "re-use memzone already "
 129                                         "allocated for %s", queue_name);
 130                         return mz;
 131                 }
 132
 133                 QAT_LOG(ERR, "Incompatible memzone already "
 134                                 "allocated %s, size %u, socket %d. "
 135                                 "Requested size %u, socket %u",
 136                                 queue_name, (uint32_t)mz->len,
 137                                 mz->socket_id, queue_size, socket_id);
 138                 return NULL;
 139         }
 140
 141         QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
 142                                         queue_name, queue_size, socket_id);
 143         return rte_memzone_reserve_aligned(queue_name, queue_size,
 144                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 145 }
 146
 147 int qat_qp_setup(struct qat_pci_device *qat_dev,
 148                 struct qat_qp **qp_addr,
 149                 uint16_t queue_pair_id,
 150                 struct qat_qp_config *qat_qp_conf)
 151
 152 {
 153         struct qat_qp *qp;
 154         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 155         char op_cookie_pool_name[RTE_RING_NAMESIZE];
 156         uint32_t i;
 157
 158         QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
 159                 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
 160
 161         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
 162                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
 163                 QAT_LOG(ERR, "Can't create qp for %u descriptors",
 164                                 qat_qp_conf->nb_descriptors);
 165                 return -EINVAL;
 166         }
 167
 168         if (pci_dev->mem_resource[0].addr == NULL) {
 169                 QAT_LOG(ERR, "Could not find VF config space "
 170                                 "(UIO driver attached?).");
 171                 return -EINVAL;
 172         }
 173
 174         /* Allocate the queue pair data structure. */
 175         qp = rte_zmalloc("qat PMD qp metadata",
 176                         sizeof(*qp), RTE_CACHE_LINE_SIZE);
 177         if (qp == NULL) {
 178                 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
 179                 return -ENOMEM;
 180         }
 181         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
 182         qp->op_cookies = rte_zmalloc("qat PMD op cookie pointer",
 183                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 184                         RTE_CACHE_LINE_SIZE);
 185         if (qp->op_cookies == NULL) {
 186                 QAT_LOG(ERR, "Failed to alloc mem for cookie");
 187                 rte_free(qp);
 188                 return -ENOMEM;
 189         }
 190
 191         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 192         qp->inflights16 = 0;
 193
 194         if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
 195                                         ADF_RING_DIR_TX) != 0) {
 196                 QAT_LOG(ERR, "Tx queue create failed "
 197                                 "queue_pair_id=%u", queue_pair_id);
 198                 goto create_err;
 199         }
 200
 201         if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
 202                                         ADF_RING_DIR_RX) != 0) {
 203                 QAT_LOG(ERR, "Rx queue create failed "
 204                                 "queue_pair_id=%hu", queue_pair_id);
 205                 qat_queue_delete(&(qp->tx_q));
 206                 goto create_err;
 207         }
 208
 209         adf_configure_queues(qp);
 210         adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
 211                                         &qat_dev->arb_csr_lock);
 212
 213         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
 214                                         "%s%d_cookies_%s_qp%hu",
 215                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 216                 qat_qp_conf->service_str, queue_pair_id);
 217
 218         QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
 219         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 220         if (qp->op_cookie_pool == NULL)
 221                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 222                                 qp->nb_descriptors,
 223                                 qat_qp_conf->cookie_size, 64, 0,
 224                                 NULL, NULL, NULL, NULL, qat_qp_conf->socket_id,
 225                                 0);
 226         if (!qp->op_cookie_pool) {
 227                 QAT_LOG(ERR, "QAT PMD Cannot create"
 228                                 " op mempool");
 229                 goto create_err;
 230         }
 231
 232         for (i = 0; i < qp->nb_descriptors; i++) {
 233                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 234                         QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
 235                         goto create_err;
 236                 }
 237         }
 238
 239         qp->qat_dev_gen = qat_dev->qat_dev_gen;
 240         qp->build_request = qat_qp_conf->build_request;
 241         qp->service_type = qat_qp_conf->hw->service_type;
 242         qp->qat_dev = qat_dev;
 243
 244         QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
 245                         queue_pair_id, op_cookie_pool_name);
 246
 247         *qp_addr = qp;
 248         return 0;
 249
 250 create_err:
 251         if (qp->op_cookie_pool)
 252                 rte_mempool_free(qp->op_cookie_pool);
 253         rte_free(qp->op_cookies);
 254         rte_free(qp);
 255         return -EFAULT;
 256 }
 257
 258 int qat_qp_release(struct qat_qp **qp_addr)
 259 {
 260         struct qat_qp *qp = *qp_addr;
 261         uint32_t i;
 262
 263         if (qp == NULL) {
 264                 QAT_LOG(DEBUG, "qp already freed");
 265                 return 0;
 266         }
 267
 268         QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
 269                                 qp->qat_dev->qat_dev_id);
 270
 271         /* Don't free memory if there are still responses to be processed */
 272         if (qp->inflights16 == 0) {
 273                 qat_queue_delete(&(qp->tx_q));
 274                 qat_queue_delete(&(qp->rx_q));
 275         } else {
 276                 return -EAGAIN;
 277         }
 278
 279         adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
 280                                         &qp->qat_dev->arb_csr_lock);
 281
 282         for (i = 0; i < qp->nb_descriptors; i++)
 283                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 284
 285         if (qp->op_cookie_pool)
 286                 rte_mempool_free(qp->op_cookie_pool);
 287
 288         rte_free(qp->op_cookies);
 289         rte_free(qp);
 290         *qp_addr = NULL;
 291         return 0;
 292 }
 293
 294
 295 static void qat_queue_delete(struct qat_queue *queue)
 296 {
 297         const struct rte_memzone *mz;
 298         int status = 0;
 299
 300         if (queue == NULL) {
 301                 QAT_LOG(DEBUG, "Invalid queue");
 302                 return;
 303         }
 304         QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
 305                         queue->hw_queue_number, queue->memz_name);
 306
 307         mz = rte_memzone_lookup(queue->memz_name);
 308         if (mz != NULL) {
 309                 /* Write an unused pattern to the queue memory. */
 310                 memset(queue->base_addr, 0x7F, queue->queue_size);
 311                 status = rte_memzone_free(mz);
 312                 if (status != 0)
 313                         QAT_LOG(ERR, "Error %d on freeing queue %s",
 314                                         status, queue->memz_name);
 315         } else {
 316                 QAT_LOG(DEBUG, "queue %s doesn't exist",
 317                                 queue->memz_name);
 318         }
 319 }
 320
 321 static int
 322 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 323                 struct qat_qp_config *qp_conf, uint8_t dir)
 324 {
 325         uint64_t queue_base;
 326         void *io_addr;
 327         const struct rte_memzone *qp_mz;
 328         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 329         int ret = 0;
 330         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
 331                         qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
 332         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
 333
 334         queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
 335         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
 336                         qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
 337
 338         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 339                 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
 340                 return -EINVAL;
 341         }
 342
 343         /*
 344          * Allocate a memzone for the queue - create a unique name.
 345          */
 346         snprintf(queue->memz_name, sizeof(queue->memz_name),
 347                         "%s_%d_%s_%s_%d_%d",
 348                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 349                 qp_conf->service_str, "qp_mem",
 350                 queue->hw_bundle_number, queue->hw_queue_number);
 351         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 352                         qp_conf->socket_id);
 353         if (qp_mz == NULL) {
 354                 QAT_LOG(ERR, "Failed to allocate ring memzone");
 355                 return -ENOMEM;
 356         }
 357
 358         queue->base_addr = (char *)qp_mz->addr;
 359         queue->base_phys_addr = qp_mz->iova;
 360         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 361                         queue_size_bytes)) {
 362                 QAT_LOG(ERR, "Invalid alignment on queue create "
 363                                         " 0x%"PRIx64"\n",
 364                                         queue->base_phys_addr);
 365                 ret = -EFAULT;
 366                 goto queue_create_err;
 367         }
 368
 369         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
 370                         &(queue->queue_size)) != 0) {
 371                 QAT_LOG(ERR, "Invalid num inflights");
 372                 ret = -EINVAL;
 373                 goto queue_create_err;
 374         }
 375
 376         queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
 377                                         ADF_BYTES_TO_MSG_SIZE(desc_size));
 378         queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
 379
 380         if (queue->max_inflights < 2) {
 381                 QAT_LOG(ERR, "Invalid num inflights");
 382                 ret = -EINVAL;
 383                 goto queue_create_err;
 384         }
 385         queue->head = 0;
 386         queue->tail = 0;
 387         queue->msg_size = desc_size;
 388
 389         /*
 390          * Write an unused pattern to the queue memory.
 391          */
 392         memset(queue->base_addr, 0x7F, queue_size_bytes);
 393
 394         queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 395                                         queue->queue_size);
 396
 397         io_addr = pci_dev->mem_resource[0].addr;
 398
 399         WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 400                         queue->hw_queue_number, queue_base);
 401
 402         QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
 403                 " nb msgs %u, msg_size %u, max_inflights %u modulo mask %u",
 404                         queue->memz_name,
 405                         queue->queue_size, queue_size_bytes,
 406                         qp_conf->nb_descriptors, desc_size,
 407                         queue->max_inflights, queue->modulo_mask);
 408
 409         return 0;
 410
 411 queue_create_err:
 412         rte_memzone_free(qp_mz);
 413         return ret;
 414 }
 415
 416 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 417                                         uint32_t queue_size_bytes)
 418 {
 419         if (((queue_size_bytes - 1) & phys_addr) != 0)
 420                 return -EINVAL;
 421         return 0;
 422 }
 423
 424 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 425         uint32_t *p_queue_size_for_csr)
 426 {
 427         uint8_t i = ADF_MIN_RING_SIZE;
 428
 429         for (; i <= ADF_MAX_RING_SIZE; i++)
 430                 if ((msg_size * msg_num) ==
 431                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 432                         *p_queue_size_for_csr = i;
 433                         return 0;
 434                 }
 435         QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 436         return -EINVAL;
 437 }
 438
 439 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 440                                         rte_spinlock_t *lock)
 441 {
 442         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 443                                         (ADF_ARB_REG_SLOT *
 444                                                         txq->hw_bundle_number);
 445         uint32_t value;
 446
 447         rte_spinlock_lock(lock);
 448         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 449         value |= (0x01 << txq->hw_queue_number);
 450         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 451         rte_spinlock_unlock(lock);
 452 }
 453
 454 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 455                                         rte_spinlock_t *lock)
 456 {
 457         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 458                                         (ADF_ARB_REG_SLOT *
 459                                                         txq->hw_bundle_number);
 460         uint32_t value;
 461
 462         rte_spinlock_lock(lock);
 463         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 464         value &= ~(0x01 << txq->hw_queue_number);
 465         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 466         rte_spinlock_unlock(lock);
 467 }
 468
 469 static void adf_configure_queues(struct qat_qp *qp)
 470 {
 471         uint32_t queue_config;
 472         struct qat_queue *queue = &qp->tx_q;
 473
 474         queue_config = BUILD_RING_CONFIG(queue->queue_size);
 475
 476         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 477                         queue->hw_queue_number, queue_config);
 478
 479         queue = &qp->rx_q;
 480         queue_config =
 481                         BUILD_RESP_RING_CONFIG(queue->queue_size,
 482                                         ADF_RING_NEAR_WATERMARK_512,
 483                                         ADF_RING_NEAR_WATERMARK_0);
 484
 485         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 486                         queue->hw_queue_number, queue_config);
 487 }
 488
 489 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
 490 {
 491         return data & modulo_mask;
 492 }
 493
 494 static inline void
 495 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
 496         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 497                         q->hw_queue_number, q->tail);
 498         q->nb_pending_requests = 0;
 499         q->csr_tail = q->tail;
 500 }
 501
 502 static inline
 503 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
 504 {
 505         uint32_t old_head, new_head;
 506         uint32_t max_head;
 507
 508         old_head = q->csr_head;
 509         new_head = q->head;
 510         max_head = qp->nb_descriptors * q->msg_size;
 511
 512         /* write out free descriptors */
 513         void *cur_desc = (uint8_t *)q->base_addr + old_head;
 514
 515         if (new_head < old_head) {
 516                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
 517                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
 518         } else {
 519                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
 520         }
 521         q->nb_processed_responses = 0;
 522         q->csr_head = new_head;
 523
 524         /* write current head to CSR */
 525         WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
 526                             q->hw_queue_number, new_head);
 527 }
 528
 529 uint16_t
 530 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 531 {
 532         register struct qat_queue *queue;
 533         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 534         register uint32_t nb_ops_sent = 0;
 535         register int ret;
 536         uint16_t nb_ops_possible = nb_ops;
 537         register uint8_t *base_addr;
 538         register uint32_t tail;
 539         int overflow;
 540
 541         if (unlikely(nb_ops == 0))
 542                 return 0;
 543
 544         /* read params used a lot in main loop into registers */
 545         queue = &(tmp_qp->tx_q);
 546         base_addr = (uint8_t *)queue->base_addr;
 547         tail = queue->tail;
 548
 549         /* Find how many can actually fit on the ring */
 550         tmp_qp->inflights16 += nb_ops;
 551         overflow = tmp_qp->inflights16 - queue->max_inflights;
 552         if (overflow > 0) {
 553                 tmp_qp->inflights16 -= overflow;
 554                 nb_ops_possible = nb_ops - overflow;
 555                 if (nb_ops_possible == 0)
 556                         return 0;
 557         }
 558
 559         while (nb_ops_sent != nb_ops_possible) {
 560                 ret = tmp_qp->build_request(*ops, base_addr + tail,
 561                                 tmp_qp->op_cookies[tail / queue->msg_size],
 562                                 tmp_qp->qat_dev_gen);
 563                 if (ret != 0) {
 564                         tmp_qp->stats.enqueue_err_count++;
 565                         /*
 566                          * This message cannot be enqueued,
 567                          * decrease number of ops that wasn't sent
 568                          */
 569                         tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
 570                         if (nb_ops_sent == 0)
 571                                 return 0;
 572                         goto kick_tail;
 573                 }
 574
 575                 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
 576                 ops++;
 577                 nb_ops_sent++;
 578         }
 579 kick_tail:
 580         queue->tail = tail;
 581         tmp_qp->stats.enqueued_count += nb_ops_sent;
 582         queue->nb_pending_requests += nb_ops_sent;
 583         if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
 584                     queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
 585                 txq_write_tail(tmp_qp, queue);
 586         }
 587         return nb_ops_sent;
 588 }
 589
 590 uint16_t
 591 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 592 {
 593         struct qat_queue *rx_queue, *tx_queue;
 594         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 595         uint32_t head;
 596         uint32_t resp_counter = 0;
 597         uint8_t *resp_msg;
 598
 599         rx_queue = &(tmp_qp->rx_q);
 600         tx_queue = &(tmp_qp->tx_q);
 601         head = rx_queue->head;
 602         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
 603
 604         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 605                         resp_counter != nb_ops) {
 606
 607                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 608                         qat_sym_process_response(ops, resp_msg);
 609                 /* add qat_asym_process_response here */
 610                 /* add qat_comp_process_response here */
 611
 612                 head = adf_modulo(head + rx_queue->msg_size,
 613                                   rx_queue->modulo_mask);
 614
 615                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
 616                 ops++;
 617                 resp_counter++;
 618         }
 619         if (resp_counter > 0) {
 620                 rx_queue->head = head;
 621                 tmp_qp->stats.dequeued_count += resp_counter;
 622                 rx_queue->nb_processed_responses += resp_counter;
 623                 tmp_qp->inflights16 -= resp_counter;
 624
 625                 if (rx_queue->nb_processed_responses >
 626                                                 QAT_CSR_HEAD_WRITE_THRESH)
 627                         rxq_free_desc(tmp_qp, rx_queue);
 628         }
 629         /* also check if tail needs to be advanced */
 630         if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
 631                 tx_queue->tail != tx_queue->csr_tail) {
 632                 txq_write_tail(tmp_qp, tx_queue);
 633         }
 634         return resp_counter;
 635 }