drivers/crypto/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_dev.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_atomic.h>
  12 #include <rte_prefetch.h>
  13
  14 #include "qat_logs.h"
  15 #include "qat_device.h"
  16 #include "qat_qp.h"
  17 #include "adf_transport_access_macros.h"
  18
  19
  20 #define ADF_MAX_DESC                            4096
  21 #define ADF_MIN_DESC                            128
  22
  23 #define ADF_ARB_REG_SLOT                        0x1000
  24 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  25
  26 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  27         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  28         (ADF_ARB_REG_SLOT * index), value)
  29
  30 __extension__
  31 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
  32                                          [ADF_MAX_QPS_PER_BUNDLE] = {
  33         /* queue pairs which provide an asymmetric crypto service */
  34         [QAT_SERVICE_ASYMMETRIC] = {
  35                 {
  36                         .service_type = QAT_SERVICE_ASYMMETRIC,
  37                         .hw_bundle_num = 0,
  38                         .tx_ring_num = 0,
  39                         .rx_ring_num = 8,
  40                         .tx_msg_size = 64,
  41                         .rx_msg_size = 32,
  42
  43                 }, {
  44                         .service_type = QAT_SERVICE_ASYMMETRIC,
  45                         .tx_ring_num = 1,
  46                         .rx_ring_num = 9,
  47                         .tx_msg_size = 64,
  48                         .rx_msg_size = 32,
  49                 }, {
  50                         .service_type = QAT_SERVICE_INVALID,
  51                 }, {
  52                         .service_type = QAT_SERVICE_INVALID,
  53                 }
  54         },
  55         /* queue pairs which provide a symmetric crypto service */
  56         [QAT_SERVICE_SYMMETRIC] = {
  57                 {
  58                         .service_type = QAT_SERVICE_SYMMETRIC,
  59                         .hw_bundle_num = 0,
  60                         .tx_ring_num = 2,
  61                         .rx_ring_num = 10,
  62                         .tx_msg_size = 128,
  63                         .rx_msg_size = 32,
  64                 },
  65                 {
  66                         .service_type = QAT_SERVICE_SYMMETRIC,
  67                         .hw_bundle_num = 0,
  68                         .tx_ring_num = 3,
  69                         .rx_ring_num = 11,
  70                         .tx_msg_size = 128,
  71                         .rx_msg_size = 32,
  72                 }, {
  73                         .service_type = QAT_SERVICE_INVALID,
  74                 }, {
  75                         .service_type = QAT_SERVICE_INVALID,
  76                 }
  77         },
  78         /* queue pairs which provide a compression service */
  79         [QAT_SERVICE_COMPRESSION] = {
  80                 {
  81                         .service_type = QAT_SERVICE_COMPRESSION,
  82                         .hw_bundle_num = 0,
  83                         .tx_ring_num = 6,
  84                         .rx_ring_num = 14,
  85                         .tx_msg_size = 128,
  86                         .rx_msg_size = 32,
  87                 }, {
  88                         .service_type = QAT_SERVICE_COMPRESSION,
  89                         .hw_bundle_num = 0,
  90                         .tx_ring_num = 7,
  91                         .rx_ring_num = 15,
  92                         .tx_msg_size = 128,
  93                         .rx_msg_size = 32,
  94                 }, {
  95                         .service_type = QAT_SERVICE_INVALID,
  96                 }, {
  97                         .service_type = QAT_SERVICE_INVALID,
  98                 }
  99         }
 100 };
 101
 102 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 103         uint32_t queue_size_bytes);
 104 static void qat_queue_delete(struct qat_queue *queue);
 105 static int qat_queue_create(struct qat_pci_device *qat_dev,
 106         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
 107 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 108         uint32_t *queue_size_for_csr);
 109 static void adf_configure_queues(struct qat_qp *queue);
 110 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 111         rte_spinlock_t *lock);
 112 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 113         rte_spinlock_t *lock);
 114
 115
 116 int qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data,
 117                 enum qat_service_type service)
 118 {
 119         int i, count;
 120
 121         for (i = 0, count = 0; i < ADF_MAX_QPS_PER_BUNDLE; i++)
 122                 if (qp_hw_data[i].service_type == service)
 123                         count++;
 124         return count * ADF_NUM_BUNDLES_PER_DEV;
 125 }
 126
 127 static const struct rte_memzone *
 128 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 129                         int socket_id)
 130 {
 131         const struct rte_memzone *mz;
 132
 133         PMD_INIT_FUNC_TRACE();
 134         mz = rte_memzone_lookup(queue_name);
 135         if (mz != 0) {
 136                 if (((size_t)queue_size <= mz->len) &&
 137                                 ((socket_id == SOCKET_ID_ANY) ||
 138                                         (socket_id == mz->socket_id))) {
 139                         PMD_DRV_LOG(DEBUG, "re-use memzone already "
 140                                         "allocated for %s", queue_name);
 141                         return mz;
 142                 }
 143
 144                 PMD_DRV_LOG(ERR, "Incompatible memzone already "
 145                                 "allocated %s, size %u, socket %d. "
 146                                 "Requested size %u, socket %u",
 147                                 queue_name, (uint32_t)mz->len,
 148                                 mz->socket_id, queue_size, socket_id);
 149                 return NULL;
 150         }
 151
 152         PMD_DRV_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
 153                                         queue_name, queue_size, socket_id);
 154         return rte_memzone_reserve_aligned(queue_name, queue_size,
 155                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 156 }
 157
 158 int qat_qp_setup(struct qat_pci_device *qat_dev,
 159                 struct qat_qp **qp_addr,
 160                 uint16_t queue_pair_id,
 161                 struct qat_qp_config *qat_qp_conf)
 162
 163 {
 164         struct qat_qp *qp;
 165         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 166         char op_cookie_pool_name[RTE_RING_NAMESIZE];
 167         uint32_t i;
 168
 169         PMD_DRV_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
 170                 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
 171
 172         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
 173                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
 174                 PMD_DRV_LOG(ERR, "Can't create qp for %u descriptors",
 175                                 qat_qp_conf->nb_descriptors);
 176                 return -EINVAL;
 177         }
 178
 179         if (pci_dev->mem_resource[0].addr == NULL) {
 180                 PMD_DRV_LOG(ERR, "Could not find VF config space "
 181                                 "(UIO driver attached?).");
 182                 return -EINVAL;
 183         }
 184
 185         /* Allocate the queue pair data structure. */
 186         qp = rte_zmalloc("qat PMD qp metadata",
 187                         sizeof(*qp), RTE_CACHE_LINE_SIZE);
 188         if (qp == NULL) {
 189                 PMD_DRV_LOG(ERR, "Failed to alloc mem for qp struct");
 190                 return -ENOMEM;
 191         }
 192         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
 193         qp->op_cookies = rte_zmalloc("qat PMD op cookie pointer",
 194                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 195                         RTE_CACHE_LINE_SIZE);
 196         if (qp->op_cookies == NULL) {
 197                 PMD_DRV_LOG(ERR, "Failed to alloc mem for cookie");
 198                 rte_free(qp);
 199                 return -ENOMEM;
 200         }
 201
 202         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 203         qp->inflights16 = 0;
 204
 205         if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
 206                                         ADF_RING_DIR_TX) != 0) {
 207                 PMD_INIT_LOG(ERR, "Tx queue create failed "
 208                                 "queue_pair_id=%u", queue_pair_id);
 209                 goto create_err;
 210         }
 211
 212         if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
 213                                         ADF_RING_DIR_RX) != 0) {
 214                 PMD_DRV_LOG(ERR, "Rx queue create failed "
 215                                 "queue_pair_id=%hu", queue_pair_id);
 216                 qat_queue_delete(&(qp->tx_q));
 217                 goto create_err;
 218         }
 219
 220         adf_configure_queues(qp);
 221         adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
 222                                         &qat_dev->arb_csr_lock);
 223
 224         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
 225                                         "%s%d_cookies_%s_qp%hu",
 226                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 227                 qat_qp_conf->service_str, queue_pair_id);
 228
 229         PMD_DRV_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
 230         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 231         if (qp->op_cookie_pool == NULL)
 232                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 233                                 qp->nb_descriptors,
 234                                 qat_qp_conf->cookie_size, 64, 0,
 235                                 NULL, NULL, NULL, NULL, qat_qp_conf->socket_id,
 236                                 0);
 237         if (!qp->op_cookie_pool) {
 238                 PMD_DRV_LOG(ERR, "QAT PMD Cannot create"
 239                                 " op mempool");
 240                 goto create_err;
 241         }
 242
 243         for (i = 0; i < qp->nb_descriptors; i++) {
 244                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 245                         PMD_DRV_LOG(ERR, "QAT PMD Cannot get op_cookie");
 246                         goto create_err;
 247                 }
 248         }
 249
 250         qp->qat_dev_gen = qat_dev->qat_dev_gen;
 251         qp->build_request = qat_qp_conf->build_request;
 252         qp->process_response = qat_qp_conf->process_response;
 253         qp->qat_dev = qat_dev;
 254
 255         PMD_DRV_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
 256                         queue_pair_id, op_cookie_pool_name);
 257
 258         *qp_addr = qp;
 259         return 0;
 260
 261 create_err:
 262         rte_free(qp);
 263         return -EFAULT;
 264 }
 265
 266 int qat_qp_release(struct qat_qp **qp_addr)
 267 {
 268         struct qat_qp *qp = *qp_addr;
 269         uint32_t i;
 270
 271         PMD_INIT_FUNC_TRACE();
 272         if (qp == NULL) {
 273                 PMD_DRV_LOG(DEBUG, "qp already freed");
 274                 return 0;
 275         }
 276
 277         PMD_DRV_LOG(DEBUG, "Free qp on qat_pci device %d",
 278                                 qp->qat_dev->qat_dev_id);
 279
 280         /* Don't free memory if there are still responses to be processed */
 281         if (qp->inflights16 == 0) {
 282                 qat_queue_delete(&(qp->tx_q));
 283                 qat_queue_delete(&(qp->rx_q));
 284         } else {
 285                 return -EAGAIN;
 286         }
 287
 288         adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
 289                                         &qp->qat_dev->arb_csr_lock);
 290
 291         for (i = 0; i < qp->nb_descriptors; i++)
 292                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 293
 294         if (qp->op_cookie_pool)
 295                 rte_mempool_free(qp->op_cookie_pool);
 296
 297         rte_free(qp->op_cookies);
 298         rte_free(qp);
 299         *qp_addr = NULL;
 300         return 0;
 301 }
 302
 303
 304 static void qat_queue_delete(struct qat_queue *queue)
 305 {
 306         const struct rte_memzone *mz;
 307         int status = 0;
 308
 309         if (queue == NULL) {
 310                 PMD_DRV_LOG(DEBUG, "Invalid queue");
 311                 return;
 312         }
 313         PMD_DRV_LOG(DEBUG, "Free ring %d, memzone: %s",
 314                         queue->hw_queue_number, queue->memz_name);
 315
 316         mz = rte_memzone_lookup(queue->memz_name);
 317         if (mz != NULL) {
 318                 /* Write an unused pattern to the queue memory. */
 319                 memset(queue->base_addr, 0x7F, queue->queue_size);
 320                 status = rte_memzone_free(mz);
 321                 if (status != 0)
 322                         PMD_DRV_LOG(ERR, "Error %d on freeing queue %s",
 323                                         status, queue->memz_name);
 324         } else {
 325                 PMD_DRV_LOG(DEBUG, "queue %s doesn't exist",
 326                                 queue->memz_name);
 327         }
 328 }
 329
 330 static int
 331 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 332                 struct qat_qp_config *qp_conf, uint8_t dir)
 333 {
 334         uint64_t queue_base;
 335         void *io_addr;
 336         const struct rte_memzone *qp_mz;
 337         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 338         int ret = 0;
 339         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
 340                         qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
 341         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
 342
 343         queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
 344         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
 345                         qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
 346
 347         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 348                 PMD_DRV_LOG(ERR, "Invalid descriptor size %d", desc_size);
 349                 return -EINVAL;
 350         }
 351
 352         /*
 353          * Allocate a memzone for the queue - create a unique name.
 354          */
 355         snprintf(queue->memz_name, sizeof(queue->memz_name),
 356                         "%s_%d_%s_%s_%d_%d",
 357                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 358                 qp_conf->service_str, "qp_mem",
 359                 queue->hw_bundle_number, queue->hw_queue_number);
 360         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 361                         qp_conf->socket_id);
 362         if (qp_mz == NULL) {
 363                 PMD_DRV_LOG(ERR, "Failed to allocate ring memzone");
 364                 return -ENOMEM;
 365         }
 366
 367         queue->base_addr = (char *)qp_mz->addr;
 368         queue->base_phys_addr = qp_mz->iova;
 369         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 370                         queue_size_bytes)) {
 371                 PMD_DRV_LOG(ERR, "Invalid alignment on queue create "
 372                                         " 0x%"PRIx64"\n",
 373                                         queue->base_phys_addr);
 374                 ret = -EFAULT;
 375                 goto queue_create_err;
 376         }
 377
 378         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
 379                         &(queue->queue_size)) != 0) {
 380                 PMD_DRV_LOG(ERR, "Invalid num inflights");
 381                 ret = -EINVAL;
 382                 goto queue_create_err;
 383         }
 384
 385         queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
 386                                         ADF_BYTES_TO_MSG_SIZE(desc_size));
 387         queue->modulo = ADF_RING_SIZE_MODULO(queue->queue_size);
 388
 389         if (queue->max_inflights < 2) {
 390                 PMD_DRV_LOG(ERR, "Invalid num inflights");
 391                 ret = -EINVAL;
 392                 goto queue_create_err;
 393         }
 394         queue->head = 0;
 395         queue->tail = 0;
 396         queue->msg_size = desc_size;
 397
 398         /*
 399          * Write an unused pattern to the queue memory.
 400          */
 401         memset(queue->base_addr, 0x7F, queue_size_bytes);
 402
 403         queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 404                                         queue->queue_size);
 405
 406         io_addr = pci_dev->mem_resource[0].addr;
 407
 408         WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 409                         queue->hw_queue_number, queue_base);
 410
 411         PMD_DRV_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
 412                         " nb msgs %u, msg_size %u, max_inflights %u modulo %u",
 413                         queue->memz_name,
 414                         queue->queue_size, queue_size_bytes,
 415                         qp_conf->nb_descriptors, desc_size,
 416                         queue->max_inflights, queue->modulo);
 417
 418         return 0;
 419
 420 queue_create_err:
 421         rte_memzone_free(qp_mz);
 422         return ret;
 423 }
 424
 425 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 426                                         uint32_t queue_size_bytes)
 427 {
 428         PMD_INIT_FUNC_TRACE();
 429         if (((queue_size_bytes - 1) & phys_addr) != 0)
 430                 return -EINVAL;
 431         return 0;
 432 }
 433
 434 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 435         uint32_t *p_queue_size_for_csr)
 436 {
 437         uint8_t i = ADF_MIN_RING_SIZE;
 438
 439         PMD_INIT_FUNC_TRACE();
 440         for (; i <= ADF_MAX_RING_SIZE; i++)
 441                 if ((msg_size * msg_num) ==
 442                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 443                         *p_queue_size_for_csr = i;
 444                         return 0;
 445                 }
 446         PMD_DRV_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 447         return -EINVAL;
 448 }
 449
 450 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 451                                         rte_spinlock_t *lock)
 452 {
 453         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 454                                         (ADF_ARB_REG_SLOT *
 455                                                         txq->hw_bundle_number);
 456         uint32_t value;
 457
 458         PMD_INIT_FUNC_TRACE();
 459
 460         rte_spinlock_lock(lock);
 461         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 462         value |= (0x01 << txq->hw_queue_number);
 463         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 464         rte_spinlock_unlock(lock);
 465 }
 466
 467 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 468                                         rte_spinlock_t *lock)
 469 {
 470         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 471                                         (ADF_ARB_REG_SLOT *
 472                                                         txq->hw_bundle_number);
 473         uint32_t value;
 474
 475         PMD_INIT_FUNC_TRACE();
 476
 477         rte_spinlock_lock(lock);
 478         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 479         value &= ~(0x01 << txq->hw_queue_number);
 480         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 481         rte_spinlock_unlock(lock);
 482 }
 483
 484 static void adf_configure_queues(struct qat_qp *qp)
 485 {
 486         uint32_t queue_config;
 487         struct qat_queue *queue = &qp->tx_q;
 488
 489         PMD_INIT_FUNC_TRACE();
 490         queue_config = BUILD_RING_CONFIG(queue->queue_size);
 491
 492         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 493                         queue->hw_queue_number, queue_config);
 494
 495         queue = &qp->rx_q;
 496         queue_config =
 497                         BUILD_RESP_RING_CONFIG(queue->queue_size,
 498                                         ADF_RING_NEAR_WATERMARK_512,
 499                                         ADF_RING_NEAR_WATERMARK_0);
 500
 501         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 502                         queue->hw_queue_number, queue_config);
 503 }
 504
 505
 506 static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
 507 {
 508         uint32_t div = data >> shift;
 509         uint32_t mult = div << shift;
 510
 511         return data - mult;
 512 }
 513
 514 static inline void
 515 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
 516         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 517                         q->hw_queue_number, q->tail);
 518         q->nb_pending_requests = 0;
 519         q->csr_tail = q->tail;
 520 }
 521
 522 static inline
 523 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
 524 {
 525         uint32_t old_head, new_head;
 526         uint32_t max_head;
 527
 528         old_head = q->csr_head;
 529         new_head = q->head;
 530         max_head = qp->nb_descriptors * q->msg_size;
 531
 532         /* write out free descriptors */
 533         void *cur_desc = (uint8_t *)q->base_addr + old_head;
 534
 535         if (new_head < old_head) {
 536                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
 537                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
 538         } else {
 539                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
 540         }
 541         q->nb_processed_responses = 0;
 542         q->csr_head = new_head;
 543
 544         /* write current head to CSR */
 545         WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
 546                             q->hw_queue_number, new_head);
 547 }
 548
 549 uint16_t
 550 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 551 {
 552         register struct qat_queue *queue;
 553         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 554         register uint32_t nb_ops_sent = 0;
 555         register int ret;
 556         uint16_t nb_ops_possible = nb_ops;
 557         register uint8_t *base_addr;
 558         register uint32_t tail;
 559         int overflow;
 560
 561         if (unlikely(nb_ops == 0))
 562                 return 0;
 563
 564         /* read params used a lot in main loop into registers */
 565         queue = &(tmp_qp->tx_q);
 566         base_addr = (uint8_t *)queue->base_addr;
 567         tail = queue->tail;
 568
 569         /* Find how many can actually fit on the ring */
 570         tmp_qp->inflights16 += nb_ops;
 571         overflow = tmp_qp->inflights16 - queue->max_inflights;
 572         if (overflow > 0) {
 573                 tmp_qp->inflights16 -= overflow;
 574                 nb_ops_possible = nb_ops - overflow;
 575                 if (nb_ops_possible == 0)
 576                         return 0;
 577         }
 578
 579         while (nb_ops_sent != nb_ops_possible) {
 580                 ret = tmp_qp->build_request(*ops, base_addr + tail,
 581                                 tmp_qp->op_cookies[tail / queue->msg_size],
 582                                 tmp_qp->qat_dev_gen);
 583                 if (ret != 0) {
 584                         tmp_qp->stats.enqueue_err_count++;
 585                         /*
 586                          * This message cannot be enqueued,
 587                          * decrease number of ops that wasn't sent
 588                          */
 589                         tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
 590                         if (nb_ops_sent == 0)
 591                                 return 0;
 592                         goto kick_tail;
 593                 }
 594
 595                 tail = adf_modulo(tail + queue->msg_size, queue->modulo);
 596                 ops++;
 597                 nb_ops_sent++;
 598         }
 599 kick_tail:
 600         queue->tail = tail;
 601         tmp_qp->stats.enqueued_count += nb_ops_sent;
 602         queue->nb_pending_requests += nb_ops_sent;
 603         if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
 604                     queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
 605                 txq_write_tail(tmp_qp, queue);
 606         }
 607         return nb_ops_sent;
 608 }
 609
 610 uint16_t
 611 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 612 {
 613         struct qat_queue *rx_queue, *tx_queue;
 614         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 615         uint32_t head;
 616         uint32_t resp_counter = 0;
 617         uint8_t *resp_msg;
 618
 619         rx_queue = &(tmp_qp->rx_q);
 620         tx_queue = &(tmp_qp->tx_q);
 621         head = rx_queue->head;
 622         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
 623
 624         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 625                         resp_counter != nb_ops) {
 626
 627                 tmp_qp->process_response(ops, resp_msg,
 628                         tmp_qp->op_cookies[head / rx_queue->msg_size],
 629                         tmp_qp->qat_dev_gen);
 630
 631                 head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo);
 632
 633                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
 634                 ops++;
 635                 resp_counter++;
 636         }
 637         if (resp_counter > 0) {
 638                 rx_queue->head = head;
 639                 tmp_qp->stats.dequeued_count += resp_counter;
 640                 rx_queue->nb_processed_responses += resp_counter;
 641                 tmp_qp->inflights16 -= resp_counter;
 642
 643                 if (rx_queue->nb_processed_responses >
 644                                                 QAT_CSR_HEAD_WRITE_THRESH)
 645                         rxq_free_desc(tmp_qp, rx_queue);
 646         }
 647         /* also check if tail needs to be advanced */
 648         if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
 649                 tx_queue->tail != tx_queue->csr_tail) {
 650                 txq_write_tail(tmp_qp, tx_queue);
 651         }
 652         return resp_counter;
 653 }