drivers/common/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_dev.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_pci.h>
  10 #include <rte_bus_pci.h>
  11 #include <rte_atomic.h>
  12 #include <rte_prefetch.h>
  13
  14 #include "qat_logs.h"
  15 #include "qat_device.h"
  16 #include "qat_qp.h"
  17 #include "qat_sym.h"
  18 #include "qat_asym.h"
  19 #include "qat_comp.h"
  20 #include "adf_transport_access_macros.h"
  21
  22
  23 #define ADF_MAX_DESC                            4096
  24 #define ADF_MIN_DESC                            128
  25
  26 #define ADF_ARB_REG_SLOT                        0x1000
  27 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  28
  29 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  30         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  31         (ADF_ARB_REG_SLOT * index), value)
  32
  33 __extension__
  34 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
  35                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  36         /* queue pairs which provide an asymmetric crypto service */
  37         [QAT_SERVICE_ASYMMETRIC] = {
  38                 {
  39                         .service_type = QAT_SERVICE_ASYMMETRIC,
  40                         .hw_bundle_num = 0,
  41                         .tx_ring_num = 0,
  42                         .rx_ring_num = 8,
  43                         .tx_msg_size = 64,
  44                         .rx_msg_size = 32,
  45
  46                 }, {
  47                         .service_type = QAT_SERVICE_ASYMMETRIC,
  48                         .hw_bundle_num = 0,
  49                         .tx_ring_num = 1,
  50                         .rx_ring_num = 9,
  51                         .tx_msg_size = 64,
  52                         .rx_msg_size = 32,
  53                 }
  54         },
  55         /* queue pairs which provide a symmetric crypto service */
  56         [QAT_SERVICE_SYMMETRIC] = {
  57                 {
  58                         .service_type = QAT_SERVICE_SYMMETRIC,
  59                         .hw_bundle_num = 0,
  60                         .tx_ring_num = 2,
  61                         .rx_ring_num = 10,
  62                         .tx_msg_size = 128,
  63                         .rx_msg_size = 32,
  64                 },
  65                 {
  66                         .service_type = QAT_SERVICE_SYMMETRIC,
  67                         .hw_bundle_num = 0,
  68                         .tx_ring_num = 3,
  69                         .rx_ring_num = 11,
  70                         .tx_msg_size = 128,
  71                         .rx_msg_size = 32,
  72                 }
  73         },
  74         /* queue pairs which provide a compression service */
  75         [QAT_SERVICE_COMPRESSION] = {
  76                 {
  77                         .service_type = QAT_SERVICE_COMPRESSION,
  78                         .hw_bundle_num = 0,
  79                         .tx_ring_num = 6,
  80                         .rx_ring_num = 14,
  81                         .tx_msg_size = 128,
  82                         .rx_msg_size = 32,
  83                 }, {
  84                         .service_type = QAT_SERVICE_COMPRESSION,
  85                         .hw_bundle_num = 0,
  86                         .tx_ring_num = 7,
  87                         .rx_ring_num = 15,
  88                         .tx_msg_size = 128,
  89                         .rx_msg_size = 32,
  90                 }
  91         }
  92 };
  93
  94 __extension__
  95 const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
  96                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  97         /* queue pairs which provide an asymmetric crypto service */
  98         [QAT_SERVICE_ASYMMETRIC] = {
  99                 {
 100                         .service_type = QAT_SERVICE_ASYMMETRIC,
 101                         .hw_bundle_num = 0,
 102                         .tx_ring_num = 0,
 103                         .rx_ring_num = 4,
 104                         .tx_msg_size = 64,
 105                         .rx_msg_size = 32,
 106                 }
 107         },
 108         /* queue pairs which provide a symmetric crypto service */
 109         [QAT_SERVICE_SYMMETRIC] = {
 110                 {
 111                         .service_type = QAT_SERVICE_SYMMETRIC,
 112                         .hw_bundle_num = 0,
 113                         .tx_ring_num = 1,
 114                         .rx_ring_num = 5,
 115                         .tx_msg_size = 128,
 116                         .rx_msg_size = 32,
 117                 }
 118         },
 119         /* queue pairs which provide a compression service */
 120         [QAT_SERVICE_COMPRESSION] = {
 121                 {
 122                         .service_type = QAT_SERVICE_COMPRESSION,
 123                         .hw_bundle_num = 0,
 124                         .tx_ring_num = 3,
 125                         .rx_ring_num = 7,
 126                         .tx_msg_size = 128,
 127                         .rx_msg_size = 32,
 128                 }
 129         }
 130 };
 131
 132 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 133         uint32_t queue_size_bytes);
 134 static void qat_queue_delete(struct qat_queue *queue);
 135 static int qat_queue_create(struct qat_pci_device *qat_dev,
 136         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
 137 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 138         uint32_t *queue_size_for_csr);
 139 static void adf_configure_queues(struct qat_qp *queue);
 140 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 141         rte_spinlock_t *lock);
 142 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 143         rte_spinlock_t *lock);
 144
 145
 146 int qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data,
 147                 enum qat_service_type service)
 148 {
 149         int i, count;
 150
 151         for (i = 0, count = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++)
 152                 if (qp_hw_data[i].service_type == service)
 153                         count++;
 154         return count;
 155 }
 156
 157 static const struct rte_memzone *
 158 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 159                         int socket_id)
 160 {
 161         const struct rte_memzone *mz;
 162
 163         mz = rte_memzone_lookup(queue_name);
 164         if (mz != 0) {
 165                 if (((size_t)queue_size <= mz->len) &&
 166                                 ((socket_id == SOCKET_ID_ANY) ||
 167                                         (socket_id == mz->socket_id))) {
 168                         QAT_LOG(DEBUG, "re-use memzone already "
 169                                         "allocated for %s", queue_name);
 170                         return mz;
 171                 }
 172
 173                 QAT_LOG(ERR, "Incompatible memzone already "
 174                                 "allocated %s, size %u, socket %d. "
 175                                 "Requested size %u, socket %u",
 176                                 queue_name, (uint32_t)mz->len,
 177                                 mz->socket_id, queue_size, socket_id);
 178                 return NULL;
 179         }
 180
 181         QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
 182                                         queue_name, queue_size, socket_id);
 183         return rte_memzone_reserve_aligned(queue_name, queue_size,
 184                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 185 }
 186
 187 int qat_qp_setup(struct qat_pci_device *qat_dev,
 188                 struct qat_qp **qp_addr,
 189                 uint16_t queue_pair_id,
 190                 struct qat_qp_config *qat_qp_conf)
 191
 192 {
 193         struct qat_qp *qp;
 194         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 195         char op_cookie_pool_name[RTE_RING_NAMESIZE];
 196         uint32_t i;
 197
 198         QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
 199                 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
 200
 201         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
 202                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
 203                 QAT_LOG(ERR, "Can't create qp for %u descriptors",
 204                                 qat_qp_conf->nb_descriptors);
 205                 return -EINVAL;
 206         }
 207
 208         if (pci_dev->mem_resource[0].addr == NULL) {
 209                 QAT_LOG(ERR, "Could not find VF config space "
 210                                 "(UIO driver attached?).");
 211                 return -EINVAL;
 212         }
 213
 214         /* Allocate the queue pair data structure. */
 215         qp = rte_zmalloc_socket("qat PMD qp metadata",
 216                                 sizeof(*qp), RTE_CACHE_LINE_SIZE,
 217                                 qat_qp_conf->socket_id);
 218         if (qp == NULL) {
 219                 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
 220                 return -ENOMEM;
 221         }
 222         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
 223         qp->op_cookies = rte_zmalloc_socket("qat PMD op cookie pointer",
 224                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 225                         RTE_CACHE_LINE_SIZE, qat_qp_conf->socket_id);
 226         if (qp->op_cookies == NULL) {
 227                 QAT_LOG(ERR, "Failed to alloc mem for cookie");
 228                 rte_free(qp);
 229                 return -ENOMEM;
 230         }
 231
 232         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 233         qp->enqueued = qp->dequeued = 0;
 234
 235         if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
 236                                         ADF_RING_DIR_TX) != 0) {
 237                 QAT_LOG(ERR, "Tx queue create failed "
 238                                 "queue_pair_id=%u", queue_pair_id);
 239                 goto create_err;
 240         }
 241
 242         qp->max_inflights = ADF_MAX_INFLIGHTS(qp->tx_q.queue_size,
 243                                 ADF_BYTES_TO_MSG_SIZE(qp->tx_q.msg_size));
 244
 245         if (qp->max_inflights < 2) {
 246                 QAT_LOG(ERR, "Invalid num inflights");
 247                 qat_queue_delete(&(qp->tx_q));
 248                 goto create_err;
 249         }
 250
 251         if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
 252                                         ADF_RING_DIR_RX) != 0) {
 253                 QAT_LOG(ERR, "Rx queue create failed "
 254                                 "queue_pair_id=%hu", queue_pair_id);
 255                 qat_queue_delete(&(qp->tx_q));
 256                 goto create_err;
 257         }
 258
 259         adf_configure_queues(qp);
 260         adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
 261                                         &qat_dev->arb_csr_lock);
 262
 263         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
 264                                         "%s%d_cookies_%s_qp%hu",
 265                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 266                 qat_qp_conf->service_str, queue_pair_id);
 267
 268         QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
 269         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 270         if (qp->op_cookie_pool == NULL)
 271                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 272                                 qp->nb_descriptors,
 273                                 qat_qp_conf->cookie_size, 64, 0,
 274                                 NULL, NULL, NULL, NULL,
 275                                 qat_dev->pci_dev->device.numa_node,
 276                                 0);
 277         if (!qp->op_cookie_pool) {
 278                 QAT_LOG(ERR, "QAT PMD Cannot create"
 279                                 " op mempool");
 280                 goto create_err;
 281         }
 282
 283         for (i = 0; i < qp->nb_descriptors; i++) {
 284                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 285                         QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
 286                         goto create_err;
 287                 }
 288                 memset(qp->op_cookies[i], 0, qat_qp_conf->cookie_size);
 289         }
 290
 291         qp->qat_dev_gen = qat_dev->qat_dev_gen;
 292         qp->build_request = qat_qp_conf->build_request;
 293         qp->service_type = qat_qp_conf->hw->service_type;
 294         qp->qat_dev = qat_dev;
 295
 296         QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
 297                         queue_pair_id, op_cookie_pool_name);
 298
 299         *qp_addr = qp;
 300         return 0;
 301
 302 create_err:
 303         if (qp->op_cookie_pool)
 304                 rte_mempool_free(qp->op_cookie_pool);
 305         rte_free(qp->op_cookies);
 306         rte_free(qp);
 307         return -EFAULT;
 308 }
 309
 310 int qat_qp_release(struct qat_qp **qp_addr)
 311 {
 312         struct qat_qp *qp = *qp_addr;
 313         uint32_t i;
 314
 315         if (qp == NULL) {
 316                 QAT_LOG(DEBUG, "qp already freed");
 317                 return 0;
 318         }
 319
 320         QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
 321                                 qp->qat_dev->qat_dev_id);
 322
 323         /* Don't free memory if there are still responses to be processed */
 324         if ((qp->enqueued - qp->dequeued) == 0) {
 325                 qat_queue_delete(&(qp->tx_q));
 326                 qat_queue_delete(&(qp->rx_q));
 327         } else {
 328                 return -EAGAIN;
 329         }
 330
 331         adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
 332                                         &qp->qat_dev->arb_csr_lock);
 333
 334         for (i = 0; i < qp->nb_descriptors; i++)
 335                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 336
 337         if (qp->op_cookie_pool)
 338                 rte_mempool_free(qp->op_cookie_pool);
 339
 340         rte_free(qp->op_cookies);
 341         rte_free(qp);
 342         *qp_addr = NULL;
 343         return 0;
 344 }
 345
 346
 347 static void qat_queue_delete(struct qat_queue *queue)
 348 {
 349         const struct rte_memzone *mz;
 350         int status = 0;
 351
 352         if (queue == NULL) {
 353                 QAT_LOG(DEBUG, "Invalid queue");
 354                 return;
 355         }
 356         QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
 357                         queue->hw_queue_number, queue->memz_name);
 358
 359         mz = rte_memzone_lookup(queue->memz_name);
 360         if (mz != NULL) {
 361                 /* Write an unused pattern to the queue memory. */
 362                 memset(queue->base_addr, 0x7F, queue->queue_size);
 363                 status = rte_memzone_free(mz);
 364                 if (status != 0)
 365                         QAT_LOG(ERR, "Error %d on freeing queue %s",
 366                                         status, queue->memz_name);
 367         } else {
 368                 QAT_LOG(DEBUG, "queue %s doesn't exist",
 369                                 queue->memz_name);
 370         }
 371 }
 372
 373 static int
 374 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 375                 struct qat_qp_config *qp_conf, uint8_t dir)
 376 {
 377         uint64_t queue_base;
 378         void *io_addr;
 379         const struct rte_memzone *qp_mz;
 380         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
 381         int ret = 0;
 382         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
 383                         qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
 384         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
 385
 386         queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
 387         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
 388                         qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
 389
 390         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 391                 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
 392                 return -EINVAL;
 393         }
 394
 395         /*
 396          * Allocate a memzone for the queue - create a unique name.
 397          */
 398         snprintf(queue->memz_name, sizeof(queue->memz_name),
 399                         "%s_%d_%s_%s_%d_%d",
 400                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 401                 qp_conf->service_str, "qp_mem",
 402                 queue->hw_bundle_number, queue->hw_queue_number);
 403         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 404                         qat_dev->pci_dev->device.numa_node);
 405         if (qp_mz == NULL) {
 406                 QAT_LOG(ERR, "Failed to allocate ring memzone");
 407                 return -ENOMEM;
 408         }
 409
 410         queue->base_addr = (char *)qp_mz->addr;
 411         queue->base_phys_addr = qp_mz->iova;
 412         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 413                         queue_size_bytes)) {
 414                 QAT_LOG(ERR, "Invalid alignment on queue create "
 415                                         " 0x%"PRIx64"\n",
 416                                         queue->base_phys_addr);
 417                 ret = -EFAULT;
 418                 goto queue_create_err;
 419         }
 420
 421         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
 422                         &(queue->queue_size)) != 0) {
 423                 QAT_LOG(ERR, "Invalid num inflights");
 424                 ret = -EINVAL;
 425                 goto queue_create_err;
 426         }
 427
 428         queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
 429         queue->head = 0;
 430         queue->tail = 0;
 431         queue->msg_size = desc_size;
 432
 433         /* For fast calculation of cookie index, relies on msg_size being 2^n */
 434         queue->trailz = __builtin_ctz(desc_size);
 435
 436         /*
 437          * Write an unused pattern to the queue memory.
 438          */
 439         memset(queue->base_addr, 0x7F, queue_size_bytes);
 440
 441         queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 442                                         queue->queue_size);
 443
 444         io_addr = pci_dev->mem_resource[0].addr;
 445
 446         WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 447                         queue->hw_queue_number, queue_base);
 448
 449         QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
 450                 " nb msgs %u, msg_size %u, modulo mask %u",
 451                         queue->memz_name,
 452                         queue->queue_size, queue_size_bytes,
 453                         qp_conf->nb_descriptors, desc_size,
 454                         queue->modulo_mask);
 455
 456         return 0;
 457
 458 queue_create_err:
 459         rte_memzone_free(qp_mz);
 460         return ret;
 461 }
 462
 463 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 464                                         uint32_t queue_size_bytes)
 465 {
 466         if (((queue_size_bytes - 1) & phys_addr) != 0)
 467                 return -EINVAL;
 468         return 0;
 469 }
 470
 471 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 472         uint32_t *p_queue_size_for_csr)
 473 {
 474         uint8_t i = ADF_MIN_RING_SIZE;
 475
 476         for (; i <= ADF_MAX_RING_SIZE; i++)
 477                 if ((msg_size * msg_num) ==
 478                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 479                         *p_queue_size_for_csr = i;
 480                         return 0;
 481                 }
 482         QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 483         return -EINVAL;
 484 }
 485
 486 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
 487                                         rte_spinlock_t *lock)
 488 {
 489         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 490                                         (ADF_ARB_REG_SLOT *
 491                                                         txq->hw_bundle_number);
 492         uint32_t value;
 493
 494         rte_spinlock_lock(lock);
 495         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 496         value |= (0x01 << txq->hw_queue_number);
 497         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 498         rte_spinlock_unlock(lock);
 499 }
 500
 501 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
 502                                         rte_spinlock_t *lock)
 503 {
 504         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 505                                         (ADF_ARB_REG_SLOT *
 506                                                         txq->hw_bundle_number);
 507         uint32_t value;
 508
 509         rte_spinlock_lock(lock);
 510         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 511         value &= ~(0x01 << txq->hw_queue_number);
 512         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 513         rte_spinlock_unlock(lock);
 514 }
 515
 516 static void adf_configure_queues(struct qat_qp *qp)
 517 {
 518         uint32_t queue_config;
 519         struct qat_queue *queue = &qp->tx_q;
 520
 521         queue_config = BUILD_RING_CONFIG(queue->queue_size);
 522
 523         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 524                         queue->hw_queue_number, queue_config);
 525
 526         queue = &qp->rx_q;
 527         queue_config =
 528                         BUILD_RESP_RING_CONFIG(queue->queue_size,
 529                                         ADF_RING_NEAR_WATERMARK_512,
 530                                         ADF_RING_NEAR_WATERMARK_0);
 531
 532         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 533                         queue->hw_queue_number, queue_config);
 534 }
 535
 536 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
 537 {
 538         return data & modulo_mask;
 539 }
 540
 541 static inline void
 542 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
 543         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 544                         q->hw_queue_number, q->tail);
 545         q->csr_tail = q->tail;
 546 }
 547
 548 static inline
 549 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
 550 {
 551         uint32_t old_head, new_head;
 552         uint32_t max_head;
 553
 554         old_head = q->csr_head;
 555         new_head = q->head;
 556         max_head = qp->nb_descriptors * q->msg_size;
 557
 558         /* write out free descriptors */
 559         void *cur_desc = (uint8_t *)q->base_addr + old_head;
 560
 561         if (new_head < old_head) {
 562                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
 563                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
 564         } else {
 565                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
 566         }
 567         q->nb_processed_responses = 0;
 568         q->csr_head = new_head;
 569
 570         /* write current head to CSR */
 571         WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
 572                             q->hw_queue_number, new_head);
 573 }
 574
 575 uint16_t
 576 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 577 {
 578         register struct qat_queue *queue;
 579         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 580         register uint32_t nb_ops_sent = 0;
 581         register int ret;
 582         uint16_t nb_ops_possible = nb_ops;
 583         register uint8_t *base_addr;
 584         register uint32_t tail;
 585
 586         if (unlikely(nb_ops == 0))
 587                 return 0;
 588
 589         /* read params used a lot in main loop into registers */
 590         queue = &(tmp_qp->tx_q);
 591         base_addr = (uint8_t *)queue->base_addr;
 592         tail = queue->tail;
 593
 594         /* Find how many can actually fit on the ring */
 595         {
 596                 /* dequeued can only be written by one thread, but it may not
 597                  * be this thread. As it's 4-byte aligned it will be read
 598                  * atomically here by any Intel CPU.
 599                  * enqueued can wrap before dequeued, but cannot
 600                  * lap it as var size of enq/deq (uint32_t) > var size of
 601                  * max_inflights (uint16_t). In reality inflights is never
 602                  * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
 603                  * On wrapping, the calculation still returns the correct
 604                  * positive value as all three vars are unsigned.
 605                  */
 606                 uint32_t inflights =
 607                         tmp_qp->enqueued - tmp_qp->dequeued;
 608
 609                 if ((inflights + nb_ops) > tmp_qp->max_inflights) {
 610                         nb_ops_possible = tmp_qp->max_inflights - inflights;
 611                         if (nb_ops_possible == 0)
 612                                 return 0;
 613                 }
 614                 /* QAT has plenty of work queued already, so don't waste cycles
 615                  * enqueueing, wait til the application has gathered a bigger
 616                  * burst or some completed ops have been dequeued
 617                  */
 618                 if (tmp_qp->min_enq_burst_threshold && inflights >
 619                                 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
 620                                 tmp_qp->min_enq_burst_threshold) {
 621                         tmp_qp->stats.threshold_hit_count++;
 622                         return 0;
 623                 }
 624         }
 625
 626
 627         while (nb_ops_sent != nb_ops_possible) {
 628                 ret = tmp_qp->build_request(*ops, base_addr + tail,
 629                                 tmp_qp->op_cookies[tail >> queue->trailz],
 630                                 tmp_qp->qat_dev_gen);
 631                 if (ret != 0) {
 632                         tmp_qp->stats.enqueue_err_count++;
 633                         /* This message cannot be enqueued */
 634                         if (nb_ops_sent == 0)
 635                                 return 0;
 636                         goto kick_tail;
 637                 }
 638
 639                 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
 640                 ops++;
 641                 nb_ops_sent++;
 642         }
 643 kick_tail:
 644         queue->tail = tail;
 645         tmp_qp->enqueued += nb_ops_sent;
 646         tmp_qp->stats.enqueued_count += nb_ops_sent;
 647         txq_write_tail(tmp_qp, queue);
 648         return nb_ops_sent;
 649 }
 650
 651 uint16_t
 652 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 653 {
 654         struct qat_queue *rx_queue;
 655         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 656         uint32_t head;
 657         uint32_t resp_counter = 0;
 658         uint8_t *resp_msg;
 659
 660         rx_queue = &(tmp_qp->rx_q);
 661         head = rx_queue->head;
 662         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
 663
 664         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 665                         resp_counter != nb_ops) {
 666
 667                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 668                         qat_sym_process_response(ops, resp_msg);
 669                 else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
 670                         qat_comp_process_response(ops, resp_msg,
 671                                 tmp_qp->op_cookies[head >> rx_queue->trailz],
 672                                 &tmp_qp->stats.dequeue_err_count);
 673                 else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
 674 #ifdef BUILD_QAT_ASYM
 675                         qat_asym_process_response(ops, resp_msg,
 676                                 tmp_qp->op_cookies[head >> rx_queue->trailz]);
 677 #endif
 678                 }
 679
 680                 head = adf_modulo(head + rx_queue->msg_size,
 681                                   rx_queue->modulo_mask);
 682
 683                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
 684                 ops++;
 685                 resp_counter++;
 686         }
 687         if (resp_counter > 0) {
 688                 rx_queue->head = head;
 689                 tmp_qp->dequeued += resp_counter;
 690                 tmp_qp->stats.dequeued_count += resp_counter;
 691                 rx_queue->nb_processed_responses += resp_counter;
 692
 693                 if (rx_queue->nb_processed_responses >
 694                                                 QAT_CSR_HEAD_WRITE_THRESH)
 695                         rxq_free_desc(tmp_qp, rx_queue);
 696         }
 697
 698         return resp_counter;
 699 }
 700
 701 __rte_weak int
 702 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
 703                           void *op_cookie __rte_unused,
 704                           uint64_t *dequeue_err_count __rte_unused)
 705 {
 706         return  0;
 707 }