net/sfc/base: simplify EF10 family conditional code checks
[dpdk.git] / drivers / common / qat / qat_qp.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2015-2018 Intel Corporation
3  */
4
5 #include <rte_common.h>
6 #include <rte_dev.h>
7 #include <rte_malloc.h>
8 #include <rte_memzone.h>
9 #include <rte_pci.h>
10 #include <rte_bus_pci.h>
11 #include <rte_atomic.h>
12 #include <rte_prefetch.h>
13
14 #include "qat_logs.h"
15 #include "qat_device.h"
16 #include "qat_qp.h"
17 #include "qat_sym.h"
18 #include "qat_comp.h"
19 #include "adf_transport_access_macros.h"
20
21
22 #define ADF_MAX_DESC                            4096
23 #define ADF_MIN_DESC                            128
24
25 #define ADF_ARB_REG_SLOT                        0x1000
26 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
27
28 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
29         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
30         (ADF_ARB_REG_SLOT * index), value)
31
32 __extension__
33 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
34                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
35         /* queue pairs which provide an asymmetric crypto service */
36         [QAT_SERVICE_ASYMMETRIC] = {
37                 {
38                         .service_type = QAT_SERVICE_ASYMMETRIC,
39                         .hw_bundle_num = 0,
40                         .tx_ring_num = 0,
41                         .rx_ring_num = 8,
42                         .tx_msg_size = 64,
43                         .rx_msg_size = 32,
44
45                 }, {
46                         .service_type = QAT_SERVICE_ASYMMETRIC,
47                         .hw_bundle_num = 0,
48                         .tx_ring_num = 1,
49                         .rx_ring_num = 9,
50                         .tx_msg_size = 64,
51                         .rx_msg_size = 32,
52                 }
53         },
54         /* queue pairs which provide a symmetric crypto service */
55         [QAT_SERVICE_SYMMETRIC] = {
56                 {
57                         .service_type = QAT_SERVICE_SYMMETRIC,
58                         .hw_bundle_num = 0,
59                         .tx_ring_num = 2,
60                         .rx_ring_num = 10,
61                         .tx_msg_size = 128,
62                         .rx_msg_size = 32,
63                 },
64                 {
65                         .service_type = QAT_SERVICE_SYMMETRIC,
66                         .hw_bundle_num = 0,
67                         .tx_ring_num = 3,
68                         .rx_ring_num = 11,
69                         .tx_msg_size = 128,
70                         .rx_msg_size = 32,
71                 }
72         },
73         /* queue pairs which provide a compression service */
74         [QAT_SERVICE_COMPRESSION] = {
75                 {
76                         .service_type = QAT_SERVICE_COMPRESSION,
77                         .hw_bundle_num = 0,
78                         .tx_ring_num = 6,
79                         .rx_ring_num = 14,
80                         .tx_msg_size = 128,
81                         .rx_msg_size = 32,
82                 }, {
83                         .service_type = QAT_SERVICE_COMPRESSION,
84                         .hw_bundle_num = 0,
85                         .tx_ring_num = 7,
86                         .rx_ring_num = 15,
87                         .tx_msg_size = 128,
88                         .rx_msg_size = 32,
89                 }
90         }
91 };
92
93 __extension__
94 const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
95                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
96         /* queue pairs which provide an asymmetric crypto service */
97         [QAT_SERVICE_ASYMMETRIC] = {
98                 {
99                         .service_type = QAT_SERVICE_ASYMMETRIC,
100                         .hw_bundle_num = 0,
101                         .tx_ring_num = 0,
102                         .rx_ring_num = 4,
103                         .tx_msg_size = 64,
104                         .rx_msg_size = 32,
105                 }
106         },
107         /* queue pairs which provide a symmetric crypto service */
108         [QAT_SERVICE_SYMMETRIC] = {
109                 {
110                         .service_type = QAT_SERVICE_SYMMETRIC,
111                         .hw_bundle_num = 0,
112                         .tx_ring_num = 1,
113                         .rx_ring_num = 5,
114                         .tx_msg_size = 128,
115                         .rx_msg_size = 32,
116                 }
117         },
118         /* queue pairs which provide a compression service */
119         [QAT_SERVICE_COMPRESSION] = {
120                 {
121                         .service_type = QAT_SERVICE_COMPRESSION,
122                         .hw_bundle_num = 0,
123                         .tx_ring_num = 3,
124                         .rx_ring_num = 7,
125                         .tx_msg_size = 128,
126                         .rx_msg_size = 32,
127                 }
128         }
129 };
130
131 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
132         uint32_t queue_size_bytes);
133 static void qat_queue_delete(struct qat_queue *queue);
134 static int qat_queue_create(struct qat_pci_device *qat_dev,
135         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
136 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
137         uint32_t *queue_size_for_csr);
138 static void adf_configure_queues(struct qat_qp *queue);
139 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
140         rte_spinlock_t *lock);
141 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
142         rte_spinlock_t *lock);
143
144
145 int qat_qps_per_service(const struct qat_qp_hw_data *qp_hw_data,
146                 enum qat_service_type service)
147 {
148         int i, count;
149
150         for (i = 0, count = 0; i < ADF_MAX_QPS_ON_ANY_SERVICE; i++)
151                 if (qp_hw_data[i].service_type == service)
152                         count++;
153         return count;
154 }
155
156 static const struct rte_memzone *
157 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
158                         int socket_id)
159 {
160         const struct rte_memzone *mz;
161
162         mz = rte_memzone_lookup(queue_name);
163         if (mz != 0) {
164                 if (((size_t)queue_size <= mz->len) &&
165                                 ((socket_id == SOCKET_ID_ANY) ||
166                                         (socket_id == mz->socket_id))) {
167                         QAT_LOG(DEBUG, "re-use memzone already "
168                                         "allocated for %s", queue_name);
169                         return mz;
170                 }
171
172                 QAT_LOG(ERR, "Incompatible memzone already "
173                                 "allocated %s, size %u, socket %d. "
174                                 "Requested size %u, socket %u",
175                                 queue_name, (uint32_t)mz->len,
176                                 mz->socket_id, queue_size, socket_id);
177                 return NULL;
178         }
179
180         QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
181                                         queue_name, queue_size, socket_id);
182         return rte_memzone_reserve_aligned(queue_name, queue_size,
183                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
184 }
185
186 int qat_qp_setup(struct qat_pci_device *qat_dev,
187                 struct qat_qp **qp_addr,
188                 uint16_t queue_pair_id,
189                 struct qat_qp_config *qat_qp_conf)
190
191 {
192         struct qat_qp *qp;
193         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
194         char op_cookie_pool_name[RTE_RING_NAMESIZE];
195         uint32_t i;
196
197         QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
198                 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
199
200         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
201                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
202                 QAT_LOG(ERR, "Can't create qp for %u descriptors",
203                                 qat_qp_conf->nb_descriptors);
204                 return -EINVAL;
205         }
206
207         if (pci_dev->mem_resource[0].addr == NULL) {
208                 QAT_LOG(ERR, "Could not find VF config space "
209                                 "(UIO driver attached?).");
210                 return -EINVAL;
211         }
212
213         /* Allocate the queue pair data structure. */
214         qp = rte_zmalloc("qat PMD qp metadata",
215                         sizeof(*qp), RTE_CACHE_LINE_SIZE);
216         if (qp == NULL) {
217                 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
218                 return -ENOMEM;
219         }
220         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
221         qp->op_cookies = rte_zmalloc("qat PMD op cookie pointer",
222                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
223                         RTE_CACHE_LINE_SIZE);
224         if (qp->op_cookies == NULL) {
225                 QAT_LOG(ERR, "Failed to alloc mem for cookie");
226                 rte_free(qp);
227                 return -ENOMEM;
228         }
229
230         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
231         qp->inflights16 = 0;
232
233         if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
234                                         ADF_RING_DIR_TX) != 0) {
235                 QAT_LOG(ERR, "Tx queue create failed "
236                                 "queue_pair_id=%u", queue_pair_id);
237                 goto create_err;
238         }
239
240         if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
241                                         ADF_RING_DIR_RX) != 0) {
242                 QAT_LOG(ERR, "Rx queue create failed "
243                                 "queue_pair_id=%hu", queue_pair_id);
244                 qat_queue_delete(&(qp->tx_q));
245                 goto create_err;
246         }
247
248         adf_configure_queues(qp);
249         adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr,
250                                         &qat_dev->arb_csr_lock);
251
252         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
253                                         "%s%d_cookies_%s_qp%hu",
254                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
255                 qat_qp_conf->service_str, queue_pair_id);
256
257         QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
258         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
259         if (qp->op_cookie_pool == NULL)
260                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
261                                 qp->nb_descriptors,
262                                 qat_qp_conf->cookie_size, 64, 0,
263                                 NULL, NULL, NULL, NULL, qat_qp_conf->socket_id,
264                                 0);
265         if (!qp->op_cookie_pool) {
266                 QAT_LOG(ERR, "QAT PMD Cannot create"
267                                 " op mempool");
268                 goto create_err;
269         }
270
271         for (i = 0; i < qp->nb_descriptors; i++) {
272                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
273                         QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
274                         goto create_err;
275                 }
276         }
277
278         qp->qat_dev_gen = qat_dev->qat_dev_gen;
279         qp->build_request = qat_qp_conf->build_request;
280         qp->service_type = qat_qp_conf->hw->service_type;
281         qp->qat_dev = qat_dev;
282
283         QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
284                         queue_pair_id, op_cookie_pool_name);
285
286         *qp_addr = qp;
287         return 0;
288
289 create_err:
290         if (qp->op_cookie_pool)
291                 rte_mempool_free(qp->op_cookie_pool);
292         rte_free(qp->op_cookies);
293         rte_free(qp);
294         return -EFAULT;
295 }
296
297 int qat_qp_release(struct qat_qp **qp_addr)
298 {
299         struct qat_qp *qp = *qp_addr;
300         uint32_t i;
301
302         if (qp == NULL) {
303                 QAT_LOG(DEBUG, "qp already freed");
304                 return 0;
305         }
306
307         QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
308                                 qp->qat_dev->qat_dev_id);
309
310         /* Don't free memory if there are still responses to be processed */
311         if (qp->inflights16 == 0) {
312                 qat_queue_delete(&(qp->tx_q));
313                 qat_queue_delete(&(qp->rx_q));
314         } else {
315                 return -EAGAIN;
316         }
317
318         adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr,
319                                         &qp->qat_dev->arb_csr_lock);
320
321         for (i = 0; i < qp->nb_descriptors; i++)
322                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
323
324         if (qp->op_cookie_pool)
325                 rte_mempool_free(qp->op_cookie_pool);
326
327         rte_free(qp->op_cookies);
328         rte_free(qp);
329         *qp_addr = NULL;
330         return 0;
331 }
332
333
334 static void qat_queue_delete(struct qat_queue *queue)
335 {
336         const struct rte_memzone *mz;
337         int status = 0;
338
339         if (queue == NULL) {
340                 QAT_LOG(DEBUG, "Invalid queue");
341                 return;
342         }
343         QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
344                         queue->hw_queue_number, queue->memz_name);
345
346         mz = rte_memzone_lookup(queue->memz_name);
347         if (mz != NULL) {
348                 /* Write an unused pattern to the queue memory. */
349                 memset(queue->base_addr, 0x7F, queue->queue_size);
350                 status = rte_memzone_free(mz);
351                 if (status != 0)
352                         QAT_LOG(ERR, "Error %d on freeing queue %s",
353                                         status, queue->memz_name);
354         } else {
355                 QAT_LOG(DEBUG, "queue %s doesn't exist",
356                                 queue->memz_name);
357         }
358 }
359
360 static int
361 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
362                 struct qat_qp_config *qp_conf, uint8_t dir)
363 {
364         uint64_t queue_base;
365         void *io_addr;
366         const struct rte_memzone *qp_mz;
367         struct rte_pci_device *pci_dev = qat_dev->pci_dev;
368         int ret = 0;
369         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
370                         qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
371         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
372
373         queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
374         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
375                         qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
376
377         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
378                 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
379                 return -EINVAL;
380         }
381
382         /*
383          * Allocate a memzone for the queue - create a unique name.
384          */
385         snprintf(queue->memz_name, sizeof(queue->memz_name),
386                         "%s_%d_%s_%s_%d_%d",
387                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
388                 qp_conf->service_str, "qp_mem",
389                 queue->hw_bundle_number, queue->hw_queue_number);
390         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
391                         qp_conf->socket_id);
392         if (qp_mz == NULL) {
393                 QAT_LOG(ERR, "Failed to allocate ring memzone");
394                 return -ENOMEM;
395         }
396
397         queue->base_addr = (char *)qp_mz->addr;
398         queue->base_phys_addr = qp_mz->iova;
399         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
400                         queue_size_bytes)) {
401                 QAT_LOG(ERR, "Invalid alignment on queue create "
402                                         " 0x%"PRIx64"\n",
403                                         queue->base_phys_addr);
404                 ret = -EFAULT;
405                 goto queue_create_err;
406         }
407
408         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
409                         &(queue->queue_size)) != 0) {
410                 QAT_LOG(ERR, "Invalid num inflights");
411                 ret = -EINVAL;
412                 goto queue_create_err;
413         }
414
415         queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
416                                         ADF_BYTES_TO_MSG_SIZE(desc_size));
417         queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
418
419         if (queue->max_inflights < 2) {
420                 QAT_LOG(ERR, "Invalid num inflights");
421                 ret = -EINVAL;
422                 goto queue_create_err;
423         }
424         queue->head = 0;
425         queue->tail = 0;
426         queue->msg_size = desc_size;
427
428         /*
429          * Write an unused pattern to the queue memory.
430          */
431         memset(queue->base_addr, 0x7F, queue_size_bytes);
432
433         queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
434                                         queue->queue_size);
435
436         io_addr = pci_dev->mem_resource[0].addr;
437
438         WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
439                         queue->hw_queue_number, queue_base);
440
441         QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
442                 " nb msgs %u, msg_size %u, max_inflights %u modulo mask %u",
443                         queue->memz_name,
444                         queue->queue_size, queue_size_bytes,
445                         qp_conf->nb_descriptors, desc_size,
446                         queue->max_inflights, queue->modulo_mask);
447
448         return 0;
449
450 queue_create_err:
451         rte_memzone_free(qp_mz);
452         return ret;
453 }
454
455 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
456                                         uint32_t queue_size_bytes)
457 {
458         if (((queue_size_bytes - 1) & phys_addr) != 0)
459                 return -EINVAL;
460         return 0;
461 }
462
463 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
464         uint32_t *p_queue_size_for_csr)
465 {
466         uint8_t i = ADF_MIN_RING_SIZE;
467
468         for (; i <= ADF_MAX_RING_SIZE; i++)
469                 if ((msg_size * msg_num) ==
470                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
471                         *p_queue_size_for_csr = i;
472                         return 0;
473                 }
474         QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
475         return -EINVAL;
476 }
477
478 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr,
479                                         rte_spinlock_t *lock)
480 {
481         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
482                                         (ADF_ARB_REG_SLOT *
483                                                         txq->hw_bundle_number);
484         uint32_t value;
485
486         rte_spinlock_lock(lock);
487         value = ADF_CSR_RD(base_addr, arb_csr_offset);
488         value |= (0x01 << txq->hw_queue_number);
489         ADF_CSR_WR(base_addr, arb_csr_offset, value);
490         rte_spinlock_unlock(lock);
491 }
492
493 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr,
494                                         rte_spinlock_t *lock)
495 {
496         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
497                                         (ADF_ARB_REG_SLOT *
498                                                         txq->hw_bundle_number);
499         uint32_t value;
500
501         rte_spinlock_lock(lock);
502         value = ADF_CSR_RD(base_addr, arb_csr_offset);
503         value &= ~(0x01 << txq->hw_queue_number);
504         ADF_CSR_WR(base_addr, arb_csr_offset, value);
505         rte_spinlock_unlock(lock);
506 }
507
508 static void adf_configure_queues(struct qat_qp *qp)
509 {
510         uint32_t queue_config;
511         struct qat_queue *queue = &qp->tx_q;
512
513         queue_config = BUILD_RING_CONFIG(queue->queue_size);
514
515         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
516                         queue->hw_queue_number, queue_config);
517
518         queue = &qp->rx_q;
519         queue_config =
520                         BUILD_RESP_RING_CONFIG(queue->queue_size,
521                                         ADF_RING_NEAR_WATERMARK_512,
522                                         ADF_RING_NEAR_WATERMARK_0);
523
524         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
525                         queue->hw_queue_number, queue_config);
526 }
527
528 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
529 {
530         return data & modulo_mask;
531 }
532
533 static inline void
534 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
535         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
536                         q->hw_queue_number, q->tail);
537         q->nb_pending_requests = 0;
538         q->csr_tail = q->tail;
539 }
540
541 static inline
542 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
543 {
544         uint32_t old_head, new_head;
545         uint32_t max_head;
546
547         old_head = q->csr_head;
548         new_head = q->head;
549         max_head = qp->nb_descriptors * q->msg_size;
550
551         /* write out free descriptors */
552         void *cur_desc = (uint8_t *)q->base_addr + old_head;
553
554         if (new_head < old_head) {
555                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
556                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
557         } else {
558                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
559         }
560         q->nb_processed_responses = 0;
561         q->csr_head = new_head;
562
563         /* write current head to CSR */
564         WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
565                             q->hw_queue_number, new_head);
566 }
567
568 uint16_t
569 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
570 {
571         register struct qat_queue *queue;
572         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
573         register uint32_t nb_ops_sent = 0;
574         register int ret;
575         uint16_t nb_ops_possible = nb_ops;
576         register uint8_t *base_addr;
577         register uint32_t tail;
578         int overflow;
579
580         if (unlikely(nb_ops == 0))
581                 return 0;
582
583         /* read params used a lot in main loop into registers */
584         queue = &(tmp_qp->tx_q);
585         base_addr = (uint8_t *)queue->base_addr;
586         tail = queue->tail;
587
588         /* Find how many can actually fit on the ring */
589         tmp_qp->inflights16 += nb_ops;
590         overflow = tmp_qp->inflights16 - queue->max_inflights;
591         if (overflow > 0) {
592                 tmp_qp->inflights16 -= overflow;
593                 nb_ops_possible = nb_ops - overflow;
594                 if (nb_ops_possible == 0)
595                         return 0;
596         }
597
598         while (nb_ops_sent != nb_ops_possible) {
599                 ret = tmp_qp->build_request(*ops, base_addr + tail,
600                                 tmp_qp->op_cookies[tail / queue->msg_size],
601                                 tmp_qp->qat_dev_gen);
602                 if (ret != 0) {
603                         tmp_qp->stats.enqueue_err_count++;
604                         /*
605                          * This message cannot be enqueued,
606                          * decrease number of ops that wasn't sent
607                          */
608                         tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
609                         if (nb_ops_sent == 0)
610                                 return 0;
611                         goto kick_tail;
612                 }
613
614                 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
615                 ops++;
616                 nb_ops_sent++;
617         }
618 kick_tail:
619         queue->tail = tail;
620         tmp_qp->stats.enqueued_count += nb_ops_sent;
621         queue->nb_pending_requests += nb_ops_sent;
622         if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
623                     queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
624                 txq_write_tail(tmp_qp, queue);
625         }
626         return nb_ops_sent;
627 }
628
629 uint16_t
630 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
631 {
632         struct qat_queue *rx_queue, *tx_queue;
633         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
634         uint32_t head;
635         uint32_t resp_counter = 0;
636         uint8_t *resp_msg;
637
638         rx_queue = &(tmp_qp->rx_q);
639         tx_queue = &(tmp_qp->tx_q);
640         head = rx_queue->head;
641         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
642
643         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
644                         resp_counter != nb_ops) {
645
646                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
647                         qat_sym_process_response(ops, resp_msg);
648                 else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
649                         qat_comp_process_response(ops, resp_msg,
650                                         &tmp_qp->stats.dequeue_err_count);
651
652                 head = adf_modulo(head + rx_queue->msg_size,
653                                   rx_queue->modulo_mask);
654
655                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
656                 ops++;
657                 resp_counter++;
658         }
659         if (resp_counter > 0) {
660                 rx_queue->head = head;
661                 tmp_qp->stats.dequeued_count += resp_counter;
662                 rx_queue->nb_processed_responses += resp_counter;
663                 tmp_qp->inflights16 -= resp_counter;
664
665                 if (rx_queue->nb_processed_responses >
666                                                 QAT_CSR_HEAD_WRITE_THRESH)
667                         rxq_free_desc(tmp_qp, rx_queue);
668         }
669         /* also check if tail needs to be advanced */
670         if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
671                 tx_queue->tail != tx_queue->csr_tail) {
672                 txq_write_tail(tmp_qp, tx_queue);
673         }
674         return resp_counter;
675 }
676
677 __rte_weak int
678 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
679                           uint64_t *dequeue_err_count __rte_unused)
680 {
681         return  0;
682 }