drivers/common/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_cycles.h>
   7 #include <rte_dev.h>
   8 #include <rte_malloc.h>
   9 #include <rte_memzone.h>
  10 #include <rte_pci.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_atomic.h>
  13 #include <rte_prefetch.h>
  14
  15 #include "qat_logs.h"
  16 #include "qat_device.h"
  17 #include "qat_qp.h"
  18 #include "qat_sym.h"
  19 #include "qat_asym.h"
  20 #include "qat_comp.h"
  21 #include "adf_transport_access_macros.h"
  22 #include "adf_transport_access_macros_gen4vf.h"
  23 #include "dev/qat_dev_gens.h"
  24
  25 #define QAT_CQ_MAX_DEQ_RETRIES 10
  26
  27 #define ADF_MAX_DESC                            4096
  28 #define ADF_MIN_DESC                            128
  29
  30 #define ADF_ARB_REG_SLOT                        0x1000
  31 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  32
  33 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  34         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  35         (ADF_ARB_REG_SLOT * index), value)
  36
  37 __extension__
  38 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
  39                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  40         /* queue pairs which provide an asymmetric crypto service */
  41         [QAT_SERVICE_ASYMMETRIC] = {
  42                 {
  43                         .service_type = QAT_SERVICE_ASYMMETRIC,
  44                         .hw_bundle_num = 0,
  45                         .tx_ring_num = 0,
  46                         .rx_ring_num = 8,
  47                         .tx_msg_size = 64,
  48                         .rx_msg_size = 32,
  49
  50                 }, {
  51                         .service_type = QAT_SERVICE_ASYMMETRIC,
  52                         .hw_bundle_num = 0,
  53                         .tx_ring_num = 1,
  54                         .rx_ring_num = 9,
  55                         .tx_msg_size = 64,
  56                         .rx_msg_size = 32,
  57                 }
  58         },
  59         /* queue pairs which provide a symmetric crypto service */
  60         [QAT_SERVICE_SYMMETRIC] = {
  61                 {
  62                         .service_type = QAT_SERVICE_SYMMETRIC,
  63                         .hw_bundle_num = 0,
  64                         .tx_ring_num = 2,
  65                         .rx_ring_num = 10,
  66                         .tx_msg_size = 128,
  67                         .rx_msg_size = 32,
  68                 },
  69                 {
  70                         .service_type = QAT_SERVICE_SYMMETRIC,
  71                         .hw_bundle_num = 0,
  72                         .tx_ring_num = 3,
  73                         .rx_ring_num = 11,
  74                         .tx_msg_size = 128,
  75                         .rx_msg_size = 32,
  76                 }
  77         },
  78         /* queue pairs which provide a compression service */
  79         [QAT_SERVICE_COMPRESSION] = {
  80                 {
  81                         .service_type = QAT_SERVICE_COMPRESSION,
  82                         .hw_bundle_num = 0,
  83                         .tx_ring_num = 6,
  84                         .rx_ring_num = 14,
  85                         .tx_msg_size = 128,
  86                         .rx_msg_size = 32,
  87                 }, {
  88                         .service_type = QAT_SERVICE_COMPRESSION,
  89                         .hw_bundle_num = 0,
  90                         .tx_ring_num = 7,
  91                         .rx_ring_num = 15,
  92                         .tx_msg_size = 128,
  93                         .rx_msg_size = 32,
  94                 }
  95         }
  96 };
  97
  98 __extension__
  99 const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
 100                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
 101         /* queue pairs which provide an asymmetric crypto service */
 102         [QAT_SERVICE_ASYMMETRIC] = {
 103                 {
 104                         .service_type = QAT_SERVICE_ASYMMETRIC,
 105                         .hw_bundle_num = 0,
 106                         .tx_ring_num = 0,
 107                         .rx_ring_num = 4,
 108                         .tx_msg_size = 64,
 109                         .rx_msg_size = 32,
 110                 }
 111         },
 112         /* queue pairs which provide a symmetric crypto service */
 113         [QAT_SERVICE_SYMMETRIC] = {
 114                 {
 115                         .service_type = QAT_SERVICE_SYMMETRIC,
 116                         .hw_bundle_num = 0,
 117                         .tx_ring_num = 1,
 118                         .rx_ring_num = 5,
 119                         .tx_msg_size = 128,
 120                         .rx_msg_size = 32,
 121                 }
 122         },
 123         /* queue pairs which provide a compression service */
 124         [QAT_SERVICE_COMPRESSION] = {
 125                 {
 126                         .service_type = QAT_SERVICE_COMPRESSION,
 127                         .hw_bundle_num = 0,
 128                         .tx_ring_num = 3,
 129                         .rx_ring_num = 7,
 130                         .tx_msg_size = 128,
 131                         .rx_msg_size = 32,
 132                 }
 133         }
 134 };
 135
 136 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 137         uint32_t queue_size_bytes);
 138 static void qat_queue_delete(struct qat_queue *queue);
 139 static int qat_queue_create(struct qat_pci_device *qat_dev,
 140         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
 141 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 142         uint32_t *queue_size_for_csr);
 143 static void adf_configure_queues(struct qat_qp *queue,
 144         enum qat_device_gen qat_dev_gen);
 145 static void adf_queue_arb_enable(enum qat_device_gen qat_dev_gen,
 146         struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
 147 static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
 148         struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
 149
 150 int qat_qps_per_service(struct qat_pci_device *qat_dev,
 151                 enum qat_service_type service)
 152 {
 153         int i = 0, count = 0, max_ops_per_srv = 0;
 154
 155         if (qat_dev->qat_dev_gen == QAT_GEN4) {
 156                 max_ops_per_srv = QAT_GEN4_BUNDLE_NUM;
 157                 for (i = 0, count = 0; i < max_ops_per_srv; i++)
 158                         if (qat_dev->qp_gen4_data[i][0].service_type == service)
 159                                 count++;
 160         } else {
 161                 const struct qat_qp_hw_data *sym_hw_qps =
 162                                 qat_gen_config[qat_dev->qat_dev_gen]
 163                                 .qp_hw_data[service];
 164
 165                 max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE;
 166                 for (i = 0, count = 0; i < max_ops_per_srv; i++)
 167                         if (sym_hw_qps[i].service_type == service)
 168                                 count++;
 169         }
 170
 171         return count;
 172 }
 173
 174 static const struct rte_memzone *
 175 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 176                         int socket_id)
 177 {
 178         const struct rte_memzone *mz;
 179
 180         mz = rte_memzone_lookup(queue_name);
 181         if (mz != 0) {
 182                 if (((size_t)queue_size <= mz->len) &&
 183                                 ((socket_id == SOCKET_ID_ANY) ||
 184                                         (socket_id == mz->socket_id))) {
 185                         QAT_LOG(DEBUG, "re-use memzone already "
 186                                         "allocated for %s", queue_name);
 187                         return mz;
 188                 }
 189
 190                 QAT_LOG(ERR, "Incompatible memzone already "
 191                                 "allocated %s, size %u, socket %d. "
 192                                 "Requested size %u, socket %u",
 193                                 queue_name, (uint32_t)mz->len,
 194                                 mz->socket_id, queue_size, socket_id);
 195                 return NULL;
 196         }
 197
 198         QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
 199                                         queue_name, queue_size, socket_id);
 200         return rte_memzone_reserve_aligned(queue_name, queue_size,
 201                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 202 }
 203
 204 int qat_qp_setup(struct qat_pci_device *qat_dev,
 205                 struct qat_qp **qp_addr,
 206                 uint16_t queue_pair_id,
 207                 struct qat_qp_config *qat_qp_conf)
 208 {
 209         struct qat_qp *qp;
 210         struct rte_pci_device *pci_dev =
 211                         qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
 212         char op_cookie_pool_name[RTE_RING_NAMESIZE];
 213         enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
 214         uint32_t i;
 215
 216         QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
 217                 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
 218
 219         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
 220                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
 221                 QAT_LOG(ERR, "Can't create qp for %u descriptors",
 222                                 qat_qp_conf->nb_descriptors);
 223                 return -EINVAL;
 224         }
 225
 226         if (pci_dev->mem_resource[0].addr == NULL) {
 227                 QAT_LOG(ERR, "Could not find VF config space "
 228                                 "(UIO driver attached?).");
 229                 return -EINVAL;
 230         }
 231
 232         /* Allocate the queue pair data structure. */
 233         qp = rte_zmalloc_socket("qat PMD qp metadata",
 234                                 sizeof(*qp), RTE_CACHE_LINE_SIZE,
 235                                 qat_qp_conf->socket_id);
 236         if (qp == NULL) {
 237                 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
 238                 return -ENOMEM;
 239         }
 240         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
 241         qp->op_cookies = rte_zmalloc_socket("qat PMD op cookie pointer",
 242                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 243                         RTE_CACHE_LINE_SIZE, qat_qp_conf->socket_id);
 244         if (qp->op_cookies == NULL) {
 245                 QAT_LOG(ERR, "Failed to alloc mem for cookie");
 246                 rte_free(qp);
 247                 return -ENOMEM;
 248         }
 249
 250         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 251         qp->enqueued = qp->dequeued = 0;
 252
 253         if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
 254                                         ADF_RING_DIR_TX) != 0) {
 255                 QAT_LOG(ERR, "Tx queue create failed "
 256                                 "queue_pair_id=%u", queue_pair_id);
 257                 goto create_err;
 258         }
 259
 260         qp->max_inflights = ADF_MAX_INFLIGHTS(qp->tx_q.queue_size,
 261                                 ADF_BYTES_TO_MSG_SIZE(qp->tx_q.msg_size));
 262
 263         if (qp->max_inflights < 2) {
 264                 QAT_LOG(ERR, "Invalid num inflights");
 265                 qat_queue_delete(&(qp->tx_q));
 266                 goto create_err;
 267         }
 268
 269         if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
 270                                         ADF_RING_DIR_RX) != 0) {
 271                 QAT_LOG(ERR, "Rx queue create failed "
 272                                 "queue_pair_id=%hu", queue_pair_id);
 273                 qat_queue_delete(&(qp->tx_q));
 274                 goto create_err;
 275         }
 276
 277         adf_configure_queues(qp, qat_dev_gen);
 278         adf_queue_arb_enable(qat_dev_gen, &qp->tx_q, qp->mmap_bar_addr,
 279                                         &qat_dev->arb_csr_lock);
 280
 281         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
 282                                         "%s%d_cookies_%s_qp%hu",
 283                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 284                 qat_qp_conf->service_str, queue_pair_id);
 285
 286         QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
 287         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 288         if (qp->op_cookie_pool == NULL)
 289                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 290                                 qp->nb_descriptors,
 291                                 qat_qp_conf->cookie_size, 64, 0,
 292                                 NULL, NULL, NULL, NULL,
 293                                 pci_dev->device.numa_node,
 294                                 0);
 295         if (!qp->op_cookie_pool) {
 296                 QAT_LOG(ERR, "QAT PMD Cannot create"
 297                                 " op mempool");
 298                 goto create_err;
 299         }
 300
 301         for (i = 0; i < qp->nb_descriptors; i++) {
 302                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 303                         QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
 304                         goto create_err;
 305                 }
 306                 memset(qp->op_cookies[i], 0, qat_qp_conf->cookie_size);
 307         }
 308
 309         qp->qat_dev_gen = qat_dev->qat_dev_gen;
 310         qp->service_type = qat_qp_conf->hw->service_type;
 311         qp->qat_dev = qat_dev;
 312
 313         QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
 314                         queue_pair_id, op_cookie_pool_name);
 315
 316         *qp_addr = qp;
 317         return 0;
 318
 319 create_err:
 320         if (qp->op_cookie_pool)
 321                 rte_mempool_free(qp->op_cookie_pool);
 322         rte_free(qp->op_cookies);
 323         rte_free(qp);
 324         return -EFAULT;
 325 }
 326
 327
 328 int qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr)
 329 {
 330         struct qat_qp *qp = *qp_addr;
 331         uint32_t i;
 332
 333         if (qp == NULL) {
 334                 QAT_LOG(DEBUG, "qp already freed");
 335                 return 0;
 336         }
 337
 338         QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
 339                                 qp->qat_dev->qat_dev_id);
 340
 341         /* Don't free memory if there are still responses to be processed */
 342         if ((qp->enqueued - qp->dequeued) == 0) {
 343                 qat_queue_delete(&(qp->tx_q));
 344                 qat_queue_delete(&(qp->rx_q));
 345         } else {
 346                 return -EAGAIN;
 347         }
 348
 349         adf_queue_arb_disable(qat_dev_gen, &(qp->tx_q), qp->mmap_bar_addr,
 350                                 &qp->qat_dev->arb_csr_lock);
 351
 352         for (i = 0; i < qp->nb_descriptors; i++)
 353                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 354
 355         if (qp->op_cookie_pool)
 356                 rte_mempool_free(qp->op_cookie_pool);
 357
 358         rte_free(qp->op_cookies);
 359         rte_free(qp);
 360         *qp_addr = NULL;
 361         return 0;
 362 }
 363
 364
 365 static void qat_queue_delete(struct qat_queue *queue)
 366 {
 367         const struct rte_memzone *mz;
 368         int status = 0;
 369
 370         if (queue == NULL) {
 371                 QAT_LOG(DEBUG, "Invalid queue");
 372                 return;
 373         }
 374         QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
 375                         queue->hw_queue_number, queue->memz_name);
 376
 377         mz = rte_memzone_lookup(queue->memz_name);
 378         if (mz != NULL) {
 379                 /* Write an unused pattern to the queue memory. */
 380                 memset(queue->base_addr, 0x7F, queue->queue_size);
 381                 status = rte_memzone_free(mz);
 382                 if (status != 0)
 383                         QAT_LOG(ERR, "Error %d on freeing queue %s",
 384                                         status, queue->memz_name);
 385         } else {
 386                 QAT_LOG(DEBUG, "queue %s doesn't exist",
 387                                 queue->memz_name);
 388         }
 389 }
 390
 391 static int
 392 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 393                 struct qat_qp_config *qp_conf, uint8_t dir)
 394 {
 395         uint64_t queue_base;
 396         void *io_addr;
 397         const struct rte_memzone *qp_mz;
 398         struct rte_pci_device *pci_dev =
 399                         qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
 400         enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
 401         int ret = 0;
 402         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
 403                         qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
 404         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
 405
 406         queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
 407         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
 408                         qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
 409
 410         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 411                 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
 412                 return -EINVAL;
 413         }
 414
 415         /*
 416          * Allocate a memzone for the queue - create a unique name.
 417          */
 418         snprintf(queue->memz_name, sizeof(queue->memz_name),
 419                         "%s_%d_%s_%s_%d_%d",
 420                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 421                 qp_conf->service_str, "qp_mem",
 422                 queue->hw_bundle_number, queue->hw_queue_number);
 423         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 424                         pci_dev->device.numa_node);
 425         if (qp_mz == NULL) {
 426                 QAT_LOG(ERR, "Failed to allocate ring memzone");
 427                 return -ENOMEM;
 428         }
 429
 430         queue->base_addr = (char *)qp_mz->addr;
 431         queue->base_phys_addr = qp_mz->iova;
 432         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 433                         queue_size_bytes)) {
 434                 QAT_LOG(ERR, "Invalid alignment on queue create "
 435                                         " 0x%"PRIx64"\n",
 436                                         queue->base_phys_addr);
 437                 ret = -EFAULT;
 438                 goto queue_create_err;
 439         }
 440
 441         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
 442                         &(queue->queue_size)) != 0) {
 443                 QAT_LOG(ERR, "Invalid num inflights");
 444                 ret = -EINVAL;
 445                 goto queue_create_err;
 446         }
 447
 448         queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
 449         queue->head = 0;
 450         queue->tail = 0;
 451         queue->msg_size = desc_size;
 452
 453         /* For fast calculation of cookie index, relies on msg_size being 2^n */
 454         queue->trailz = __builtin_ctz(desc_size);
 455
 456         /*
 457          * Write an unused pattern to the queue memory.
 458          */
 459         memset(queue->base_addr, 0x7F, queue_size_bytes);
 460         io_addr = pci_dev->mem_resource[0].addr;
 461
 462         if (qat_dev_gen == QAT_GEN4) {
 463                 queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr,
 464                                         queue->queue_size);
 465                 WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number,
 466                         queue->hw_queue_number, queue_base);
 467         } else {
 468                 queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 469                                 queue->queue_size);
 470                 WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 471                         queue->hw_queue_number, queue_base);
 472         }
 473
 474         QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
 475                 " nb msgs %u, msg_size %u, modulo mask %u",
 476                         queue->memz_name,
 477                         queue->queue_size, queue_size_bytes,
 478                         qp_conf->nb_descriptors, desc_size,
 479                         queue->modulo_mask);
 480
 481         return 0;
 482
 483 queue_create_err:
 484         rte_memzone_free(qp_mz);
 485         return ret;
 486 }
 487
 488 int
 489 qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id,
 490                         enum qat_service_type service_type)
 491 {
 492         if (qat_dev->qat_dev_gen == QAT_GEN4) {
 493                 int i = 0, valid_qps = 0;
 494
 495                 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
 496                         if (qat_dev->qp_gen4_data[i][0].service_type ==
 497                                 service_type) {
 498                                 if (valid_qps == qp_id)
 499                                         return i;
 500                                 ++valid_qps;
 501                         }
 502                 }
 503         }
 504         return -1;
 505 }
 506
 507 int
 508 qat_read_qp_config(struct qat_pci_device *qat_dev)
 509 {
 510         int i = 0;
 511         enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
 512
 513         if (qat_dev_gen == QAT_GEN4) {
 514                 uint16_t svc = 0;
 515
 516                 if (qat_query_svc_gen4(qat_dev, (uint8_t *)&svc))
 517                         return -(EFAULT);
 518                 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
 519                         struct qat_qp_hw_data *hw_data =
 520                                 &qat_dev->qp_gen4_data[i][0];
 521                         uint8_t svc1 = (svc >> (3 * i)) & 0x7;
 522                         enum qat_service_type service_type = QAT_SERVICE_INVALID;
 523
 524                         if (svc1 == QAT_SVC_SYM) {
 525                                 service_type = QAT_SERVICE_SYMMETRIC;
 526                                 QAT_LOG(DEBUG,
 527                                         "Discovered SYMMETRIC service on bundle %d",
 528                                         i);
 529                         } else if (svc1 == QAT_SVC_COMPRESSION) {
 530                                 service_type = QAT_SERVICE_COMPRESSION;
 531                                 QAT_LOG(DEBUG,
 532                                         "Discovered COPRESSION service on bundle %d",
 533                                         i);
 534                         } else if (svc1 == QAT_SVC_ASYM) {
 535                                 service_type = QAT_SERVICE_ASYMMETRIC;
 536                                 QAT_LOG(DEBUG,
 537                                         "Discovered ASYMMETRIC service on bundle %d",
 538                                         i);
 539                         } else {
 540                                 QAT_LOG(ERR,
 541                                         "Unrecognized service on bundle %d",
 542                                         i);
 543                                 return -(EFAULT);
 544                         }
 545
 546                         memset(hw_data, 0, sizeof(*hw_data));
 547                         hw_data->service_type = service_type;
 548                         if (service_type == QAT_SERVICE_ASYMMETRIC) {
 549                                 hw_data->tx_msg_size = 64;
 550                                 hw_data->rx_msg_size = 32;
 551                         } else if (service_type == QAT_SERVICE_SYMMETRIC ||
 552                                         service_type ==
 553                                                 QAT_SERVICE_COMPRESSION) {
 554                                 hw_data->tx_msg_size = 128;
 555                                 hw_data->rx_msg_size = 32;
 556                         }
 557                         hw_data->tx_ring_num = 0;
 558                         hw_data->rx_ring_num = 1;
 559                         hw_data->hw_bundle_num = i;
 560                 }
 561                 return 0;
 562         }
 563         return -(EINVAL);
 564 }
 565
 566 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 567                                         uint32_t queue_size_bytes)
 568 {
 569         if (((queue_size_bytes - 1) & phys_addr) != 0)
 570                 return -EINVAL;
 571         return 0;
 572 }
 573
 574 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 575         uint32_t *p_queue_size_for_csr)
 576 {
 577         uint8_t i = ADF_MIN_RING_SIZE;
 578
 579         for (; i <= ADF_MAX_RING_SIZE; i++)
 580                 if ((msg_size * msg_num) ==
 581                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 582                         *p_queue_size_for_csr = i;
 583                         return 0;
 584                 }
 585         QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 586         return -EINVAL;
 587 }
 588
 589 static void
 590 adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq,
 591                         void *base_addr, rte_spinlock_t *lock)
 592 {
 593         uint32_t arb_csr_offset = 0, value;
 594
 595         rte_spinlock_lock(lock);
 596         if (qat_dev_gen == QAT_GEN4) {
 597                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 598                                 (ADF_RING_BUNDLE_SIZE_GEN4 *
 599                                 txq->hw_bundle_number);
 600                 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
 601                                 arb_csr_offset);
 602         } else {
 603                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 604                                 (ADF_ARB_REG_SLOT *
 605                                 txq->hw_bundle_number);
 606                 value = ADF_CSR_RD(base_addr,
 607                                 arb_csr_offset);
 608         }
 609         value |= (0x01 << txq->hw_queue_number);
 610         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 611         rte_spinlock_unlock(lock);
 612 }
 613
 614 static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
 615                 struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock)
 616 {
 617         uint32_t arb_csr_offset = 0, value;
 618
 619         rte_spinlock_lock(lock);
 620         if (qat_dev_gen == QAT_GEN4) {
 621                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 622                                 (ADF_RING_BUNDLE_SIZE_GEN4 *
 623                                 txq->hw_bundle_number);
 624                 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
 625                                 arb_csr_offset);
 626         } else {
 627                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 628                                 (ADF_ARB_REG_SLOT *
 629                                 txq->hw_bundle_number);
 630                 value = ADF_CSR_RD(base_addr,
 631                                 arb_csr_offset);
 632         }
 633         value &= ~(0x01 << txq->hw_queue_number);
 634         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 635         rte_spinlock_unlock(lock);
 636 }
 637
 638 static void adf_configure_queues(struct qat_qp *qp,
 639                 enum qat_device_gen qat_dev_gen)
 640 {
 641         uint32_t q_tx_config, q_resp_config;
 642         struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q;
 643
 644         q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size);
 645         q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size,
 646                         ADF_RING_NEAR_WATERMARK_512,
 647                         ADF_RING_NEAR_WATERMARK_0);
 648
 649         if (qat_dev_gen == QAT_GEN4) {
 650                 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
 651                         q_tx->hw_bundle_number, q_tx->hw_queue_number,
 652                         q_tx_config);
 653                 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
 654                         q_rx->hw_bundle_number, q_rx->hw_queue_number,
 655                         q_resp_config);
 656         } else {
 657                 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
 658                         q_tx->hw_bundle_number, q_tx->hw_queue_number,
 659                         q_tx_config);
 660                 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
 661                         q_rx->hw_bundle_number, q_rx->hw_queue_number,
 662                         q_resp_config);
 663         }
 664 }
 665
 666 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
 667 {
 668         return data & modulo_mask;
 669 }
 670
 671 static inline void
 672 txq_write_tail(enum qat_device_gen qat_dev_gen,
 673                 struct qat_qp *qp, struct qat_queue *q) {
 674
 675         if (qat_dev_gen == QAT_GEN4) {
 676                 WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr,
 677                         q->hw_bundle_number, q->hw_queue_number, q->tail);
 678         } else {
 679                 WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 680                         q->hw_queue_number, q->tail);
 681         }
 682 }
 683
 684 static inline
 685 void rxq_free_desc(enum qat_device_gen qat_dev_gen, struct qat_qp *qp,
 686                                 struct qat_queue *q)
 687 {
 688         uint32_t old_head, new_head;
 689         uint32_t max_head;
 690
 691         old_head = q->csr_head;
 692         new_head = q->head;
 693         max_head = qp->nb_descriptors * q->msg_size;
 694
 695         /* write out free descriptors */
 696         void *cur_desc = (uint8_t *)q->base_addr + old_head;
 697
 698         if (new_head < old_head) {
 699                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
 700                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
 701         } else {
 702                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
 703         }
 704         q->nb_processed_responses = 0;
 705         q->csr_head = new_head;
 706
 707         /* write current head to CSR */
 708         if (qat_dev_gen == QAT_GEN4) {
 709                 WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr,
 710                         q->hw_bundle_number, q->hw_queue_number, new_head);
 711         } else {
 712                 WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
 713                                 q->hw_queue_number, new_head);
 714         }
 715
 716 }
 717
 718 uint16_t
 719 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 720 {
 721         register struct qat_queue *queue;
 722         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 723         register uint32_t nb_ops_sent = 0;
 724         register int ret = -1;
 725         uint16_t nb_ops_possible = nb_ops;
 726         register uint8_t *base_addr;
 727         register uint32_t tail;
 728
 729         if (unlikely(nb_ops == 0))
 730                 return 0;
 731
 732         /* read params used a lot in main loop into registers */
 733         queue = &(tmp_qp->tx_q);
 734         base_addr = (uint8_t *)queue->base_addr;
 735         tail = queue->tail;
 736
 737         /* Find how many can actually fit on the ring */
 738         {
 739                 /* dequeued can only be written by one thread, but it may not
 740                  * be this thread. As it's 4-byte aligned it will be read
 741                  * atomically here by any Intel CPU.
 742                  * enqueued can wrap before dequeued, but cannot
 743                  * lap it as var size of enq/deq (uint32_t) > var size of
 744                  * max_inflights (uint16_t). In reality inflights is never
 745                  * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
 746                  * On wrapping, the calculation still returns the correct
 747                  * positive value as all three vars are unsigned.
 748                  */
 749                 uint32_t inflights =
 750                         tmp_qp->enqueued - tmp_qp->dequeued;
 751
 752                 if ((inflights + nb_ops) > tmp_qp->max_inflights) {
 753                         nb_ops_possible = tmp_qp->max_inflights - inflights;
 754                         if (nb_ops_possible == 0)
 755                                 return 0;
 756                 }
 757                 /* QAT has plenty of work queued already, so don't waste cycles
 758                  * enqueueing, wait til the application has gathered a bigger
 759                  * burst or some completed ops have been dequeued
 760                  */
 761                 if (tmp_qp->min_enq_burst_threshold && inflights >
 762                                 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
 763                                 tmp_qp->min_enq_burst_threshold) {
 764                         tmp_qp->stats.threshold_hit_count++;
 765                         return 0;
 766                 }
 767         }
 768
 769 #ifdef BUILD_QAT_SYM
 770         if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 771                 qat_sym_preprocess_requests(ops, nb_ops_possible);
 772 #endif
 773
 774         while (nb_ops_sent != nb_ops_possible) {
 775                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
 776 #ifdef BUILD_QAT_SYM
 777                         ret = qat_sym_build_request(*ops, base_addr + tail,
 778                                 tmp_qp->op_cookies[tail >> queue->trailz],
 779                                 tmp_qp->qat_dev_gen);
 780 #endif
 781                 } else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
 782                         ret = qat_comp_build_request(*ops, base_addr + tail,
 783                                 tmp_qp->op_cookies[tail >> queue->trailz],
 784                                 tmp_qp->qat_dev_gen);
 785                 } else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
 786 #ifdef BUILD_QAT_ASYM
 787                         ret = qat_asym_build_request(*ops, base_addr + tail,
 788                                 tmp_qp->op_cookies[tail >> queue->trailz],
 789                                 tmp_qp->qat_dev_gen);
 790 #endif
 791                 }
 792                 if (ret != 0) {
 793                         tmp_qp->stats.enqueue_err_count++;
 794                         /* This message cannot be enqueued */
 795                         if (nb_ops_sent == 0)
 796                                 return 0;
 797                         goto kick_tail;
 798                 }
 799
 800                 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
 801                 ops++;
 802                 nb_ops_sent++;
 803         }
 804 kick_tail:
 805         queue->tail = tail;
 806         tmp_qp->enqueued += nb_ops_sent;
 807         tmp_qp->stats.enqueued_count += nb_ops_sent;
 808         txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
 809         return nb_ops_sent;
 810 }
 811
 812 /* Use this for compression only - but keep consistent with above common
 813  * function as much as possible.
 814  */
 815 uint16_t
 816 qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 817 {
 818         register struct qat_queue *queue;
 819         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 820         register uint32_t nb_ops_sent = 0;
 821         register int nb_desc_to_build;
 822         uint16_t nb_ops_possible = nb_ops;
 823         register uint8_t *base_addr;
 824         register uint32_t tail;
 825
 826         int descriptors_built, total_descriptors_built = 0;
 827         int nb_remaining_descriptors;
 828         int overflow = 0;
 829
 830         if (unlikely(nb_ops == 0))
 831                 return 0;
 832
 833         /* read params used a lot in main loop into registers */
 834         queue = &(tmp_qp->tx_q);
 835         base_addr = (uint8_t *)queue->base_addr;
 836         tail = queue->tail;
 837
 838         /* Find how many can actually fit on the ring */
 839         {
 840                 /* dequeued can only be written by one thread, but it may not
 841                  * be this thread. As it's 4-byte aligned it will be read
 842                  * atomically here by any Intel CPU.
 843                  * enqueued can wrap before dequeued, but cannot
 844                  * lap it as var size of enq/deq (uint32_t) > var size of
 845                  * max_inflights (uint16_t). In reality inflights is never
 846                  * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
 847                  * On wrapping, the calculation still returns the correct
 848                  * positive value as all three vars are unsigned.
 849                  */
 850                 uint32_t inflights =
 851                         tmp_qp->enqueued - tmp_qp->dequeued;
 852
 853                 /* Find how many can actually fit on the ring */
 854                 overflow = (inflights + nb_ops) - tmp_qp->max_inflights;
 855                 if (overflow > 0) {
 856                         nb_ops_possible = nb_ops - overflow;
 857                         if (nb_ops_possible == 0)
 858                                 return 0;
 859                 }
 860
 861                 /* QAT has plenty of work queued already, so don't waste cycles
 862                  * enqueueing, wait til the application has gathered a bigger
 863                  * burst or some completed ops have been dequeued
 864                  */
 865                 if (tmp_qp->min_enq_burst_threshold && inflights >
 866                                 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
 867                                 tmp_qp->min_enq_burst_threshold) {
 868                         tmp_qp->stats.threshold_hit_count++;
 869                         return 0;
 870                 }
 871         }
 872
 873         /* At this point nb_ops_possible is assuming a 1:1 mapping
 874          * between ops and descriptors.
 875          * Fewer may be sent if some ops have to be split.
 876          * nb_ops_possible is <= burst size.
 877          * Find out how many spaces are actually available on the qp in case
 878          * more are needed.
 879          */
 880         nb_remaining_descriptors = nb_ops_possible
 881                          + ((overflow >= 0) ? 0 : overflow * (-1));
 882         QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d",
 883                         nb_ops, nb_remaining_descriptors);
 884
 885         while (nb_ops_sent != nb_ops_possible &&
 886                                 nb_remaining_descriptors > 0) {
 887                 struct qat_comp_op_cookie *cookie =
 888                                 tmp_qp->op_cookies[tail >> queue->trailz];
 889
 890                 descriptors_built = 0;
 891
 892                 QAT_DP_LOG(DEBUG, "--- data length: %u",
 893                            ((struct rte_comp_op *)*ops)->src.length);
 894
 895                 nb_desc_to_build = qat_comp_build_request(*ops,
 896                                 base_addr + tail, cookie, tmp_qp->qat_dev_gen);
 897                 QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, "
 898                         "%d ops sent, %d descriptors needed",
 899                         total_descriptors_built, nb_remaining_descriptors,
 900                         nb_ops_sent, nb_desc_to_build);
 901
 902                 if (unlikely(nb_desc_to_build < 0)) {
 903                         /* this message cannot be enqueued */
 904                         tmp_qp->stats.enqueue_err_count++;
 905                         if (nb_ops_sent == 0)
 906                                 return 0;
 907                         goto kick_tail;
 908                 } else if (unlikely(nb_desc_to_build > 1)) {
 909                         /* this op is too big and must be split - get more
 910                          * descriptors and retry
 911                          */
 912
 913                         QAT_DP_LOG(DEBUG, "Build %d descriptors for this op",
 914                                         nb_desc_to_build);
 915
 916                         nb_remaining_descriptors -= nb_desc_to_build;
 917                         if (nb_remaining_descriptors >= 0) {
 918                                 /* There are enough remaining descriptors
 919                                  * so retry
 920                                  */
 921                                 int ret2 = qat_comp_build_multiple_requests(
 922                                                 *ops, tmp_qp, tail,
 923                                                 nb_desc_to_build);
 924
 925                                 if (unlikely(ret2 < 1)) {
 926                                         QAT_DP_LOG(DEBUG,
 927                                                         "Failed to build (%d) descriptors, status %d",
 928                                                         nb_desc_to_build, ret2);
 929
 930                                         qat_comp_free_split_op_memzones(cookie,
 931                                                         nb_desc_to_build - 1);
 932
 933                                         tmp_qp->stats.enqueue_err_count++;
 934
 935                                         /* This message cannot be enqueued */
 936                                         if (nb_ops_sent == 0)
 937                                                 return 0;
 938                                         goto kick_tail;
 939                                 } else {
 940                                         descriptors_built = ret2;
 941                                         total_descriptors_built +=
 942                                                         descriptors_built;
 943                                         nb_remaining_descriptors -=
 944                                                         descriptors_built;
 945                                         QAT_DP_LOG(DEBUG,
 946                                                         "Multiple descriptors (%d) built ok",
 947                                                         descriptors_built);
 948                                 }
 949                         } else {
 950                                 QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) "
 951                                                 "exceeds number of available descriptors (%d)",
 952                                                 nb_desc_to_build,
 953                                                 nb_remaining_descriptors +
 954                                                         nb_desc_to_build);
 955
 956                                 qat_comp_free_split_op_memzones(cookie,
 957                                                 nb_desc_to_build - 1);
 958
 959                                 /* Not enough extra descriptors */
 960                                 if (nb_ops_sent == 0)
 961                                         return 0;
 962                                 goto kick_tail;
 963                         }
 964                 } else {
 965                         descriptors_built = 1;
 966                         total_descriptors_built++;
 967                         nb_remaining_descriptors--;
 968                         QAT_DP_LOG(DEBUG, "Single descriptor built ok");
 969                 }
 970
 971                 tail = adf_modulo(tail + (queue->msg_size * descriptors_built),
 972                                   queue->modulo_mask);
 973                 ops++;
 974                 nb_ops_sent++;
 975         }
 976
 977 kick_tail:
 978         queue->tail = tail;
 979         tmp_qp->enqueued += total_descriptors_built;
 980         tmp_qp->stats.enqueued_count += nb_ops_sent;
 981         txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
 982         return nb_ops_sent;
 983 }
 984
 985 uint16_t
 986 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 987 {
 988         struct qat_queue *rx_queue;
 989         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 990         uint32_t head;
 991         uint32_t op_resp_counter = 0, fw_resp_counter = 0;
 992         uint8_t *resp_msg;
 993         int nb_fw_responses;
 994
 995         rx_queue = &(tmp_qp->rx_q);
 996         head = rx_queue->head;
 997         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
 998
 999         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
1000                         op_resp_counter != nb_ops) {
1001
1002                 nb_fw_responses = 1;
1003
1004                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
1005                         qat_sym_process_response(ops, resp_msg,
1006                                 tmp_qp->op_cookies[head >> rx_queue->trailz]);
1007                 else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
1008                         nb_fw_responses = qat_comp_process_response(
1009                                 ops, resp_msg,
1010                                 tmp_qp->op_cookies[head >> rx_queue->trailz],
1011                                 &tmp_qp->stats.dequeue_err_count);
1012 #ifdef BUILD_QAT_ASYM
1013                 else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
1014                         qat_asym_process_response(ops, resp_msg,
1015                                 tmp_qp->op_cookies[head >> rx_queue->trailz]);
1016 #endif
1017
1018                 head = adf_modulo(head + rx_queue->msg_size,
1019                                   rx_queue->modulo_mask);
1020
1021                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
1022
1023                 if (nb_fw_responses) {
1024                         /* only move on to next op if one was ready to return
1025                          * to API
1026                          */
1027                         ops++;
1028                         op_resp_counter++;
1029                 }
1030
1031                  /* A compression op may be broken up into multiple fw requests.
1032                   * Only count fw responses as complete once ALL the responses
1033                   * associated with an op have been processed, as the cookie
1034                   * data from the first response must be available until
1035                   * finished with all firmware responses.
1036                   */
1037                 fw_resp_counter += nb_fw_responses;
1038
1039                 rx_queue->nb_processed_responses++;
1040         }
1041
1042         tmp_qp->dequeued += fw_resp_counter;
1043         tmp_qp->stats.dequeued_count += op_resp_counter;
1044
1045         rx_queue->head = head;
1046         if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
1047                 rxq_free_desc(tmp_qp->qat_dev_gen, tmp_qp, rx_queue);
1048
1049         QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u",
1050                         op_resp_counter, fw_resp_counter);
1051
1052         return op_resp_counter;
1053 }
1054
1055 /* This is almost same as dequeue_op_burst, without the atomic, without stats
1056  * and without the op. Dequeues one response.
1057  */
1058 static uint8_t
1059 qat_cq_dequeue_response(struct qat_qp *qp, void *out_data)
1060 {
1061         uint8_t result = 0;
1062         uint8_t retries = 0;
1063         struct qat_queue *queue = &(qp->rx_q);
1064         struct icp_qat_fw_comn_resp *resp_msg = (struct icp_qat_fw_comn_resp *)
1065                         ((uint8_t *)queue->base_addr + queue->head);
1066
1067         while (retries++ < QAT_CQ_MAX_DEQ_RETRIES &&
1068                         *(uint32_t *)resp_msg == ADF_RING_EMPTY_SIG) {
1069                 /* loop waiting for response until we reach the timeout */
1070                 rte_delay_ms(20);
1071         }
1072
1073         if (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG) {
1074                 /* response received */
1075                 result = 1;
1076
1077                 /* check status flag */
1078                 if (ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
1079                                 resp_msg->comn_hdr.comn_status) ==
1080                                 ICP_QAT_FW_COMN_STATUS_FLAG_OK) {
1081                         /* success */
1082                         memcpy(out_data, resp_msg, queue->msg_size);
1083                 } else {
1084                         memset(out_data, 0, queue->msg_size);
1085                 }
1086
1087                 queue->head = adf_modulo(queue->head + queue->msg_size,
1088                                 queue->modulo_mask);
1089                 rxq_free_desc(qp->qat_dev_gen, qp, queue);
1090         }
1091
1092         return result;
1093 }
1094
1095 /* Sends a NULL message and extracts QAT fw version from the response.
1096  * Used to determine detailed capabilities based on the fw version number.
1097  * This assumes that there are no inflight messages, i.e. assumes there's space
1098  * on the qp, one message is sent and only one response collected.
1099  * Returns fw version number or 0 for unknown version or a negative error code.
1100  */
1101 int
1102 qat_cq_get_fw_version(struct qat_qp *qp)
1103 {
1104         struct qat_queue *queue = &(qp->tx_q);
1105         uint8_t *base_addr = (uint8_t *)queue->base_addr;
1106         struct icp_qat_fw_comn_req null_msg;
1107         struct icp_qat_fw_comn_resp response;
1108
1109         /* prepare the NULL request */
1110         memset(&null_msg, 0, sizeof(null_msg));
1111         null_msg.comn_hdr.hdr_flags =
1112                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
1113         null_msg.comn_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
1114         null_msg.comn_hdr.service_cmd_id = ICP_QAT_FW_NULL_REQ_SERV_ID;
1115
1116 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
1117         QAT_DP_HEXDUMP_LOG(DEBUG, "NULL request", &null_msg, sizeof(null_msg));
1118 #endif
1119
1120         /* send the NULL request */
1121         memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg));
1122         queue->tail = adf_modulo(queue->tail + queue->msg_size,
1123                         queue->modulo_mask);
1124         txq_write_tail(qp->qat_dev_gen, qp, queue);
1125
1126         /* receive a response */
1127         if (qat_cq_dequeue_response(qp, &response)) {
1128
1129 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
1130                 QAT_DP_HEXDUMP_LOG(DEBUG, "NULL response:", &response,
1131                                 sizeof(response));
1132 #endif
1133                 /* if LW0 bit 24 is set - then the fw version was returned */
1134                 if (QAT_FIELD_GET(response.comn_hdr.hdr_flags,
1135                                 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_BITPOS,
1136                                 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_MASK))
1137                         return response.resrvd[0]; /* return LW4 */
1138                 else
1139                         return 0; /* not set - we don't know fw version */
1140         }
1141
1142         QAT_LOG(ERR, "No response received");
1143         return -EINVAL;
1144 }
1145
1146 __rte_weak int
1147 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
1148                           void *op_cookie __rte_unused,
1149                           uint64_t *dequeue_err_count __rte_unused)
1150 {
1151         return  0;
1152 }