drivers/common/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_cycles.h>
   7 #include <rte_dev.h>
   8 #include <rte_malloc.h>
   9 #include <rte_memzone.h>
  10 #include <rte_pci.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_atomic.h>
  13 #include <rte_prefetch.h>
  14
  15 #include "qat_logs.h"
  16 #include "qat_device.h"
  17 #include "qat_qp.h"
  18 #include "qat_sym.h"
  19 #include "qat_asym.h"
  20 #include "qat_comp.h"
  21 #include "adf_transport_access_macros.h"
  22 #include "adf_transport_access_macros_gen4vf.h"
  23
  24 #define QAT_CQ_MAX_DEQ_RETRIES 10
  25
  26 #define ADF_MAX_DESC                            4096
  27 #define ADF_MIN_DESC                            128
  28
  29 #define ADF_ARB_REG_SLOT                        0x1000
  30 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  31
  32 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  33         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  34         (ADF_ARB_REG_SLOT * index), value)
  35
  36 __extension__
  37 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
  38                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  39         /* queue pairs which provide an asymmetric crypto service */
  40         [QAT_SERVICE_ASYMMETRIC] = {
  41                 {
  42                         .service_type = QAT_SERVICE_ASYMMETRIC,
  43                         .hw_bundle_num = 0,
  44                         .tx_ring_num = 0,
  45                         .rx_ring_num = 8,
  46                         .tx_msg_size = 64,
  47                         .rx_msg_size = 32,
  48
  49                 }, {
  50                         .service_type = QAT_SERVICE_ASYMMETRIC,
  51                         .hw_bundle_num = 0,
  52                         .tx_ring_num = 1,
  53                         .rx_ring_num = 9,
  54                         .tx_msg_size = 64,
  55                         .rx_msg_size = 32,
  56                 }
  57         },
  58         /* queue pairs which provide a symmetric crypto service */
  59         [QAT_SERVICE_SYMMETRIC] = {
  60                 {
  61                         .service_type = QAT_SERVICE_SYMMETRIC,
  62                         .hw_bundle_num = 0,
  63                         .tx_ring_num = 2,
  64                         .rx_ring_num = 10,
  65                         .tx_msg_size = 128,
  66                         .rx_msg_size = 32,
  67                 },
  68                 {
  69                         .service_type = QAT_SERVICE_SYMMETRIC,
  70                         .hw_bundle_num = 0,
  71                         .tx_ring_num = 3,
  72                         .rx_ring_num = 11,
  73                         .tx_msg_size = 128,
  74                         .rx_msg_size = 32,
  75                 }
  76         },
  77         /* queue pairs which provide a compression service */
  78         [QAT_SERVICE_COMPRESSION] = {
  79                 {
  80                         .service_type = QAT_SERVICE_COMPRESSION,
  81                         .hw_bundle_num = 0,
  82                         .tx_ring_num = 6,
  83                         .rx_ring_num = 14,
  84                         .tx_msg_size = 128,
  85                         .rx_msg_size = 32,
  86                 }, {
  87                         .service_type = QAT_SERVICE_COMPRESSION,
  88                         .hw_bundle_num = 0,
  89                         .tx_ring_num = 7,
  90                         .rx_ring_num = 15,
  91                         .tx_msg_size = 128,
  92                         .rx_msg_size = 32,
  93                 }
  94         }
  95 };
  96
  97 __extension__
  98 const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
  99                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
 100         /* queue pairs which provide an asymmetric crypto service */
 101         [QAT_SERVICE_ASYMMETRIC] = {
 102                 {
 103                         .service_type = QAT_SERVICE_ASYMMETRIC,
 104                         .hw_bundle_num = 0,
 105                         .tx_ring_num = 0,
 106                         .rx_ring_num = 4,
 107                         .tx_msg_size = 64,
 108                         .rx_msg_size = 32,
 109                 }
 110         },
 111         /* queue pairs which provide a symmetric crypto service */
 112         [QAT_SERVICE_SYMMETRIC] = {
 113                 {
 114                         .service_type = QAT_SERVICE_SYMMETRIC,
 115                         .hw_bundle_num = 0,
 116                         .tx_ring_num = 1,
 117                         .rx_ring_num = 5,
 118                         .tx_msg_size = 128,
 119                         .rx_msg_size = 32,
 120                 }
 121         },
 122         /* queue pairs which provide a compression service */
 123         [QAT_SERVICE_COMPRESSION] = {
 124                 {
 125                         .service_type = QAT_SERVICE_COMPRESSION,
 126                         .hw_bundle_num = 0,
 127                         .tx_ring_num = 3,
 128                         .rx_ring_num = 7,
 129                         .tx_msg_size = 128,
 130                         .rx_msg_size = 32,
 131                 }
 132         }
 133 };
 134
 135 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 136         uint32_t queue_size_bytes);
 137 static void qat_queue_delete(struct qat_queue *queue);
 138 static int qat_queue_create(struct qat_pci_device *qat_dev,
 139         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
 140 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 141         uint32_t *queue_size_for_csr);
 142 static void adf_configure_queues(struct qat_qp *queue,
 143         enum qat_device_gen qat_dev_gen);
 144 static void adf_queue_arb_enable(enum qat_device_gen qat_dev_gen,
 145         struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
 146 static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
 147         struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
 148
 149 int qat_qps_per_service(struct qat_pci_device *qat_dev,
 150                 enum qat_service_type service)
 151 {
 152         int i = 0, count = 0, max_ops_per_srv = 0;
 153
 154         if (qat_dev->qat_dev_gen == QAT_GEN4) {
 155                 max_ops_per_srv = QAT_GEN4_BUNDLE_NUM;
 156                 for (i = 0, count = 0; i < max_ops_per_srv; i++)
 157                         if (qat_dev->qp_gen4_data[i][0].service_type == service)
 158                                 count++;
 159         } else {
 160                 const struct qat_qp_hw_data *sym_hw_qps =
 161                                 qat_gen_config[qat_dev->qat_dev_gen]
 162                                 .qp_hw_data[service];
 163
 164                 max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE;
 165                 for (i = 0, count = 0; i < max_ops_per_srv; i++)
 166                         if (sym_hw_qps[i].service_type == service)
 167                                 count++;
 168         }
 169
 170         return count;
 171 }
 172
 173 static const struct rte_memzone *
 174 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 175                         int socket_id)
 176 {
 177         const struct rte_memzone *mz;
 178
 179         mz = rte_memzone_lookup(queue_name);
 180         if (mz != 0) {
 181                 if (((size_t)queue_size <= mz->len) &&
 182                                 ((socket_id == SOCKET_ID_ANY) ||
 183                                         (socket_id == mz->socket_id))) {
 184                         QAT_LOG(DEBUG, "re-use memzone already "
 185                                         "allocated for %s", queue_name);
 186                         return mz;
 187                 }
 188
 189                 QAT_LOG(ERR, "Incompatible memzone already "
 190                                 "allocated %s, size %u, socket %d. "
 191                                 "Requested size %u, socket %u",
 192                                 queue_name, (uint32_t)mz->len,
 193                                 mz->socket_id, queue_size, socket_id);
 194                 return NULL;
 195         }
 196
 197         QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
 198                                         queue_name, queue_size, socket_id);
 199         return rte_memzone_reserve_aligned(queue_name, queue_size,
 200                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 201 }
 202
 203 int qat_qp_setup(struct qat_pci_device *qat_dev,
 204                 struct qat_qp **qp_addr,
 205                 uint16_t queue_pair_id,
 206                 struct qat_qp_config *qat_qp_conf)
 207 {
 208         struct qat_qp *qp;
 209         struct rte_pci_device *pci_dev =
 210                         qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
 211         char op_cookie_pool_name[RTE_RING_NAMESIZE];
 212         enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
 213         uint32_t i;
 214
 215         QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
 216                 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
 217
 218         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
 219                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
 220                 QAT_LOG(ERR, "Can't create qp for %u descriptors",
 221                                 qat_qp_conf->nb_descriptors);
 222                 return -EINVAL;
 223         }
 224
 225         if (pci_dev->mem_resource[0].addr == NULL) {
 226                 QAT_LOG(ERR, "Could not find VF config space "
 227                                 "(UIO driver attached?).");
 228                 return -EINVAL;
 229         }
 230
 231         /* Allocate the queue pair data structure. */
 232         qp = rte_zmalloc_socket("qat PMD qp metadata",
 233                                 sizeof(*qp), RTE_CACHE_LINE_SIZE,
 234                                 qat_qp_conf->socket_id);
 235         if (qp == NULL) {
 236                 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
 237                 return -ENOMEM;
 238         }
 239         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
 240         qp->op_cookies = rte_zmalloc_socket("qat PMD op cookie pointer",
 241                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 242                         RTE_CACHE_LINE_SIZE, qat_qp_conf->socket_id);
 243         if (qp->op_cookies == NULL) {
 244                 QAT_LOG(ERR, "Failed to alloc mem for cookie");
 245                 rte_free(qp);
 246                 return -ENOMEM;
 247         }
 248
 249         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 250         qp->enqueued = qp->dequeued = 0;
 251
 252         if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
 253                                         ADF_RING_DIR_TX) != 0) {
 254                 QAT_LOG(ERR, "Tx queue create failed "
 255                                 "queue_pair_id=%u", queue_pair_id);
 256                 goto create_err;
 257         }
 258
 259         qp->max_inflights = ADF_MAX_INFLIGHTS(qp->tx_q.queue_size,
 260                                 ADF_BYTES_TO_MSG_SIZE(qp->tx_q.msg_size));
 261
 262         if (qp->max_inflights < 2) {
 263                 QAT_LOG(ERR, "Invalid num inflights");
 264                 qat_queue_delete(&(qp->tx_q));
 265                 goto create_err;
 266         }
 267
 268         if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
 269                                         ADF_RING_DIR_RX) != 0) {
 270                 QAT_LOG(ERR, "Rx queue create failed "
 271                                 "queue_pair_id=%hu", queue_pair_id);
 272                 qat_queue_delete(&(qp->tx_q));
 273                 goto create_err;
 274         }
 275
 276         adf_configure_queues(qp, qat_dev_gen);
 277         adf_queue_arb_enable(qat_dev_gen, &qp->tx_q, qp->mmap_bar_addr,
 278                                         &qat_dev->arb_csr_lock);
 279
 280         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
 281                                         "%s%d_cookies_%s_qp%hu",
 282                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 283                 qat_qp_conf->service_str, queue_pair_id);
 284
 285         QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
 286         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 287         if (qp->op_cookie_pool == NULL)
 288                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 289                                 qp->nb_descriptors,
 290                                 qat_qp_conf->cookie_size, 64, 0,
 291                                 NULL, NULL, NULL, NULL,
 292                                 pci_dev->device.numa_node,
 293                                 0);
 294         if (!qp->op_cookie_pool) {
 295                 QAT_LOG(ERR, "QAT PMD Cannot create"
 296                                 " op mempool");
 297                 goto create_err;
 298         }
 299
 300         for (i = 0; i < qp->nb_descriptors; i++) {
 301                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 302                         QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
 303                         goto create_err;
 304                 }
 305                 memset(qp->op_cookies[i], 0, qat_qp_conf->cookie_size);
 306         }
 307
 308         qp->qat_dev_gen = qat_dev->qat_dev_gen;
 309         qp->service_type = qat_qp_conf->hw->service_type;
 310         qp->qat_dev = qat_dev;
 311
 312         QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
 313                         queue_pair_id, op_cookie_pool_name);
 314
 315         *qp_addr = qp;
 316         return 0;
 317
 318 create_err:
 319         if (qp->op_cookie_pool)
 320                 rte_mempool_free(qp->op_cookie_pool);
 321         rte_free(qp->op_cookies);
 322         rte_free(qp);
 323         return -EFAULT;
 324 }
 325
 326
 327 int qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr)
 328 {
 329         struct qat_qp *qp = *qp_addr;
 330         uint32_t i;
 331
 332         if (qp == NULL) {
 333                 QAT_LOG(DEBUG, "qp already freed");
 334                 return 0;
 335         }
 336
 337         QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
 338                                 qp->qat_dev->qat_dev_id);
 339
 340         /* Don't free memory if there are still responses to be processed */
 341         if ((qp->enqueued - qp->dequeued) == 0) {
 342                 qat_queue_delete(&(qp->tx_q));
 343                 qat_queue_delete(&(qp->rx_q));
 344         } else {
 345                 return -EAGAIN;
 346         }
 347
 348         adf_queue_arb_disable(qat_dev_gen, &(qp->tx_q), qp->mmap_bar_addr,
 349                                 &qp->qat_dev->arb_csr_lock);
 350
 351         for (i = 0; i < qp->nb_descriptors; i++)
 352                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 353
 354         if (qp->op_cookie_pool)
 355                 rte_mempool_free(qp->op_cookie_pool);
 356
 357         rte_free(qp->op_cookies);
 358         rte_free(qp);
 359         *qp_addr = NULL;
 360         return 0;
 361 }
 362
 363
 364 static void qat_queue_delete(struct qat_queue *queue)
 365 {
 366         const struct rte_memzone *mz;
 367         int status = 0;
 368
 369         if (queue == NULL) {
 370                 QAT_LOG(DEBUG, "Invalid queue");
 371                 return;
 372         }
 373         QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
 374                         queue->hw_queue_number, queue->memz_name);
 375
 376         mz = rte_memzone_lookup(queue->memz_name);
 377         if (mz != NULL) {
 378                 /* Write an unused pattern to the queue memory. */
 379                 memset(queue->base_addr, 0x7F, queue->queue_size);
 380                 status = rte_memzone_free(mz);
 381                 if (status != 0)
 382                         QAT_LOG(ERR, "Error %d on freeing queue %s",
 383                                         status, queue->memz_name);
 384         } else {
 385                 QAT_LOG(DEBUG, "queue %s doesn't exist",
 386                                 queue->memz_name);
 387         }
 388 }
 389
 390 static int
 391 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 392                 struct qat_qp_config *qp_conf, uint8_t dir)
 393 {
 394         uint64_t queue_base;
 395         void *io_addr;
 396         const struct rte_memzone *qp_mz;
 397         struct rte_pci_device *pci_dev =
 398                         qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
 399         enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
 400         int ret = 0;
 401         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
 402                         qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
 403         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
 404
 405         queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
 406         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
 407                         qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
 408
 409         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 410                 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
 411                 return -EINVAL;
 412         }
 413
 414         /*
 415          * Allocate a memzone for the queue - create a unique name.
 416          */
 417         snprintf(queue->memz_name, sizeof(queue->memz_name),
 418                         "%s_%d_%s_%s_%d_%d",
 419                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 420                 qp_conf->service_str, "qp_mem",
 421                 queue->hw_bundle_number, queue->hw_queue_number);
 422         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 423                         pci_dev->device.numa_node);
 424         if (qp_mz == NULL) {
 425                 QAT_LOG(ERR, "Failed to allocate ring memzone");
 426                 return -ENOMEM;
 427         }
 428
 429         queue->base_addr = (char *)qp_mz->addr;
 430         queue->base_phys_addr = qp_mz->iova;
 431         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 432                         queue_size_bytes)) {
 433                 QAT_LOG(ERR, "Invalid alignment on queue create "
 434                                         " 0x%"PRIx64"\n",
 435                                         queue->base_phys_addr);
 436                 ret = -EFAULT;
 437                 goto queue_create_err;
 438         }
 439
 440         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
 441                         &(queue->queue_size)) != 0) {
 442                 QAT_LOG(ERR, "Invalid num inflights");
 443                 ret = -EINVAL;
 444                 goto queue_create_err;
 445         }
 446
 447         queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
 448         queue->head = 0;
 449         queue->tail = 0;
 450         queue->msg_size = desc_size;
 451
 452         /* For fast calculation of cookie index, relies on msg_size being 2^n */
 453         queue->trailz = __builtin_ctz(desc_size);
 454
 455         /*
 456          * Write an unused pattern to the queue memory.
 457          */
 458         memset(queue->base_addr, 0x7F, queue_size_bytes);
 459         io_addr = pci_dev->mem_resource[0].addr;
 460
 461         if (qat_dev_gen == QAT_GEN4) {
 462                 queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr,
 463                                         queue->queue_size);
 464                 WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number,
 465                         queue->hw_queue_number, queue_base);
 466         } else {
 467                 queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 468                                 queue->queue_size);
 469                 WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 470                         queue->hw_queue_number, queue_base);
 471         }
 472
 473         QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
 474                 " nb msgs %u, msg_size %u, modulo mask %u",
 475                         queue->memz_name,
 476                         queue->queue_size, queue_size_bytes,
 477                         qp_conf->nb_descriptors, desc_size,
 478                         queue->modulo_mask);
 479
 480         return 0;
 481
 482 queue_create_err:
 483         rte_memzone_free(qp_mz);
 484         return ret;
 485 }
 486
 487 int
 488 qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id,
 489                         enum qat_service_type service_type)
 490 {
 491         if (qat_dev->qat_dev_gen == QAT_GEN4) {
 492                 int i = 0, valid_qps = 0;
 493
 494                 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
 495                         if (qat_dev->qp_gen4_data[i][0].service_type ==
 496                                 service_type) {
 497                                 if (valid_qps == qp_id)
 498                                         return i;
 499                                 ++valid_qps;
 500                         }
 501                 }
 502         }
 503         return -1;
 504 }
 505
 506 int
 507 qat_read_qp_config(struct qat_pci_device *qat_dev)
 508 {
 509         int i = 0;
 510         enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
 511
 512         if (qat_dev_gen == QAT_GEN4) {
 513                 uint16_t svc = 0;
 514
 515                 if (qat_query_svc(qat_dev, (uint8_t *)&svc))
 516                         return -(EFAULT);
 517                 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
 518                         struct qat_qp_hw_data *hw_data =
 519                                 &qat_dev->qp_gen4_data[i][0];
 520                         uint8_t svc1 = (svc >> (3 * i)) & 0x7;
 521                         enum qat_service_type service_type = QAT_SERVICE_INVALID;
 522
 523                         if (svc1 == QAT_SVC_SYM) {
 524                                 service_type = QAT_SERVICE_SYMMETRIC;
 525                                 QAT_LOG(DEBUG,
 526                                         "Discovered SYMMETRIC service on bundle %d",
 527                                         i);
 528                         } else if (svc1 == QAT_SVC_COMPRESSION) {
 529                                 service_type = QAT_SERVICE_COMPRESSION;
 530                                 QAT_LOG(DEBUG,
 531                                         "Discovered COPRESSION service on bundle %d",
 532                                         i);
 533                         } else if (svc1 == QAT_SVC_ASYM) {
 534                                 service_type = QAT_SERVICE_ASYMMETRIC;
 535                                 QAT_LOG(DEBUG,
 536                                         "Discovered ASYMMETRIC service on bundle %d",
 537                                         i);
 538                         } else {
 539                                 QAT_LOG(ERR,
 540                                         "Unrecognized service on bundle %d",
 541                                         i);
 542                                 return -(EFAULT);
 543                         }
 544
 545                         memset(hw_data, 0, sizeof(*hw_data));
 546                         hw_data->service_type = service_type;
 547                         if (service_type == QAT_SERVICE_ASYMMETRIC) {
 548                                 hw_data->tx_msg_size = 64;
 549                                 hw_data->rx_msg_size = 32;
 550                         } else if (service_type == QAT_SERVICE_SYMMETRIC ||
 551                                         service_type ==
 552                                                 QAT_SERVICE_COMPRESSION) {
 553                                 hw_data->tx_msg_size = 128;
 554                                 hw_data->rx_msg_size = 32;
 555                         }
 556                         hw_data->tx_ring_num = 0;
 557                         hw_data->rx_ring_num = 1;
 558                         hw_data->hw_bundle_num = i;
 559                 }
 560                 return 0;
 561         }
 562         return -(EINVAL);
 563 }
 564
 565 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 566                                         uint32_t queue_size_bytes)
 567 {
 568         if (((queue_size_bytes - 1) & phys_addr) != 0)
 569                 return -EINVAL;
 570         return 0;
 571 }
 572
 573 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 574         uint32_t *p_queue_size_for_csr)
 575 {
 576         uint8_t i = ADF_MIN_RING_SIZE;
 577
 578         for (; i <= ADF_MAX_RING_SIZE; i++)
 579                 if ((msg_size * msg_num) ==
 580                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 581                         *p_queue_size_for_csr = i;
 582                         return 0;
 583                 }
 584         QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 585         return -EINVAL;
 586 }
 587
 588 static void
 589 adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq,
 590                         void *base_addr, rte_spinlock_t *lock)
 591 {
 592         uint32_t arb_csr_offset = 0, value;
 593
 594         rte_spinlock_lock(lock);
 595         if (qat_dev_gen == QAT_GEN4) {
 596                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 597                                 (ADF_RING_BUNDLE_SIZE_GEN4 *
 598                                 txq->hw_bundle_number);
 599                 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
 600                                 arb_csr_offset);
 601         } else {
 602                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 603                                 (ADF_ARB_REG_SLOT *
 604                                 txq->hw_bundle_number);
 605                 value = ADF_CSR_RD(base_addr,
 606                                 arb_csr_offset);
 607         }
 608         value |= (0x01 << txq->hw_queue_number);
 609         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 610         rte_spinlock_unlock(lock);
 611 }
 612
 613 static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
 614                 struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock)
 615 {
 616         uint32_t arb_csr_offset = 0, value;
 617
 618         rte_spinlock_lock(lock);
 619         if (qat_dev_gen == QAT_GEN4) {
 620                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 621                                 (ADF_RING_BUNDLE_SIZE_GEN4 *
 622                                 txq->hw_bundle_number);
 623                 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
 624                                 arb_csr_offset);
 625         } else {
 626                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 627                                 (ADF_ARB_REG_SLOT *
 628                                 txq->hw_bundle_number);
 629                 value = ADF_CSR_RD(base_addr,
 630                                 arb_csr_offset);
 631         }
 632         value &= ~(0x01 << txq->hw_queue_number);
 633         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 634         rte_spinlock_unlock(lock);
 635 }
 636
 637 static void adf_configure_queues(struct qat_qp *qp,
 638                 enum qat_device_gen qat_dev_gen)
 639 {
 640         uint32_t q_tx_config, q_resp_config;
 641         struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q;
 642
 643         q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size);
 644         q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size,
 645                         ADF_RING_NEAR_WATERMARK_512,
 646                         ADF_RING_NEAR_WATERMARK_0);
 647
 648         if (qat_dev_gen == QAT_GEN4) {
 649                 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
 650                         q_tx->hw_bundle_number, q_tx->hw_queue_number,
 651                         q_tx_config);
 652                 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
 653                         q_rx->hw_bundle_number, q_rx->hw_queue_number,
 654                         q_resp_config);
 655         } else {
 656                 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
 657                         q_tx->hw_bundle_number, q_tx->hw_queue_number,
 658                         q_tx_config);
 659                 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
 660                         q_rx->hw_bundle_number, q_rx->hw_queue_number,
 661                         q_resp_config);
 662         }
 663 }
 664
 665 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
 666 {
 667         return data & modulo_mask;
 668 }
 669
 670 static inline void
 671 txq_write_tail(enum qat_device_gen qat_dev_gen,
 672                 struct qat_qp *qp, struct qat_queue *q) {
 673
 674         if (qat_dev_gen == QAT_GEN4) {
 675                 WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr,
 676                         q->hw_bundle_number, q->hw_queue_number, q->tail);
 677         } else {
 678                 WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 679                         q->hw_queue_number, q->tail);
 680         }
 681 }
 682
 683 static inline
 684 void rxq_free_desc(enum qat_device_gen qat_dev_gen, struct qat_qp *qp,
 685                                 struct qat_queue *q)
 686 {
 687         uint32_t old_head, new_head;
 688         uint32_t max_head;
 689
 690         old_head = q->csr_head;
 691         new_head = q->head;
 692         max_head = qp->nb_descriptors * q->msg_size;
 693
 694         /* write out free descriptors */
 695         void *cur_desc = (uint8_t *)q->base_addr + old_head;
 696
 697         if (new_head < old_head) {
 698                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
 699                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
 700         } else {
 701                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
 702         }
 703         q->nb_processed_responses = 0;
 704         q->csr_head = new_head;
 705
 706         /* write current head to CSR */
 707         if (qat_dev_gen == QAT_GEN4) {
 708                 WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr,
 709                         q->hw_bundle_number, q->hw_queue_number, new_head);
 710         } else {
 711                 WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
 712                                 q->hw_queue_number, new_head);
 713         }
 714
 715 }
 716
 717 uint16_t
 718 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 719 {
 720         register struct qat_queue *queue;
 721         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 722         register uint32_t nb_ops_sent = 0;
 723         register int ret = -1;
 724         uint16_t nb_ops_possible = nb_ops;
 725         register uint8_t *base_addr;
 726         register uint32_t tail;
 727
 728         if (unlikely(nb_ops == 0))
 729                 return 0;
 730
 731         /* read params used a lot in main loop into registers */
 732         queue = &(tmp_qp->tx_q);
 733         base_addr = (uint8_t *)queue->base_addr;
 734         tail = queue->tail;
 735
 736         /* Find how many can actually fit on the ring */
 737         {
 738                 /* dequeued can only be written by one thread, but it may not
 739                  * be this thread. As it's 4-byte aligned it will be read
 740                  * atomically here by any Intel CPU.
 741                  * enqueued can wrap before dequeued, but cannot
 742                  * lap it as var size of enq/deq (uint32_t) > var size of
 743                  * max_inflights (uint16_t). In reality inflights is never
 744                  * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
 745                  * On wrapping, the calculation still returns the correct
 746                  * positive value as all three vars are unsigned.
 747                  */
 748                 uint32_t inflights =
 749                         tmp_qp->enqueued - tmp_qp->dequeued;
 750
 751                 if ((inflights + nb_ops) > tmp_qp->max_inflights) {
 752                         nb_ops_possible = tmp_qp->max_inflights - inflights;
 753                         if (nb_ops_possible == 0)
 754                                 return 0;
 755                 }
 756                 /* QAT has plenty of work queued already, so don't waste cycles
 757                  * enqueueing, wait til the application has gathered a bigger
 758                  * burst or some completed ops have been dequeued
 759                  */
 760                 if (tmp_qp->min_enq_burst_threshold && inflights >
 761                                 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
 762                                 tmp_qp->min_enq_burst_threshold) {
 763                         tmp_qp->stats.threshold_hit_count++;
 764                         return 0;
 765                 }
 766         }
 767
 768 #ifdef BUILD_QAT_SYM
 769         if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 770                 qat_sym_preprocess_requests(ops, nb_ops_possible);
 771 #endif
 772
 773         while (nb_ops_sent != nb_ops_possible) {
 774                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
 775 #ifdef BUILD_QAT_SYM
 776                         ret = qat_sym_build_request(*ops, base_addr + tail,
 777                                 tmp_qp->op_cookies[tail >> queue->trailz],
 778                                 tmp_qp->qat_dev_gen);
 779 #endif
 780                 } else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
 781                         ret = qat_comp_build_request(*ops, base_addr + tail,
 782                                 tmp_qp->op_cookies[tail >> queue->trailz],
 783                                 tmp_qp->qat_dev_gen);
 784                 } else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
 785 #ifdef BUILD_QAT_ASYM
 786                         ret = qat_asym_build_request(*ops, base_addr + tail,
 787                                 tmp_qp->op_cookies[tail >> queue->trailz],
 788                                 tmp_qp->qat_dev_gen);
 789 #endif
 790                 }
 791                 if (ret != 0) {
 792                         tmp_qp->stats.enqueue_err_count++;
 793                         /* This message cannot be enqueued */
 794                         if (nb_ops_sent == 0)
 795                                 return 0;
 796                         goto kick_tail;
 797                 }
 798
 799                 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
 800                 ops++;
 801                 nb_ops_sent++;
 802         }
 803 kick_tail:
 804         queue->tail = tail;
 805         tmp_qp->enqueued += nb_ops_sent;
 806         tmp_qp->stats.enqueued_count += nb_ops_sent;
 807         txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
 808         return nb_ops_sent;
 809 }
 810
 811 /* Use this for compression only - but keep consistent with above common
 812  * function as much as possible.
 813  */
 814 uint16_t
 815 qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 816 {
 817         register struct qat_queue *queue;
 818         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 819         register uint32_t nb_ops_sent = 0;
 820         register int nb_desc_to_build;
 821         uint16_t nb_ops_possible = nb_ops;
 822         register uint8_t *base_addr;
 823         register uint32_t tail;
 824
 825         int descriptors_built, total_descriptors_built = 0;
 826         int nb_remaining_descriptors;
 827         int overflow = 0;
 828
 829         if (unlikely(nb_ops == 0))
 830                 return 0;
 831
 832         /* read params used a lot in main loop into registers */
 833         queue = &(tmp_qp->tx_q);
 834         base_addr = (uint8_t *)queue->base_addr;
 835         tail = queue->tail;
 836
 837         /* Find how many can actually fit on the ring */
 838         {
 839                 /* dequeued can only be written by one thread, but it may not
 840                  * be this thread. As it's 4-byte aligned it will be read
 841                  * atomically here by any Intel CPU.
 842                  * enqueued can wrap before dequeued, but cannot
 843                  * lap it as var size of enq/deq (uint32_t) > var size of
 844                  * max_inflights (uint16_t). In reality inflights is never
 845                  * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
 846                  * On wrapping, the calculation still returns the correct
 847                  * positive value as all three vars are unsigned.
 848                  */
 849                 uint32_t inflights =
 850                         tmp_qp->enqueued - tmp_qp->dequeued;
 851
 852                 /* Find how many can actually fit on the ring */
 853                 overflow = (inflights + nb_ops) - tmp_qp->max_inflights;
 854                 if (overflow > 0) {
 855                         nb_ops_possible = nb_ops - overflow;
 856                         if (nb_ops_possible == 0)
 857                                 return 0;
 858                 }
 859
 860                 /* QAT has plenty of work queued already, so don't waste cycles
 861                  * enqueueing, wait til the application has gathered a bigger
 862                  * burst or some completed ops have been dequeued
 863                  */
 864                 if (tmp_qp->min_enq_burst_threshold && inflights >
 865                                 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
 866                                 tmp_qp->min_enq_burst_threshold) {
 867                         tmp_qp->stats.threshold_hit_count++;
 868                         return 0;
 869                 }
 870         }
 871
 872         /* At this point nb_ops_possible is assuming a 1:1 mapping
 873          * between ops and descriptors.
 874          * Fewer may be sent if some ops have to be split.
 875          * nb_ops_possible is <= burst size.
 876          * Find out how many spaces are actually available on the qp in case
 877          * more are needed.
 878          */
 879         nb_remaining_descriptors = nb_ops_possible
 880                          + ((overflow >= 0) ? 0 : overflow * (-1));
 881         QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d",
 882                         nb_ops, nb_remaining_descriptors);
 883
 884         while (nb_ops_sent != nb_ops_possible &&
 885                                 nb_remaining_descriptors > 0) {
 886                 struct qat_comp_op_cookie *cookie =
 887                                 tmp_qp->op_cookies[tail >> queue->trailz];
 888
 889                 descriptors_built = 0;
 890
 891                 QAT_DP_LOG(DEBUG, "--- data length: %u",
 892                            ((struct rte_comp_op *)*ops)->src.length);
 893
 894                 nb_desc_to_build = qat_comp_build_request(*ops,
 895                                 base_addr + tail, cookie, tmp_qp->qat_dev_gen);
 896                 QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, "
 897                         "%d ops sent, %d descriptors needed",
 898                         total_descriptors_built, nb_remaining_descriptors,
 899                         nb_ops_sent, nb_desc_to_build);
 900
 901                 if (unlikely(nb_desc_to_build < 0)) {
 902                         /* this message cannot be enqueued */
 903                         tmp_qp->stats.enqueue_err_count++;
 904                         if (nb_ops_sent == 0)
 905                                 return 0;
 906                         goto kick_tail;
 907                 } else if (unlikely(nb_desc_to_build > 1)) {
 908                         /* this op is too big and must be split - get more
 909                          * descriptors and retry
 910                          */
 911
 912                         QAT_DP_LOG(DEBUG, "Build %d descriptors for this op",
 913                                         nb_desc_to_build);
 914
 915                         nb_remaining_descriptors -= nb_desc_to_build;
 916                         if (nb_remaining_descriptors >= 0) {
 917                                 /* There are enough remaining descriptors
 918                                  * so retry
 919                                  */
 920                                 int ret2 = qat_comp_build_multiple_requests(
 921                                                 *ops, tmp_qp, tail,
 922                                                 nb_desc_to_build);
 923
 924                                 if (unlikely(ret2 < 1)) {
 925                                         QAT_DP_LOG(DEBUG,
 926                                                         "Failed to build (%d) descriptors, status %d",
 927                                                         nb_desc_to_build, ret2);
 928
 929                                         qat_comp_free_split_op_memzones(cookie,
 930                                                         nb_desc_to_build - 1);
 931
 932                                         tmp_qp->stats.enqueue_err_count++;
 933
 934                                         /* This message cannot be enqueued */
 935                                         if (nb_ops_sent == 0)
 936                                                 return 0;
 937                                         goto kick_tail;
 938                                 } else {
 939                                         descriptors_built = ret2;
 940                                         total_descriptors_built +=
 941                                                         descriptors_built;
 942                                         nb_remaining_descriptors -=
 943                                                         descriptors_built;
 944                                         QAT_DP_LOG(DEBUG,
 945                                                         "Multiple descriptors (%d) built ok",
 946                                                         descriptors_built);
 947                                 }
 948                         } else {
 949                                 QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) "
 950                                                 "exceeds number of available descriptors (%d)",
 951                                                 nb_desc_to_build,
 952                                                 nb_remaining_descriptors +
 953                                                         nb_desc_to_build);
 954
 955                                 qat_comp_free_split_op_memzones(cookie,
 956                                                 nb_desc_to_build - 1);
 957
 958                                 /* Not enough extra descriptors */
 959                                 if (nb_ops_sent == 0)
 960                                         return 0;
 961                                 goto kick_tail;
 962                         }
 963                 } else {
 964                         descriptors_built = 1;
 965                         total_descriptors_built++;
 966                         nb_remaining_descriptors--;
 967                         QAT_DP_LOG(DEBUG, "Single descriptor built ok");
 968                 }
 969
 970                 tail = adf_modulo(tail + (queue->msg_size * descriptors_built),
 971                                   queue->modulo_mask);
 972                 ops++;
 973                 nb_ops_sent++;
 974         }
 975
 976 kick_tail:
 977         queue->tail = tail;
 978         tmp_qp->enqueued += total_descriptors_built;
 979         tmp_qp->stats.enqueued_count += nb_ops_sent;
 980         txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
 981         return nb_ops_sent;
 982 }
 983
 984 uint16_t
 985 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 986 {
 987         struct qat_queue *rx_queue;
 988         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 989         uint32_t head;
 990         uint32_t op_resp_counter = 0, fw_resp_counter = 0;
 991         uint8_t *resp_msg;
 992         int nb_fw_responses;
 993
 994         rx_queue = &(tmp_qp->rx_q);
 995         head = rx_queue->head;
 996         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
 997
 998         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 999                         op_resp_counter != nb_ops) {
1000
1001                 nb_fw_responses = 1;
1002
1003                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
1004                         qat_sym_process_response(ops, resp_msg,
1005                                 tmp_qp->op_cookies[head >> rx_queue->trailz]);
1006                 else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
1007                         nb_fw_responses = qat_comp_process_response(
1008                                 ops, resp_msg,
1009                                 tmp_qp->op_cookies[head >> rx_queue->trailz],
1010                                 &tmp_qp->stats.dequeue_err_count);
1011 #ifdef BUILD_QAT_ASYM
1012                 else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
1013                         qat_asym_process_response(ops, resp_msg,
1014                                 tmp_qp->op_cookies[head >> rx_queue->trailz]);
1015 #endif
1016
1017                 head = adf_modulo(head + rx_queue->msg_size,
1018                                   rx_queue->modulo_mask);
1019
1020                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
1021
1022                 if (nb_fw_responses) {
1023                         /* only move on to next op if one was ready to return
1024                          * to API
1025                          */
1026                         ops++;
1027                         op_resp_counter++;
1028                 }
1029
1030                  /* A compression op may be broken up into multiple fw requests.
1031                   * Only count fw responses as complete once ALL the responses
1032                   * associated with an op have been processed, as the cookie
1033                   * data from the first response must be available until
1034                   * finished with all firmware responses.
1035                   */
1036                 fw_resp_counter += nb_fw_responses;
1037
1038                 rx_queue->nb_processed_responses++;
1039         }
1040
1041         tmp_qp->dequeued += fw_resp_counter;
1042         tmp_qp->stats.dequeued_count += op_resp_counter;
1043
1044         rx_queue->head = head;
1045         if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
1046                 rxq_free_desc(tmp_qp->qat_dev_gen, tmp_qp, rx_queue);
1047
1048         QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u",
1049                         op_resp_counter, fw_resp_counter);
1050
1051         return op_resp_counter;
1052 }
1053
1054 /* This is almost same as dequeue_op_burst, without the atomic, without stats
1055  * and without the op. Dequeues one response.
1056  */
1057 static uint8_t
1058 qat_cq_dequeue_response(struct qat_qp *qp, void *out_data)
1059 {
1060         uint8_t result = 0;
1061         uint8_t retries = 0;
1062         struct qat_queue *queue = &(qp->rx_q);
1063         struct icp_qat_fw_comn_resp *resp_msg = (struct icp_qat_fw_comn_resp *)
1064                         ((uint8_t *)queue->base_addr + queue->head);
1065
1066         while (retries++ < QAT_CQ_MAX_DEQ_RETRIES &&
1067                         *(uint32_t *)resp_msg == ADF_RING_EMPTY_SIG) {
1068                 /* loop waiting for response until we reach the timeout */
1069                 rte_delay_ms(20);
1070         }
1071
1072         if (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG) {
1073                 /* response received */
1074                 result = 1;
1075
1076                 /* check status flag */
1077                 if (ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
1078                                 resp_msg->comn_hdr.comn_status) ==
1079                                 ICP_QAT_FW_COMN_STATUS_FLAG_OK) {
1080                         /* success */
1081                         memcpy(out_data, resp_msg, queue->msg_size);
1082                 } else {
1083                         memset(out_data, 0, queue->msg_size);
1084                 }
1085
1086                 queue->head = adf_modulo(queue->head + queue->msg_size,
1087                                 queue->modulo_mask);
1088                 rxq_free_desc(qp->qat_dev_gen, qp, queue);
1089         }
1090
1091         return result;
1092 }
1093
1094 /* Sends a NULL message and extracts QAT fw version from the response.
1095  * Used to determine detailed capabilities based on the fw version number.
1096  * This assumes that there are no inflight messages, i.e. assumes there's space
1097  * on the qp, one message is sent and only one response collected.
1098  * Returns fw version number or 0 for unknown version or a negative error code.
1099  */
1100 int
1101 qat_cq_get_fw_version(struct qat_qp *qp)
1102 {
1103         struct qat_queue *queue = &(qp->tx_q);
1104         uint8_t *base_addr = (uint8_t *)queue->base_addr;
1105         struct icp_qat_fw_comn_req null_msg;
1106         struct icp_qat_fw_comn_resp response;
1107
1108         /* prepare the NULL request */
1109         memset(&null_msg, 0, sizeof(null_msg));
1110         null_msg.comn_hdr.hdr_flags =
1111                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
1112         null_msg.comn_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
1113         null_msg.comn_hdr.service_cmd_id = ICP_QAT_FW_NULL_REQ_SERV_ID;
1114
1115 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
1116         QAT_DP_HEXDUMP_LOG(DEBUG, "NULL request", &null_msg, sizeof(null_msg));
1117 #endif
1118
1119         /* send the NULL request */
1120         memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg));
1121         queue->tail = adf_modulo(queue->tail + queue->msg_size,
1122                         queue->modulo_mask);
1123         txq_write_tail(qp->qat_dev_gen, qp, queue);
1124
1125         /* receive a response */
1126         if (qat_cq_dequeue_response(qp, &response)) {
1127
1128 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
1129                 QAT_DP_HEXDUMP_LOG(DEBUG, "NULL response:", &response,
1130                                 sizeof(response));
1131 #endif
1132                 /* if LW0 bit 24 is set - then the fw version was returned */
1133                 if (QAT_FIELD_GET(response.comn_hdr.hdr_flags,
1134                                 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_BITPOS,
1135                                 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_MASK))
1136                         return response.resrvd[0]; /* return LW4 */
1137                 else
1138                         return 0; /* not set - we don't know fw version */
1139         }
1140
1141         QAT_LOG(ERR, "No response received");
1142         return -EINVAL;
1143 }
1144
1145 __rte_weak int
1146 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
1147                           void *op_cookie __rte_unused,
1148                           uint64_t *dequeue_err_count __rte_unused)
1149 {
1150         return  0;
1151 }