drivers/common/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_cycles.h>
   7 #include <rte_dev.h>
   8 #include <rte_malloc.h>
   9 #include <rte_memzone.h>
  10 #include <rte_pci.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_atomic.h>
  13 #include <rte_prefetch.h>
  14
  15 #include "qat_logs.h"
  16 #include "qat_device.h"
  17 #include "qat_qp.h"
  18 #include "qat_sym.h"
  19 #include "qat_asym.h"
  20 #include "qat_comp.h"
  21 #include "adf_transport_access_macros.h"
  22 #include "adf_transport_access_macros_gen4vf.h"
  23 #include "dev/qat_dev_gens.h"
  24
  25 #define QAT_CQ_MAX_DEQ_RETRIES 10
  26
  27 #define ADF_MAX_DESC                            4096
  28 #define ADF_MIN_DESC                            128
  29
  30 #define ADF_ARB_REG_SLOT                        0x1000
  31 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  32
  33 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  34         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  35         (ADF_ARB_REG_SLOT * index), value)
  36
  37 struct qat_qp_hw_spec_funcs*
  38         qat_qp_hw_spec[QAT_N_GENS];
  39
  40 __extension__
  41 const struct qat_qp_hw_data qat_gen1_qps[QAT_MAX_SERVICES]
  42                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
  43         /* queue pairs which provide an asymmetric crypto service */
  44         [QAT_SERVICE_ASYMMETRIC] = {
  45                 {
  46                         .service_type = QAT_SERVICE_ASYMMETRIC,
  47                         .hw_bundle_num = 0,
  48                         .tx_ring_num = 0,
  49                         .rx_ring_num = 8,
  50                         .tx_msg_size = 64,
  51                         .rx_msg_size = 32,
  52
  53                 }, {
  54                         .service_type = QAT_SERVICE_ASYMMETRIC,
  55                         .hw_bundle_num = 0,
  56                         .tx_ring_num = 1,
  57                         .rx_ring_num = 9,
  58                         .tx_msg_size = 64,
  59                         .rx_msg_size = 32,
  60                 }
  61         },
  62         /* queue pairs which provide a symmetric crypto service */
  63         [QAT_SERVICE_SYMMETRIC] = {
  64                 {
  65                         .service_type = QAT_SERVICE_SYMMETRIC,
  66                         .hw_bundle_num = 0,
  67                         .tx_ring_num = 2,
  68                         .rx_ring_num = 10,
  69                         .tx_msg_size = 128,
  70                         .rx_msg_size = 32,
  71                 },
  72                 {
  73                         .service_type = QAT_SERVICE_SYMMETRIC,
  74                         .hw_bundle_num = 0,
  75                         .tx_ring_num = 3,
  76                         .rx_ring_num = 11,
  77                         .tx_msg_size = 128,
  78                         .rx_msg_size = 32,
  79                 }
  80         },
  81         /* queue pairs which provide a compression service */
  82         [QAT_SERVICE_COMPRESSION] = {
  83                 {
  84                         .service_type = QAT_SERVICE_COMPRESSION,
  85                         .hw_bundle_num = 0,
  86                         .tx_ring_num = 6,
  87                         .rx_ring_num = 14,
  88                         .tx_msg_size = 128,
  89                         .rx_msg_size = 32,
  90                 }, {
  91                         .service_type = QAT_SERVICE_COMPRESSION,
  92                         .hw_bundle_num = 0,
  93                         .tx_ring_num = 7,
  94                         .rx_ring_num = 15,
  95                         .tx_msg_size = 128,
  96                         .rx_msg_size = 32,
  97                 }
  98         }
  99 };
 100
 101 __extension__
 102 const struct qat_qp_hw_data qat_gen3_qps[QAT_MAX_SERVICES]
 103                                          [ADF_MAX_QPS_ON_ANY_SERVICE] = {
 104         /* queue pairs which provide an asymmetric crypto service */
 105         [QAT_SERVICE_ASYMMETRIC] = {
 106                 {
 107                         .service_type = QAT_SERVICE_ASYMMETRIC,
 108                         .hw_bundle_num = 0,
 109                         .tx_ring_num = 0,
 110                         .rx_ring_num = 4,
 111                         .tx_msg_size = 64,
 112                         .rx_msg_size = 32,
 113                 }
 114         },
 115         /* queue pairs which provide a symmetric crypto service */
 116         [QAT_SERVICE_SYMMETRIC] = {
 117                 {
 118                         .service_type = QAT_SERVICE_SYMMETRIC,
 119                         .hw_bundle_num = 0,
 120                         .tx_ring_num = 1,
 121                         .rx_ring_num = 5,
 122                         .tx_msg_size = 128,
 123                         .rx_msg_size = 32,
 124                 }
 125         },
 126         /* queue pairs which provide a compression service */
 127         [QAT_SERVICE_COMPRESSION] = {
 128                 {
 129                         .service_type = QAT_SERVICE_COMPRESSION,
 130                         .hw_bundle_num = 0,
 131                         .tx_ring_num = 3,
 132                         .rx_ring_num = 7,
 133                         .tx_msg_size = 128,
 134                         .rx_msg_size = 32,
 135                 }
 136         }
 137 };
 138
 139 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 140         uint32_t queue_size_bytes);
 141 static void qat_queue_delete(struct qat_queue *queue);
 142 static int qat_queue_create(struct qat_pci_device *qat_dev,
 143         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
 144 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 145         uint32_t *queue_size_for_csr);
 146 static void adf_configure_queues(struct qat_qp *queue,
 147         enum qat_device_gen qat_dev_gen);
 148 static void adf_queue_arb_enable(enum qat_device_gen qat_dev_gen,
 149         struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
 150 static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
 151         struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock);
 152
 153 int qat_qps_per_service(struct qat_pci_device *qat_dev,
 154                 enum qat_service_type service)
 155 {
 156         int i = 0, count = 0, max_ops_per_srv = 0;
 157
 158         if (qat_dev->qat_dev_gen == QAT_GEN4) {
 159                 max_ops_per_srv = QAT_GEN4_BUNDLE_NUM;
 160                 for (i = 0, count = 0; i < max_ops_per_srv; i++)
 161                         if (qat_dev->qp_gen4_data[i][0].service_type == service)
 162                                 count++;
 163         } else {
 164                 const struct qat_qp_hw_data *sym_hw_qps =
 165                                 qat_gen_config[qat_dev->qat_dev_gen]
 166                                 .qp_hw_data[service];
 167
 168                 max_ops_per_srv = ADF_MAX_QPS_ON_ANY_SERVICE;
 169                 for (i = 0, count = 0; i < max_ops_per_srv; i++)
 170                         if (sym_hw_qps[i].service_type == service)
 171                                 count++;
 172         }
 173
 174         return count;
 175 }
 176
 177 static const struct rte_memzone *
 178 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 179                         int socket_id)
 180 {
 181         const struct rte_memzone *mz;
 182
 183         mz = rte_memzone_lookup(queue_name);
 184         if (mz != 0) {
 185                 if (((size_t)queue_size <= mz->len) &&
 186                                 ((socket_id == SOCKET_ID_ANY) ||
 187                                         (socket_id == mz->socket_id))) {
 188                         QAT_LOG(DEBUG, "re-use memzone already "
 189                                         "allocated for %s", queue_name);
 190                         return mz;
 191                 }
 192
 193                 QAT_LOG(ERR, "Incompatible memzone already "
 194                                 "allocated %s, size %u, socket %d. "
 195                                 "Requested size %u, socket %u",
 196                                 queue_name, (uint32_t)mz->len,
 197                                 mz->socket_id, queue_size, socket_id);
 198                 return NULL;
 199         }
 200
 201         QAT_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
 202                                         queue_name, queue_size, socket_id);
 203         return rte_memzone_reserve_aligned(queue_name, queue_size,
 204                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 205 }
 206
 207 int qat_qp_setup(struct qat_pci_device *qat_dev,
 208                 struct qat_qp **qp_addr,
 209                 uint16_t queue_pair_id,
 210                 struct qat_qp_config *qat_qp_conf)
 211 {
 212         struct qat_qp *qp;
 213         struct rte_pci_device *pci_dev =
 214                         qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
 215         char op_cookie_pool_name[RTE_RING_NAMESIZE];
 216         enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
 217         uint32_t i;
 218
 219         QAT_LOG(DEBUG, "Setup qp %u on qat pci device %d gen %d",
 220                 queue_pair_id, qat_dev->qat_dev_id, qat_dev->qat_dev_gen);
 221
 222         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
 223                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
 224                 QAT_LOG(ERR, "Can't create qp for %u descriptors",
 225                                 qat_qp_conf->nb_descriptors);
 226                 return -EINVAL;
 227         }
 228
 229         if (pci_dev->mem_resource[0].addr == NULL) {
 230                 QAT_LOG(ERR, "Could not find VF config space "
 231                                 "(UIO driver attached?).");
 232                 return -EINVAL;
 233         }
 234
 235         /* Allocate the queue pair data structure. */
 236         qp = rte_zmalloc_socket("qat PMD qp metadata",
 237                                 sizeof(*qp), RTE_CACHE_LINE_SIZE,
 238                                 qat_qp_conf->socket_id);
 239         if (qp == NULL) {
 240                 QAT_LOG(ERR, "Failed to alloc mem for qp struct");
 241                 return -ENOMEM;
 242         }
 243         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
 244         qp->op_cookies = rte_zmalloc_socket("qat PMD op cookie pointer",
 245                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 246                         RTE_CACHE_LINE_SIZE, qat_qp_conf->socket_id);
 247         if (qp->op_cookies == NULL) {
 248                 QAT_LOG(ERR, "Failed to alloc mem for cookie");
 249                 rte_free(qp);
 250                 return -ENOMEM;
 251         }
 252
 253         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 254         qp->enqueued = qp->dequeued = 0;
 255
 256         if (qat_queue_create(qat_dev, &(qp->tx_q), qat_qp_conf,
 257                                         ADF_RING_DIR_TX) != 0) {
 258                 QAT_LOG(ERR, "Tx queue create failed "
 259                                 "queue_pair_id=%u", queue_pair_id);
 260                 goto create_err;
 261         }
 262
 263         qp->max_inflights = ADF_MAX_INFLIGHTS(qp->tx_q.queue_size,
 264                                 ADF_BYTES_TO_MSG_SIZE(qp->tx_q.msg_size));
 265
 266         if (qp->max_inflights < 2) {
 267                 QAT_LOG(ERR, "Invalid num inflights");
 268                 qat_queue_delete(&(qp->tx_q));
 269                 goto create_err;
 270         }
 271
 272         if (qat_queue_create(qat_dev, &(qp->rx_q), qat_qp_conf,
 273                                         ADF_RING_DIR_RX) != 0) {
 274                 QAT_LOG(ERR, "Rx queue create failed "
 275                                 "queue_pair_id=%hu", queue_pair_id);
 276                 qat_queue_delete(&(qp->tx_q));
 277                 goto create_err;
 278         }
 279
 280         adf_configure_queues(qp, qat_dev_gen);
 281         adf_queue_arb_enable(qat_dev_gen, &qp->tx_q, qp->mmap_bar_addr,
 282                                         &qat_dev->arb_csr_lock);
 283
 284         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE,
 285                                         "%s%d_cookies_%s_qp%hu",
 286                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 287                 qat_qp_conf->service_str, queue_pair_id);
 288
 289         QAT_LOG(DEBUG, "cookiepool: %s", op_cookie_pool_name);
 290         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 291         if (qp->op_cookie_pool == NULL)
 292                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 293                                 qp->nb_descriptors,
 294                                 qat_qp_conf->cookie_size, 64, 0,
 295                                 NULL, NULL, NULL, NULL,
 296                                 pci_dev->device.numa_node,
 297                                 0);
 298         if (!qp->op_cookie_pool) {
 299                 QAT_LOG(ERR, "QAT PMD Cannot create"
 300                                 " op mempool");
 301                 goto create_err;
 302         }
 303
 304         for (i = 0; i < qp->nb_descriptors; i++) {
 305                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 306                         QAT_LOG(ERR, "QAT PMD Cannot get op_cookie");
 307                         goto create_err;
 308                 }
 309                 memset(qp->op_cookies[i], 0, qat_qp_conf->cookie_size);
 310         }
 311
 312         qp->qat_dev_gen = qat_dev->qat_dev_gen;
 313         qp->service_type = qat_qp_conf->hw->service_type;
 314         qp->qat_dev = qat_dev;
 315
 316         QAT_LOG(DEBUG, "QP setup complete: id: %d, cookiepool: %s",
 317                         queue_pair_id, op_cookie_pool_name);
 318
 319         *qp_addr = qp;
 320         return 0;
 321
 322 create_err:
 323         if (qp->op_cookie_pool)
 324                 rte_mempool_free(qp->op_cookie_pool);
 325         rte_free(qp->op_cookies);
 326         rte_free(qp);
 327         return -EFAULT;
 328 }
 329
 330
 331 int qat_qp_release(enum qat_device_gen qat_dev_gen, struct qat_qp **qp_addr)
 332 {
 333         struct qat_qp *qp = *qp_addr;
 334         uint32_t i;
 335
 336         if (qp == NULL) {
 337                 QAT_LOG(DEBUG, "qp already freed");
 338                 return 0;
 339         }
 340
 341         QAT_LOG(DEBUG, "Free qp on qat_pci device %d",
 342                                 qp->qat_dev->qat_dev_id);
 343
 344         /* Don't free memory if there are still responses to be processed */
 345         if ((qp->enqueued - qp->dequeued) == 0) {
 346                 qat_queue_delete(&(qp->tx_q));
 347                 qat_queue_delete(&(qp->rx_q));
 348         } else {
 349                 return -EAGAIN;
 350         }
 351
 352         adf_queue_arb_disable(qat_dev_gen, &(qp->tx_q), qp->mmap_bar_addr,
 353                                 &qp->qat_dev->arb_csr_lock);
 354
 355         for (i = 0; i < qp->nb_descriptors; i++)
 356                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 357
 358         if (qp->op_cookie_pool)
 359                 rte_mempool_free(qp->op_cookie_pool);
 360
 361         rte_free(qp->op_cookies);
 362         rte_free(qp);
 363         *qp_addr = NULL;
 364         return 0;
 365 }
 366
 367
 368 static void qat_queue_delete(struct qat_queue *queue)
 369 {
 370         const struct rte_memzone *mz;
 371         int status = 0;
 372
 373         if (queue == NULL) {
 374                 QAT_LOG(DEBUG, "Invalid queue");
 375                 return;
 376         }
 377         QAT_LOG(DEBUG, "Free ring %d, memzone: %s",
 378                         queue->hw_queue_number, queue->memz_name);
 379
 380         mz = rte_memzone_lookup(queue->memz_name);
 381         if (mz != NULL) {
 382                 /* Write an unused pattern to the queue memory. */
 383                 memset(queue->base_addr, 0x7F, queue->queue_size);
 384                 status = rte_memzone_free(mz);
 385                 if (status != 0)
 386                         QAT_LOG(ERR, "Error %d on freeing queue %s",
 387                                         status, queue->memz_name);
 388         } else {
 389                 QAT_LOG(DEBUG, "queue %s doesn't exist",
 390                                 queue->memz_name);
 391         }
 392 }
 393
 394 static int
 395 qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue,
 396                 struct qat_qp_config *qp_conf, uint8_t dir)
 397 {
 398         uint64_t queue_base;
 399         void *io_addr;
 400         const struct rte_memzone *qp_mz;
 401         struct rte_pci_device *pci_dev =
 402                         qat_pci_devs[qat_dev->qat_dev_id].pci_dev;
 403         enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
 404         int ret = 0;
 405         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
 406                         qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size);
 407         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
 408
 409         queue->hw_bundle_number = qp_conf->hw->hw_bundle_num;
 410         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
 411                         qp_conf->hw->tx_ring_num : qp_conf->hw->rx_ring_num);
 412
 413         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 414                 QAT_LOG(ERR, "Invalid descriptor size %d", desc_size);
 415                 return -EINVAL;
 416         }
 417
 418         /*
 419          * Allocate a memzone for the queue - create a unique name.
 420          */
 421         snprintf(queue->memz_name, sizeof(queue->memz_name),
 422                         "%s_%d_%s_%s_%d_%d",
 423                 pci_dev->driver->driver.name, qat_dev->qat_dev_id,
 424                 qp_conf->service_str, "qp_mem",
 425                 queue->hw_bundle_number, queue->hw_queue_number);
 426         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 427                         pci_dev->device.numa_node);
 428         if (qp_mz == NULL) {
 429                 QAT_LOG(ERR, "Failed to allocate ring memzone");
 430                 return -ENOMEM;
 431         }
 432
 433         queue->base_addr = (char *)qp_mz->addr;
 434         queue->base_phys_addr = qp_mz->iova;
 435         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 436                         queue_size_bytes)) {
 437                 QAT_LOG(ERR, "Invalid alignment on queue create "
 438                                         " 0x%"PRIx64"\n",
 439                                         queue->base_phys_addr);
 440                 ret = -EFAULT;
 441                 goto queue_create_err;
 442         }
 443
 444         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
 445                         &(queue->queue_size)) != 0) {
 446                 QAT_LOG(ERR, "Invalid num inflights");
 447                 ret = -EINVAL;
 448                 goto queue_create_err;
 449         }
 450
 451         queue->modulo_mask = (1 << ADF_RING_SIZE_MODULO(queue->queue_size)) - 1;
 452         queue->head = 0;
 453         queue->tail = 0;
 454         queue->msg_size = desc_size;
 455
 456         /* For fast calculation of cookie index, relies on msg_size being 2^n */
 457         queue->trailz = __builtin_ctz(desc_size);
 458
 459         /*
 460          * Write an unused pattern to the queue memory.
 461          */
 462         memset(queue->base_addr, 0x7F, queue_size_bytes);
 463         io_addr = pci_dev->mem_resource[0].addr;
 464
 465         if (qat_dev_gen == QAT_GEN4) {
 466                 queue_base = BUILD_RING_BASE_ADDR_GEN4(queue->base_phys_addr,
 467                                         queue->queue_size);
 468                 WRITE_CSR_RING_BASE_GEN4VF(io_addr, queue->hw_bundle_number,
 469                         queue->hw_queue_number, queue_base);
 470         } else {
 471                 queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 472                                 queue->queue_size);
 473                 WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 474                         queue->hw_queue_number, queue_base);
 475         }
 476
 477         QAT_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
 478                 " nb msgs %u, msg_size %u, modulo mask %u",
 479                         queue->memz_name,
 480                         queue->queue_size, queue_size_bytes,
 481                         qp_conf->nb_descriptors, desc_size,
 482                         queue->modulo_mask);
 483
 484         return 0;
 485
 486 queue_create_err:
 487         rte_memzone_free(qp_mz);
 488         return ret;
 489 }
 490
 491 int
 492 qat_select_valid_queue(struct qat_pci_device *qat_dev, int qp_id,
 493                         enum qat_service_type service_type)
 494 {
 495         if (qat_dev->qat_dev_gen == QAT_GEN4) {
 496                 int i = 0, valid_qps = 0;
 497
 498                 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
 499                         if (qat_dev->qp_gen4_data[i][0].service_type ==
 500                                 service_type) {
 501                                 if (valid_qps == qp_id)
 502                                         return i;
 503                                 ++valid_qps;
 504                         }
 505                 }
 506         }
 507         return -1;
 508 }
 509
 510 int
 511 qat_read_qp_config(struct qat_pci_device *qat_dev)
 512 {
 513         int i = 0;
 514         enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
 515
 516         if (qat_dev_gen == QAT_GEN4) {
 517                 uint16_t svc = 0;
 518
 519                 if (qat_query_svc_gen4(qat_dev, (uint8_t *)&svc))
 520                         return -(EFAULT);
 521                 for (; i < QAT_GEN4_BUNDLE_NUM; i++) {
 522                         struct qat_qp_hw_data *hw_data =
 523                                 &qat_dev->qp_gen4_data[i][0];
 524                         uint8_t svc1 = (svc >> (3 * i)) & 0x7;
 525                         enum qat_service_type service_type = QAT_SERVICE_INVALID;
 526
 527                         if (svc1 == QAT_SVC_SYM) {
 528                                 service_type = QAT_SERVICE_SYMMETRIC;
 529                                 QAT_LOG(DEBUG,
 530                                         "Discovered SYMMETRIC service on bundle %d",
 531                                         i);
 532                         } else if (svc1 == QAT_SVC_COMPRESSION) {
 533                                 service_type = QAT_SERVICE_COMPRESSION;
 534                                 QAT_LOG(DEBUG,
 535                                         "Discovered COPRESSION service on bundle %d",
 536                                         i);
 537                         } else if (svc1 == QAT_SVC_ASYM) {
 538                                 service_type = QAT_SERVICE_ASYMMETRIC;
 539                                 QAT_LOG(DEBUG,
 540                                         "Discovered ASYMMETRIC service on bundle %d",
 541                                         i);
 542                         } else {
 543                                 QAT_LOG(ERR,
 544                                         "Unrecognized service on bundle %d",
 545                                         i);
 546                                 return -(EFAULT);
 547                         }
 548
 549                         memset(hw_data, 0, sizeof(*hw_data));
 550                         hw_data->service_type = service_type;
 551                         if (service_type == QAT_SERVICE_ASYMMETRIC) {
 552                                 hw_data->tx_msg_size = 64;
 553                                 hw_data->rx_msg_size = 32;
 554                         } else if (service_type == QAT_SERVICE_SYMMETRIC ||
 555                                         service_type ==
 556                                                 QAT_SERVICE_COMPRESSION) {
 557                                 hw_data->tx_msg_size = 128;
 558                                 hw_data->rx_msg_size = 32;
 559                         }
 560                         hw_data->tx_ring_num = 0;
 561                         hw_data->rx_ring_num = 1;
 562                         hw_data->hw_bundle_num = i;
 563                 }
 564                 return 0;
 565         }
 566         return -(EINVAL);
 567 }
 568
 569 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 570                                         uint32_t queue_size_bytes)
 571 {
 572         if (((queue_size_bytes - 1) & phys_addr) != 0)
 573                 return -EINVAL;
 574         return 0;
 575 }
 576
 577 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 578         uint32_t *p_queue_size_for_csr)
 579 {
 580         uint8_t i = ADF_MIN_RING_SIZE;
 581
 582         for (; i <= ADF_MAX_RING_SIZE; i++)
 583                 if ((msg_size * msg_num) ==
 584                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 585                         *p_queue_size_for_csr = i;
 586                         return 0;
 587                 }
 588         QAT_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 589         return -EINVAL;
 590 }
 591
 592 static void
 593 adf_queue_arb_enable(enum qat_device_gen qat_dev_gen, struct qat_queue *txq,
 594                         void *base_addr, rte_spinlock_t *lock)
 595 {
 596         uint32_t arb_csr_offset = 0, value;
 597
 598         rte_spinlock_lock(lock);
 599         if (qat_dev_gen == QAT_GEN4) {
 600                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 601                                 (ADF_RING_BUNDLE_SIZE_GEN4 *
 602                                 txq->hw_bundle_number);
 603                 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
 604                                 arb_csr_offset);
 605         } else {
 606                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 607                                 (ADF_ARB_REG_SLOT *
 608                                 txq->hw_bundle_number);
 609                 value = ADF_CSR_RD(base_addr,
 610                                 arb_csr_offset);
 611         }
 612         value |= (0x01 << txq->hw_queue_number);
 613         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 614         rte_spinlock_unlock(lock);
 615 }
 616
 617 static void adf_queue_arb_disable(enum qat_device_gen qat_dev_gen,
 618                 struct qat_queue *txq, void *base_addr, rte_spinlock_t *lock)
 619 {
 620         uint32_t arb_csr_offset = 0, value;
 621
 622         rte_spinlock_lock(lock);
 623         if (qat_dev_gen == QAT_GEN4) {
 624                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 625                                 (ADF_RING_BUNDLE_SIZE_GEN4 *
 626                                 txq->hw_bundle_number);
 627                 value = ADF_CSR_RD(base_addr + ADF_RING_CSR_ADDR_OFFSET_GEN4VF,
 628                                 arb_csr_offset);
 629         } else {
 630                 arb_csr_offset = ADF_ARB_RINGSRVARBEN_OFFSET +
 631                                 (ADF_ARB_REG_SLOT *
 632                                 txq->hw_bundle_number);
 633                 value = ADF_CSR_RD(base_addr,
 634                                 arb_csr_offset);
 635         }
 636         value &= ~(0x01 << txq->hw_queue_number);
 637         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 638         rte_spinlock_unlock(lock);
 639 }
 640
 641 static void adf_configure_queues(struct qat_qp *qp,
 642                 enum qat_device_gen qat_dev_gen)
 643 {
 644         uint32_t q_tx_config, q_resp_config;
 645         struct qat_queue *q_tx = &qp->tx_q, *q_rx = &qp->rx_q;
 646
 647         q_tx_config = BUILD_RING_CONFIG(q_tx->queue_size);
 648         q_resp_config = BUILD_RESP_RING_CONFIG(q_rx->queue_size,
 649                         ADF_RING_NEAR_WATERMARK_512,
 650                         ADF_RING_NEAR_WATERMARK_0);
 651
 652         if (qat_dev_gen == QAT_GEN4) {
 653                 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
 654                         q_tx->hw_bundle_number, q_tx->hw_queue_number,
 655                         q_tx_config);
 656                 WRITE_CSR_RING_CONFIG_GEN4VF(qp->mmap_bar_addr,
 657                         q_rx->hw_bundle_number, q_rx->hw_queue_number,
 658                         q_resp_config);
 659         } else {
 660                 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
 661                         q_tx->hw_bundle_number, q_tx->hw_queue_number,
 662                         q_tx_config);
 663                 WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr,
 664                         q_rx->hw_bundle_number, q_rx->hw_queue_number,
 665                         q_resp_config);
 666         }
 667 }
 668
 669 static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
 670 {
 671         return data & modulo_mask;
 672 }
 673
 674 static inline void
 675 txq_write_tail(enum qat_device_gen qat_dev_gen,
 676                 struct qat_qp *qp, struct qat_queue *q) {
 677
 678         if (qat_dev_gen == QAT_GEN4) {
 679                 WRITE_CSR_RING_TAIL_GEN4VF(qp->mmap_bar_addr,
 680                         q->hw_bundle_number, q->hw_queue_number, q->tail);
 681         } else {
 682                 WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 683                         q->hw_queue_number, q->tail);
 684         }
 685 }
 686
 687 static inline
 688 void rxq_free_desc(enum qat_device_gen qat_dev_gen, struct qat_qp *qp,
 689                                 struct qat_queue *q)
 690 {
 691         uint32_t old_head, new_head;
 692         uint32_t max_head;
 693
 694         old_head = q->csr_head;
 695         new_head = q->head;
 696         max_head = qp->nb_descriptors * q->msg_size;
 697
 698         /* write out free descriptors */
 699         void *cur_desc = (uint8_t *)q->base_addr + old_head;
 700
 701         if (new_head < old_head) {
 702                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
 703                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
 704         } else {
 705                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
 706         }
 707         q->nb_processed_responses = 0;
 708         q->csr_head = new_head;
 709
 710         /* write current head to CSR */
 711         if (qat_dev_gen == QAT_GEN4) {
 712                 WRITE_CSR_RING_HEAD_GEN4VF(qp->mmap_bar_addr,
 713                         q->hw_bundle_number, q->hw_queue_number, new_head);
 714         } else {
 715                 WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
 716                                 q->hw_queue_number, new_head);
 717         }
 718
 719 }
 720
 721 uint16_t
 722 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 723 {
 724         register struct qat_queue *queue;
 725         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 726         register uint32_t nb_ops_sent = 0;
 727         register int ret = -1;
 728         uint16_t nb_ops_possible = nb_ops;
 729         register uint8_t *base_addr;
 730         register uint32_t tail;
 731
 732         if (unlikely(nb_ops == 0))
 733                 return 0;
 734
 735         /* read params used a lot in main loop into registers */
 736         queue = &(tmp_qp->tx_q);
 737         base_addr = (uint8_t *)queue->base_addr;
 738         tail = queue->tail;
 739
 740         /* Find how many can actually fit on the ring */
 741         {
 742                 /* dequeued can only be written by one thread, but it may not
 743                  * be this thread. As it's 4-byte aligned it will be read
 744                  * atomically here by any Intel CPU.
 745                  * enqueued can wrap before dequeued, but cannot
 746                  * lap it as var size of enq/deq (uint32_t) > var size of
 747                  * max_inflights (uint16_t). In reality inflights is never
 748                  * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
 749                  * On wrapping, the calculation still returns the correct
 750                  * positive value as all three vars are unsigned.
 751                  */
 752                 uint32_t inflights =
 753                         tmp_qp->enqueued - tmp_qp->dequeued;
 754
 755                 if ((inflights + nb_ops) > tmp_qp->max_inflights) {
 756                         nb_ops_possible = tmp_qp->max_inflights - inflights;
 757                         if (nb_ops_possible == 0)
 758                                 return 0;
 759                 }
 760                 /* QAT has plenty of work queued already, so don't waste cycles
 761                  * enqueueing, wait til the application has gathered a bigger
 762                  * burst or some completed ops have been dequeued
 763                  */
 764                 if (tmp_qp->min_enq_burst_threshold && inflights >
 765                                 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
 766                                 tmp_qp->min_enq_burst_threshold) {
 767                         tmp_qp->stats.threshold_hit_count++;
 768                         return 0;
 769                 }
 770         }
 771
 772 #ifdef BUILD_QAT_SYM
 773         if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
 774                 qat_sym_preprocess_requests(ops, nb_ops_possible);
 775 #endif
 776
 777         while (nb_ops_sent != nb_ops_possible) {
 778                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) {
 779 #ifdef BUILD_QAT_SYM
 780                         ret = qat_sym_build_request(*ops, base_addr + tail,
 781                                 tmp_qp->op_cookies[tail >> queue->trailz],
 782                                 tmp_qp->qat_dev_gen);
 783 #endif
 784                 } else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) {
 785                         ret = qat_comp_build_request(*ops, base_addr + tail,
 786                                 tmp_qp->op_cookies[tail >> queue->trailz],
 787                                 tmp_qp->qat_dev_gen);
 788                 } else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) {
 789 #ifdef BUILD_QAT_ASYM
 790                         ret = qat_asym_build_request(*ops, base_addr + tail,
 791                                 tmp_qp->op_cookies[tail >> queue->trailz],
 792                                 tmp_qp->qat_dev_gen);
 793 #endif
 794                 }
 795                 if (ret != 0) {
 796                         tmp_qp->stats.enqueue_err_count++;
 797                         /* This message cannot be enqueued */
 798                         if (nb_ops_sent == 0)
 799                                 return 0;
 800                         goto kick_tail;
 801                 }
 802
 803                 tail = adf_modulo(tail + queue->msg_size, queue->modulo_mask);
 804                 ops++;
 805                 nb_ops_sent++;
 806         }
 807 kick_tail:
 808         queue->tail = tail;
 809         tmp_qp->enqueued += nb_ops_sent;
 810         tmp_qp->stats.enqueued_count += nb_ops_sent;
 811         txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
 812         return nb_ops_sent;
 813 }
 814
 815 /* Use this for compression only - but keep consistent with above common
 816  * function as much as possible.
 817  */
 818 uint16_t
 819 qat_enqueue_comp_op_burst(void *qp, void **ops, uint16_t nb_ops)
 820 {
 821         register struct qat_queue *queue;
 822         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 823         register uint32_t nb_ops_sent = 0;
 824         register int nb_desc_to_build;
 825         uint16_t nb_ops_possible = nb_ops;
 826         register uint8_t *base_addr;
 827         register uint32_t tail;
 828
 829         int descriptors_built, total_descriptors_built = 0;
 830         int nb_remaining_descriptors;
 831         int overflow = 0;
 832
 833         if (unlikely(nb_ops == 0))
 834                 return 0;
 835
 836         /* read params used a lot in main loop into registers */
 837         queue = &(tmp_qp->tx_q);
 838         base_addr = (uint8_t *)queue->base_addr;
 839         tail = queue->tail;
 840
 841         /* Find how many can actually fit on the ring */
 842         {
 843                 /* dequeued can only be written by one thread, but it may not
 844                  * be this thread. As it's 4-byte aligned it will be read
 845                  * atomically here by any Intel CPU.
 846                  * enqueued can wrap before dequeued, but cannot
 847                  * lap it as var size of enq/deq (uint32_t) > var size of
 848                  * max_inflights (uint16_t). In reality inflights is never
 849                  * even as big as max uint16_t, as it's <= ADF_MAX_DESC.
 850                  * On wrapping, the calculation still returns the correct
 851                  * positive value as all three vars are unsigned.
 852                  */
 853                 uint32_t inflights =
 854                         tmp_qp->enqueued - tmp_qp->dequeued;
 855
 856                 /* Find how many can actually fit on the ring */
 857                 overflow = (inflights + nb_ops) - tmp_qp->max_inflights;
 858                 if (overflow > 0) {
 859                         nb_ops_possible = nb_ops - overflow;
 860                         if (nb_ops_possible == 0)
 861                                 return 0;
 862                 }
 863
 864                 /* QAT has plenty of work queued already, so don't waste cycles
 865                  * enqueueing, wait til the application has gathered a bigger
 866                  * burst or some completed ops have been dequeued
 867                  */
 868                 if (tmp_qp->min_enq_burst_threshold && inflights >
 869                                 QAT_QP_MIN_INFL_THRESHOLD && nb_ops_possible <
 870                                 tmp_qp->min_enq_burst_threshold) {
 871                         tmp_qp->stats.threshold_hit_count++;
 872                         return 0;
 873                 }
 874         }
 875
 876         /* At this point nb_ops_possible is assuming a 1:1 mapping
 877          * between ops and descriptors.
 878          * Fewer may be sent if some ops have to be split.
 879          * nb_ops_possible is <= burst size.
 880          * Find out how many spaces are actually available on the qp in case
 881          * more are needed.
 882          */
 883         nb_remaining_descriptors = nb_ops_possible
 884                          + ((overflow >= 0) ? 0 : overflow * (-1));
 885         QAT_DP_LOG(DEBUG, "Nb ops requested %d, nb descriptors remaining %d",
 886                         nb_ops, nb_remaining_descriptors);
 887
 888         while (nb_ops_sent != nb_ops_possible &&
 889                                 nb_remaining_descriptors > 0) {
 890                 struct qat_comp_op_cookie *cookie =
 891                                 tmp_qp->op_cookies[tail >> queue->trailz];
 892
 893                 descriptors_built = 0;
 894
 895                 QAT_DP_LOG(DEBUG, "--- data length: %u",
 896                            ((struct rte_comp_op *)*ops)->src.length);
 897
 898                 nb_desc_to_build = qat_comp_build_request(*ops,
 899                                 base_addr + tail, cookie, tmp_qp->qat_dev_gen);
 900                 QAT_DP_LOG(DEBUG, "%d descriptors built, %d remaining, "
 901                         "%d ops sent, %d descriptors needed",
 902                         total_descriptors_built, nb_remaining_descriptors,
 903                         nb_ops_sent, nb_desc_to_build);
 904
 905                 if (unlikely(nb_desc_to_build < 0)) {
 906                         /* this message cannot be enqueued */
 907                         tmp_qp->stats.enqueue_err_count++;
 908                         if (nb_ops_sent == 0)
 909                                 return 0;
 910                         goto kick_tail;
 911                 } else if (unlikely(nb_desc_to_build > 1)) {
 912                         /* this op is too big and must be split - get more
 913                          * descriptors and retry
 914                          */
 915
 916                         QAT_DP_LOG(DEBUG, "Build %d descriptors for this op",
 917                                         nb_desc_to_build);
 918
 919                         nb_remaining_descriptors -= nb_desc_to_build;
 920                         if (nb_remaining_descriptors >= 0) {
 921                                 /* There are enough remaining descriptors
 922                                  * so retry
 923                                  */
 924                                 int ret2 = qat_comp_build_multiple_requests(
 925                                                 *ops, tmp_qp, tail,
 926                                                 nb_desc_to_build);
 927
 928                                 if (unlikely(ret2 < 1)) {
 929                                         QAT_DP_LOG(DEBUG,
 930                                                         "Failed to build (%d) descriptors, status %d",
 931                                                         nb_desc_to_build, ret2);
 932
 933                                         qat_comp_free_split_op_memzones(cookie,
 934                                                         nb_desc_to_build - 1);
 935
 936                                         tmp_qp->stats.enqueue_err_count++;
 937
 938                                         /* This message cannot be enqueued */
 939                                         if (nb_ops_sent == 0)
 940                                                 return 0;
 941                                         goto kick_tail;
 942                                 } else {
 943                                         descriptors_built = ret2;
 944                                         total_descriptors_built +=
 945                                                         descriptors_built;
 946                                         nb_remaining_descriptors -=
 947                                                         descriptors_built;
 948                                         QAT_DP_LOG(DEBUG,
 949                                                         "Multiple descriptors (%d) built ok",
 950                                                         descriptors_built);
 951                                 }
 952                         } else {
 953                                 QAT_DP_LOG(ERR, "For the current op, number of requested descriptors (%d) "
 954                                                 "exceeds number of available descriptors (%d)",
 955                                                 nb_desc_to_build,
 956                                                 nb_remaining_descriptors +
 957                                                         nb_desc_to_build);
 958
 959                                 qat_comp_free_split_op_memzones(cookie,
 960                                                 nb_desc_to_build - 1);
 961
 962                                 /* Not enough extra descriptors */
 963                                 if (nb_ops_sent == 0)
 964                                         return 0;
 965                                 goto kick_tail;
 966                         }
 967                 } else {
 968                         descriptors_built = 1;
 969                         total_descriptors_built++;
 970                         nb_remaining_descriptors--;
 971                         QAT_DP_LOG(DEBUG, "Single descriptor built ok");
 972                 }
 973
 974                 tail = adf_modulo(tail + (queue->msg_size * descriptors_built),
 975                                   queue->modulo_mask);
 976                 ops++;
 977                 nb_ops_sent++;
 978         }
 979
 980 kick_tail:
 981         queue->tail = tail;
 982         tmp_qp->enqueued += total_descriptors_built;
 983         tmp_qp->stats.enqueued_count += nb_ops_sent;
 984         txq_write_tail(tmp_qp->qat_dev_gen, tmp_qp, queue);
 985         return nb_ops_sent;
 986 }
 987
 988 uint16_t
 989 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 990 {
 991         struct qat_queue *rx_queue;
 992         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 993         uint32_t head;
 994         uint32_t op_resp_counter = 0, fw_resp_counter = 0;
 995         uint8_t *resp_msg;
 996         int nb_fw_responses;
 997
 998         rx_queue = &(tmp_qp->rx_q);
 999         head = rx_queue->head;
1000         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
1001
1002         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
1003                         op_resp_counter != nb_ops) {
1004
1005                 nb_fw_responses = 1;
1006
1007                 if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC)
1008                         qat_sym_process_response(ops, resp_msg,
1009                                 tmp_qp->op_cookies[head >> rx_queue->trailz]);
1010                 else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION)
1011                         nb_fw_responses = qat_comp_process_response(
1012                                 ops, resp_msg,
1013                                 tmp_qp->op_cookies[head >> rx_queue->trailz],
1014                                 &tmp_qp->stats.dequeue_err_count);
1015 #ifdef BUILD_QAT_ASYM
1016                 else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC)
1017                         qat_asym_process_response(ops, resp_msg,
1018                                 tmp_qp->op_cookies[head >> rx_queue->trailz]);
1019 #endif
1020
1021                 head = adf_modulo(head + rx_queue->msg_size,
1022                                   rx_queue->modulo_mask);
1023
1024                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
1025
1026                 if (nb_fw_responses) {
1027                         /* only move on to next op if one was ready to return
1028                          * to API
1029                          */
1030                         ops++;
1031                         op_resp_counter++;
1032                 }
1033
1034                  /* A compression op may be broken up into multiple fw requests.
1035                   * Only count fw responses as complete once ALL the responses
1036                   * associated with an op have been processed, as the cookie
1037                   * data from the first response must be available until
1038                   * finished with all firmware responses.
1039                   */
1040                 fw_resp_counter += nb_fw_responses;
1041
1042                 rx_queue->nb_processed_responses++;
1043         }
1044
1045         tmp_qp->dequeued += fw_resp_counter;
1046         tmp_qp->stats.dequeued_count += op_resp_counter;
1047
1048         rx_queue->head = head;
1049         if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
1050                 rxq_free_desc(tmp_qp->qat_dev_gen, tmp_qp, rx_queue);
1051
1052         QAT_DP_LOG(DEBUG, "Dequeue burst return: %u, QAT responses: %u",
1053                         op_resp_counter, fw_resp_counter);
1054
1055         return op_resp_counter;
1056 }
1057
1058 /* This is almost same as dequeue_op_burst, without the atomic, without stats
1059  * and without the op. Dequeues one response.
1060  */
1061 static uint8_t
1062 qat_cq_dequeue_response(struct qat_qp *qp, void *out_data)
1063 {
1064         uint8_t result = 0;
1065         uint8_t retries = 0;
1066         struct qat_queue *queue = &(qp->rx_q);
1067         struct icp_qat_fw_comn_resp *resp_msg = (struct icp_qat_fw_comn_resp *)
1068                         ((uint8_t *)queue->base_addr + queue->head);
1069
1070         while (retries++ < QAT_CQ_MAX_DEQ_RETRIES &&
1071                         *(uint32_t *)resp_msg == ADF_RING_EMPTY_SIG) {
1072                 /* loop waiting for response until we reach the timeout */
1073                 rte_delay_ms(20);
1074         }
1075
1076         if (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG) {
1077                 /* response received */
1078                 result = 1;
1079
1080                 /* check status flag */
1081                 if (ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
1082                                 resp_msg->comn_hdr.comn_status) ==
1083                                 ICP_QAT_FW_COMN_STATUS_FLAG_OK) {
1084                         /* success */
1085                         memcpy(out_data, resp_msg, queue->msg_size);
1086                 } else {
1087                         memset(out_data, 0, queue->msg_size);
1088                 }
1089
1090                 queue->head = adf_modulo(queue->head + queue->msg_size,
1091                                 queue->modulo_mask);
1092                 rxq_free_desc(qp->qat_dev_gen, qp, queue);
1093         }
1094
1095         return result;
1096 }
1097
1098 /* Sends a NULL message and extracts QAT fw version from the response.
1099  * Used to determine detailed capabilities based on the fw version number.
1100  * This assumes that there are no inflight messages, i.e. assumes there's space
1101  * on the qp, one message is sent and only one response collected.
1102  * Returns fw version number or 0 for unknown version or a negative error code.
1103  */
1104 int
1105 qat_cq_get_fw_version(struct qat_qp *qp)
1106 {
1107         struct qat_queue *queue = &(qp->tx_q);
1108         uint8_t *base_addr = (uint8_t *)queue->base_addr;
1109         struct icp_qat_fw_comn_req null_msg;
1110         struct icp_qat_fw_comn_resp response;
1111
1112         /* prepare the NULL request */
1113         memset(&null_msg, 0, sizeof(null_msg));
1114         null_msg.comn_hdr.hdr_flags =
1115                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
1116         null_msg.comn_hdr.service_type = ICP_QAT_FW_COMN_REQ_NULL;
1117         null_msg.comn_hdr.service_cmd_id = ICP_QAT_FW_NULL_REQ_SERV_ID;
1118
1119 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
1120         QAT_DP_HEXDUMP_LOG(DEBUG, "NULL request", &null_msg, sizeof(null_msg));
1121 #endif
1122
1123         /* send the NULL request */
1124         memcpy(base_addr + queue->tail, &null_msg, sizeof(null_msg));
1125         queue->tail = adf_modulo(queue->tail + queue->msg_size,
1126                         queue->modulo_mask);
1127         txq_write_tail(qp->qat_dev_gen, qp, queue);
1128
1129         /* receive a response */
1130         if (qat_cq_dequeue_response(qp, &response)) {
1131
1132 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
1133                 QAT_DP_HEXDUMP_LOG(DEBUG, "NULL response:", &response,
1134                                 sizeof(response));
1135 #endif
1136                 /* if LW0 bit 24 is set - then the fw version was returned */
1137                 if (QAT_FIELD_GET(response.comn_hdr.hdr_flags,
1138                                 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_BITPOS,
1139                                 ICP_QAT_FW_COMN_NULL_VERSION_FLAG_MASK))
1140                         return response.resrvd[0]; /* return LW4 */
1141                 else
1142                         return 0; /* not set - we don't know fw version */
1143         }
1144
1145         QAT_LOG(ERR, "No response received");
1146         return -EINVAL;
1147 }
1148
1149 __rte_weak int
1150 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
1151                           void *op_cookie __rte_unused,
1152                           uint64_t *dequeue_err_count __rte_unused)
1153 {
1154         return  0;
1155 }