drivers/crypto/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2018 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_dev.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_cryptodev_pmd.h>
  10 #include <rte_pci.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_atomic.h>
  13 #include <rte_prefetch.h>
  14
  15 #include "qat_logs.h"
  16 #include "qat_qp.h"
  17 #include "qat_sym.h"
  18
  19 #include "adf_transport_access_macros.h"
  20
  21 #define ADF_MAX_DESC                            4096
  22 #define ADF_MIN_DESC                            128
  23
  24 #define ADF_ARB_REG_SLOT                        0x1000
  25 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  26
  27 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  28         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  29         (ADF_ARB_REG_SLOT * index), value)
  30
  31 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
  32         uint32_t queue_size_bytes);
  33 static void qat_queue_delete(struct qat_queue *queue);
  34 static int qat_queue_create(struct rte_cryptodev *dev,
  35         struct qat_queue *queue, struct qat_qp_config *, uint8_t dir);
  36 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
  37         uint32_t *queue_size_for_csr);
  38 static void adf_configure_queues(struct qat_qp *queue);
  39 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr);
  40 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr);
  41
  42 static const struct rte_memzone *
  43 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
  44                         int socket_id)
  45 {
  46         const struct rte_memzone *mz;
  47
  48         PMD_INIT_FUNC_TRACE();
  49         mz = rte_memzone_lookup(queue_name);
  50         if (mz != 0) {
  51                 if (((size_t)queue_size <= mz->len) &&
  52                                 ((socket_id == SOCKET_ID_ANY) ||
  53                                         (socket_id == mz->socket_id))) {
  54                         PMD_DRV_LOG(DEBUG, "re-use memzone already "
  55                                         "allocated for %s", queue_name);
  56                         return mz;
  57                 }
  58
  59                 PMD_DRV_LOG(ERR, "Incompatible memzone already "
  60                                 "allocated %s, size %u, socket %d. "
  61                                 "Requested size %u, socket %u",
  62                                 queue_name, (uint32_t)mz->len,
  63                                 mz->socket_id, queue_size, socket_id);
  64                 return NULL;
  65         }
  66
  67         PMD_DRV_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
  68                                         queue_name, queue_size, socket_id);
  69         return rte_memzone_reserve_aligned(queue_name, queue_size,
  70                 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
  71 }
  72
  73 int qat_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
  74                 struct qat_qp_config *qat_qp_conf)
  75 {
  76         struct qat_qp *qp;
  77         struct rte_pci_device *pci_dev;
  78         char op_cookie_pool_name[RTE_RING_NAMESIZE];
  79         uint32_t i;
  80
  81
  82         if ((qat_qp_conf->nb_descriptors > ADF_MAX_DESC) ||
  83                 (qat_qp_conf->nb_descriptors < ADF_MIN_DESC)) {
  84                 PMD_DRV_LOG(ERR, "Can't create qp for %u descriptors",
  85                                 qat_qp_conf->nb_descriptors);
  86                 return -EINVAL;
  87         }
  88
  89         pci_dev = RTE_DEV_TO_PCI(dev->device);
  90
  91         if (pci_dev->mem_resource[0].addr == NULL) {
  92                 PMD_DRV_LOG(ERR, "Could not find VF config space "
  93                                 "(UIO driver attached?).");
  94                 return -EINVAL;
  95         }
  96
  97         /* Allocate the queue pair data structure. */
  98         qp = rte_zmalloc("qat PMD qp metadata",
  99                         sizeof(*qp), RTE_CACHE_LINE_SIZE);
 100         if (qp == NULL) {
 101                 PMD_DRV_LOG(ERR, "Failed to alloc mem for qp struct");
 102                 return -ENOMEM;
 103         }
 104         qp->nb_descriptors = qat_qp_conf->nb_descriptors;
 105         qp->op_cookies = rte_zmalloc("qat PMD op cookie pointer",
 106                         qat_qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 107                         RTE_CACHE_LINE_SIZE);
 108         if (qp->op_cookies == NULL) {
 109                 PMD_DRV_LOG(ERR, "Failed to alloc mem for cookie");
 110                 rte_free(qp);
 111                 return -ENOMEM;
 112         }
 113
 114         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 115         qp->inflights16 = 0;
 116
 117         if (qat_queue_create(dev, &(qp->tx_q), qat_qp_conf,
 118                                         ADF_RING_DIR_TX) != 0) {
 119                 PMD_INIT_LOG(ERR, "Tx queue create failed "
 120                                 "queue_pair_id=%u", queue_pair_id);
 121                 goto create_err;
 122         }
 123
 124         if (qat_queue_create(dev, &(qp->rx_q), qat_qp_conf,
 125                                         ADF_RING_DIR_RX) != 0) {
 126                 PMD_DRV_LOG(ERR, "Rx queue create failed "
 127                                 "queue_pair_id=%hu", queue_pair_id);
 128                 qat_queue_delete(&(qp->tx_q));
 129                 goto create_err;
 130         }
 131
 132         adf_configure_queues(qp);
 133         adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr);
 134
 135         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, "%s_%s_qp_op_%d_%hu",
 136                 pci_dev->driver->driver.name, qat_qp_conf->service_str,
 137                 dev->data->dev_id, queue_pair_id);
 138
 139         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 140         if (qp->op_cookie_pool == NULL)
 141                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 142                                 qp->nb_descriptors,
 143                                 qat_qp_conf->cookie_size, 64, 0,
 144                                 NULL, NULL, NULL, NULL, qat_qp_conf->socket_id,
 145                                 0);
 146         if (!qp->op_cookie_pool) {
 147                 PMD_DRV_LOG(ERR, "QAT PMD Cannot create"
 148                                 " op mempool");
 149                 goto create_err;
 150         }
 151
 152         for (i = 0; i < qp->nb_descriptors; i++) {
 153                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 154                         PMD_DRV_LOG(ERR, "QAT PMD Cannot get op_cookie");
 155                         goto create_err;
 156                 }
 157         }
 158
 159         struct qat_pmd_private *internals
 160                 = dev->data->dev_private;
 161         qp->qat_dev_gen = internals->qat_dev_gen;
 162         qp->build_request = qat_qp_conf->build_request;
 163         qp->process_response = qat_qp_conf->process_response;
 164
 165         dev->data->queue_pairs[queue_pair_id] = qp;
 166         return 0;
 167
 168 create_err:
 169         rte_free(qp);
 170         return -EFAULT;
 171 }
 172
 173 int qat_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
 174 {
 175         struct qat_qp *qp =
 176                         (struct qat_qp *)dev->data->queue_pairs[queue_pair_id];
 177         uint32_t i;
 178
 179         PMD_INIT_FUNC_TRACE();
 180         if (qp == NULL) {
 181                 PMD_DRV_LOG(DEBUG, "qp already freed");
 182                 return 0;
 183         }
 184
 185         /* Don't free memory if there are still responses to be processed */
 186         if (qp->inflights16 == 0) {
 187                 qat_queue_delete(&(qp->tx_q));
 188                 qat_queue_delete(&(qp->rx_q));
 189         } else {
 190                 return -EAGAIN;
 191         }
 192
 193         adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr);
 194
 195         for (i = 0; i < qp->nb_descriptors; i++)
 196                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 197
 198         if (qp->op_cookie_pool)
 199                 rte_mempool_free(qp->op_cookie_pool);
 200
 201         rte_free(qp->op_cookies);
 202         rte_free(qp);
 203         dev->data->queue_pairs[queue_pair_id] = NULL;
 204         return 0;
 205 }
 206
 207
 208
 209
 210 static void qat_queue_delete(struct qat_queue *queue)
 211 {
 212         const struct rte_memzone *mz;
 213         int status = 0;
 214
 215         if (queue == NULL) {
 216                 PMD_DRV_LOG(DEBUG, "Invalid queue");
 217                 return;
 218         }
 219         mz = rte_memzone_lookup(queue->memz_name);
 220         if (mz != NULL) {
 221                 /* Write an unused pattern to the queue memory. */
 222                 memset(queue->base_addr, 0x7F, queue->queue_size);
 223                 status = rte_memzone_free(mz);
 224                 if (status != 0)
 225                         PMD_DRV_LOG(ERR, "Error %d on freeing queue %s",
 226                                         status, queue->memz_name);
 227         } else {
 228                 PMD_DRV_LOG(DEBUG, "queue %s doesn't exist",
 229                                 queue->memz_name);
 230         }
 231 }
 232
 233 static int
 234 qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue,
 235                 struct qat_qp_config *qp_conf, uint8_t dir)
 236 {
 237         uint64_t queue_base;
 238         void *io_addr;
 239         const struct rte_memzone *qp_mz;
 240         struct rte_pci_device *pci_dev;
 241         int ret = 0;
 242         uint16_t desc_size = (dir == ADF_RING_DIR_TX ?
 243                                 qp_conf->tx_msg_size : qp_conf->rx_msg_size);
 244         uint32_t queue_size_bytes = (qp_conf->nb_descriptors)*(desc_size);
 245
 246         queue->hw_bundle_number = qp_conf->hw_bundle_num;
 247         queue->hw_queue_number = (dir == ADF_RING_DIR_TX ?
 248                         qp_conf->tx_ring_num : qp_conf->rx_ring_num);
 249
 250         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 251                 PMD_DRV_LOG(ERR, "Invalid descriptor size %d", desc_size);
 252                 return -EINVAL;
 253         }
 254
 255         pci_dev = RTE_DEV_TO_PCI(dev->device);
 256
 257         /*
 258          * Allocate a memzone for the queue - create a unique name.
 259          */
 260         snprintf(queue->memz_name, sizeof(queue->memz_name),
 261                 "%s_%s_%s_%d_%d_%d",
 262                 pci_dev->driver->driver.name, qp_conf->service_str,
 263                 "qp_mem", dev->data->dev_id,
 264                 queue->hw_bundle_number, queue->hw_queue_number);
 265         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 266                         qp_conf->socket_id);
 267         if (qp_mz == NULL) {
 268                 PMD_DRV_LOG(ERR, "Failed to allocate ring memzone");
 269                 return -ENOMEM;
 270         }
 271
 272         queue->base_addr = (char *)qp_mz->addr;
 273         queue->base_phys_addr = qp_mz->iova;
 274         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 275                         queue_size_bytes)) {
 276                 PMD_DRV_LOG(ERR, "Invalid alignment on queue create "
 277                                         " 0x%"PRIx64"\n",
 278                                         queue->base_phys_addr);
 279                 ret = -EFAULT;
 280                 goto queue_create_err;
 281         }
 282
 283         if (adf_verify_queue_size(desc_size, qp_conf->nb_descriptors,
 284                         &(queue->queue_size)) != 0) {
 285                 PMD_DRV_LOG(ERR, "Invalid num inflights");
 286                 ret = -EINVAL;
 287                 goto queue_create_err;
 288         }
 289
 290         queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
 291                                         ADF_BYTES_TO_MSG_SIZE(desc_size));
 292         queue->modulo = ADF_RING_SIZE_MODULO(queue->queue_size);
 293         PMD_DRV_LOG(DEBUG, "RING: Name:%s, size in CSR: %u, in bytes %u,"
 294                         " nb msgs %u, msg_size %u, max_inflights %u modulo %u",
 295                         queue->memz_name,
 296                         queue->queue_size, queue_size_bytes,
 297                         qp_conf->nb_descriptors, desc_size,
 298                         queue->max_inflights, queue->modulo);
 299
 300         if (queue->max_inflights < 2) {
 301                 PMD_DRV_LOG(ERR, "Invalid num inflights");
 302                 ret = -EINVAL;
 303                 goto queue_create_err;
 304         }
 305         queue->head = 0;
 306         queue->tail = 0;
 307         queue->msg_size = desc_size;
 308
 309         /*
 310          * Write an unused pattern to the queue memory.
 311          */
 312         memset(queue->base_addr, 0x7F, queue_size_bytes);
 313
 314         queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 315                                         queue->queue_size);
 316
 317         io_addr = pci_dev->mem_resource[0].addr;
 318
 319         WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 320                         queue->hw_queue_number, queue_base);
 321         return 0;
 322
 323 queue_create_err:
 324         rte_memzone_free(qp_mz);
 325         return ret;
 326 }
 327
 328 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 329                                         uint32_t queue_size_bytes)
 330 {
 331         PMD_INIT_FUNC_TRACE();
 332         if (((queue_size_bytes - 1) & phys_addr) != 0)
 333                 return -EINVAL;
 334         return 0;
 335 }
 336
 337 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 338         uint32_t *p_queue_size_for_csr)
 339 {
 340         uint8_t i = ADF_MIN_RING_SIZE;
 341
 342         PMD_INIT_FUNC_TRACE();
 343         for (; i <= ADF_MAX_RING_SIZE; i++)
 344                 if ((msg_size * msg_num) ==
 345                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 346                         *p_queue_size_for_csr = i;
 347                         return 0;
 348                 }
 349         PMD_DRV_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 350         return -EINVAL;
 351 }
 352
 353 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr)
 354 {
 355         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 356                                         (ADF_ARB_REG_SLOT *
 357                                                         txq->hw_bundle_number);
 358         uint32_t value;
 359
 360         PMD_INIT_FUNC_TRACE();
 361         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 362         value |= (0x01 << txq->hw_queue_number);
 363         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 364 }
 365
 366 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr)
 367 {
 368         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 369                                         (ADF_ARB_REG_SLOT *
 370                                                         txq->hw_bundle_number);
 371         uint32_t value;
 372
 373         PMD_INIT_FUNC_TRACE();
 374         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 375         value ^= (0x01 << txq->hw_queue_number);
 376         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 377 }
 378
 379 static void adf_configure_queues(struct qat_qp *qp)
 380 {
 381         uint32_t queue_config;
 382         struct qat_queue *queue = &qp->tx_q;
 383
 384         PMD_INIT_FUNC_TRACE();
 385         queue_config = BUILD_RING_CONFIG(queue->queue_size);
 386
 387         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 388                         queue->hw_queue_number, queue_config);
 389
 390         queue = &qp->rx_q;
 391         queue_config =
 392                         BUILD_RESP_RING_CONFIG(queue->queue_size,
 393                                         ADF_RING_NEAR_WATERMARK_512,
 394                                         ADF_RING_NEAR_WATERMARK_0);
 395
 396         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 397                         queue->hw_queue_number, queue_config);
 398 }
 399
 400
 401 static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
 402 {
 403         uint32_t div = data >> shift;
 404         uint32_t mult = div << shift;
 405
 406         return data - mult;
 407 }
 408
 409 static inline void
 410 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
 411         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 412                         q->hw_queue_number, q->tail);
 413         q->nb_pending_requests = 0;
 414         q->csr_tail = q->tail;
 415 }
 416
 417 static inline
 418 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
 419 {
 420         uint32_t old_head, new_head;
 421         uint32_t max_head;
 422
 423         old_head = q->csr_head;
 424         new_head = q->head;
 425         max_head = qp->nb_descriptors * q->msg_size;
 426
 427         /* write out free descriptors */
 428         void *cur_desc = (uint8_t *)q->base_addr + old_head;
 429
 430         if (new_head < old_head) {
 431                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, max_head - old_head);
 432                 memset(q->base_addr, ADF_RING_EMPTY_SIG_BYTE, new_head);
 433         } else {
 434                 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head - old_head);
 435         }
 436         q->nb_processed_responses = 0;
 437         q->csr_head = new_head;
 438
 439         /* write current head to CSR */
 440         WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
 441                             q->hw_queue_number, new_head);
 442 }
 443
 444 uint16_t
 445 qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 446 {
 447         register struct qat_queue *queue;
 448         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 449         register uint32_t nb_ops_sent = 0;
 450         register int ret;
 451         uint16_t nb_ops_possible = nb_ops;
 452         register uint8_t *base_addr;
 453         register uint32_t tail;
 454         int overflow;
 455
 456         if (unlikely(nb_ops == 0))
 457                 return 0;
 458
 459         /* read params used a lot in main loop into registers */
 460         queue = &(tmp_qp->tx_q);
 461         base_addr = (uint8_t *)queue->base_addr;
 462         tail = queue->tail;
 463
 464         /* Find how many can actually fit on the ring */
 465         tmp_qp->inflights16 += nb_ops;
 466         overflow = tmp_qp->inflights16 - queue->max_inflights;
 467         if (overflow > 0) {
 468                 tmp_qp->inflights16 -= overflow;
 469                 nb_ops_possible = nb_ops - overflow;
 470                 if (nb_ops_possible == 0)
 471                         return 0;
 472         }
 473
 474         while (nb_ops_sent != nb_ops_possible) {
 475                 ret = tmp_qp->build_request(*ops, base_addr + tail,
 476                                 tmp_qp->op_cookies[tail / queue->msg_size],
 477                                 tmp_qp->qat_dev_gen);
 478                 if (ret != 0) {
 479                         tmp_qp->stats.enqueue_err_count++;
 480                         /*
 481                          * This message cannot be enqueued,
 482                          * decrease number of ops that wasn't sent
 483                          */
 484                         tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
 485                         if (nb_ops_sent == 0)
 486                                 return 0;
 487                         goto kick_tail;
 488                 }
 489
 490                 tail = adf_modulo(tail + queue->msg_size, queue->modulo);
 491                 ops++;
 492                 nb_ops_sent++;
 493         }
 494 kick_tail:
 495         queue->tail = tail;
 496         tmp_qp->stats.enqueued_count += nb_ops_sent;
 497         queue->nb_pending_requests += nb_ops_sent;
 498         if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
 499                     queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
 500                 txq_write_tail(tmp_qp, queue);
 501         }
 502         return nb_ops_sent;
 503 }
 504
 505 uint16_t
 506 qat_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 507 {
 508         struct qat_queue *rx_queue, *tx_queue;
 509         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 510         uint32_t head;
 511         uint32_t resp_counter = 0;
 512         uint8_t *resp_msg;
 513
 514         rx_queue = &(tmp_qp->rx_q);
 515         tx_queue = &(tmp_qp->tx_q);
 516         head = rx_queue->head;
 517         resp_msg = (uint8_t *)rx_queue->base_addr + rx_queue->head;
 518
 519         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 520                         resp_counter != nb_ops) {
 521
 522                 tmp_qp->process_response(ops, resp_msg,
 523                         tmp_qp->op_cookies[head / rx_queue->msg_size],
 524                         tmp_qp->qat_dev_gen);
 525
 526                 head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo);
 527
 528                 resp_msg = (uint8_t *)rx_queue->base_addr + head;
 529                 ops++;
 530                 resp_counter++;
 531         }
 532         if (resp_counter > 0) {
 533                 rx_queue->head = head;
 534                 tmp_qp->stats.dequeued_count += resp_counter;
 535                 rx_queue->nb_processed_responses += resp_counter;
 536                 tmp_qp->inflights16 -= resp_counter;
 537
 538                 if (rx_queue->nb_processed_responses >
 539                                                 QAT_CSR_HEAD_WRITE_THRESH)
 540                         rxq_free_desc(tmp_qp, rx_queue);
 541         }
 542         /* also check if tail needs to be advanced */
 543         if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
 544                 tx_queue->tail != tx_queue->csr_tail) {
 545                 txq_write_tail(tmp_qp, tx_queue);
 546         }
 547         return resp_counter;
 548 }