drivers/crypto/qat/qat_qp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_dev.h>
   7 #include <rte_malloc.h>
   8 #include <rte_memzone.h>
   9 #include <rte_cryptodev_pmd.h>
  10 #include <rte_pci.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_atomic.h>
  13 #include <rte_prefetch.h>
  14
  15 #include "qat_logs.h"
  16 #include "qat_crypto.h"
  17 #include "qat_algs.h"
  18 #include "adf_transport_access_macros.h"
  19
  20 #define ADF_MAX_SYM_DESC                        4096
  21 #define ADF_MIN_SYM_DESC                        128
  22 #define ADF_SYM_TX_RING_DESC_SIZE               128
  23 #define ADF_SYM_RX_RING_DESC_SIZE               32
  24 #define ADF_SYM_TX_QUEUE_STARTOFF               2
  25 /* Offset from bundle start to 1st Sym Tx queue */
  26 #define ADF_SYM_RX_QUEUE_STARTOFF               10
  27 #define ADF_ARB_REG_SLOT                        0x1000
  28 #define ADF_ARB_RINGSRVARBEN_OFFSET             0x19C
  29
  30 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
  31         ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \
  32         (ADF_ARB_REG_SLOT * index), value)
  33
  34 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
  35         uint32_t queue_size_bytes);
  36 static int qat_tx_queue_create(struct rte_cryptodev *dev,
  37         struct qat_queue *queue, uint8_t id, uint32_t nb_desc,
  38         int socket_id);
  39 static int qat_rx_queue_create(struct rte_cryptodev *dev,
  40         struct qat_queue *queue, uint8_t id, uint32_t nb_desc,
  41         int socket_id);
  42 static void qat_queue_delete(struct qat_queue *queue);
  43 static int qat_queue_create(struct rte_cryptodev *dev,
  44         struct qat_queue *queue, uint32_t nb_desc, uint8_t desc_size,
  45         int socket_id);
  46 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
  47         uint32_t *queue_size_for_csr);
  48 static void adf_configure_queues(struct qat_qp *queue);
  49 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr);
  50 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr);
  51
  52 static const struct rte_memzone *
  53 queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
  54                         int socket_id)
  55 {
  56         const struct rte_memzone *mz;
  57         unsigned memzone_flags = 0;
  58         const struct rte_memseg *ms;
  59
  60         PMD_INIT_FUNC_TRACE();
  61         mz = rte_memzone_lookup(queue_name);
  62         if (mz != 0) {
  63                 if (((size_t)queue_size <= mz->len) &&
  64                                 ((socket_id == SOCKET_ID_ANY) ||
  65                                         (socket_id == mz->socket_id))) {
  66                         PMD_DRV_LOG(DEBUG, "re-use memzone already "
  67                                         "allocated for %s", queue_name);
  68                         return mz;
  69                 }
  70
  71                 PMD_DRV_LOG(ERR, "Incompatible memzone already "
  72                                 "allocated %s, size %u, socket %d. "
  73                                 "Requested size %u, socket %u",
  74                                 queue_name, (uint32_t)mz->len,
  75                                 mz->socket_id, queue_size, socket_id);
  76                 return NULL;
  77         }
  78
  79         PMD_DRV_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
  80                                         queue_name, queue_size, socket_id);
  81         ms = rte_eal_get_physmem_layout();
  82         switch (ms[0].hugepage_sz) {
  83         case(RTE_PGSIZE_2M):
  84                 memzone_flags = RTE_MEMZONE_2MB;
  85         break;
  86         case(RTE_PGSIZE_1G):
  87                 memzone_flags = RTE_MEMZONE_1GB;
  88         break;
  89         case(RTE_PGSIZE_16M):
  90                 memzone_flags = RTE_MEMZONE_16MB;
  91         break;
  92         case(RTE_PGSIZE_16G):
  93                 memzone_flags = RTE_MEMZONE_16GB;
  94         break;
  95         default:
  96                 memzone_flags = RTE_MEMZONE_SIZE_HINT_ONLY;
  97         }
  98         return rte_memzone_reserve_aligned(queue_name, queue_size, socket_id,
  99                 memzone_flags, queue_size);
 100 }
 101
 102 int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
 103         const struct rte_cryptodev_qp_conf *qp_conf,
 104         int socket_id, struct rte_mempool *session_pool __rte_unused)
 105 {
 106         struct qat_qp *qp;
 107         struct rte_pci_device *pci_dev;
 108         int ret;
 109         char op_cookie_pool_name[RTE_RING_NAMESIZE];
 110         uint32_t i;
 111
 112         PMD_INIT_FUNC_TRACE();
 113
 114         /* If qp is already in use free ring memory and qp metadata. */
 115         if (dev->data->queue_pairs[queue_pair_id] != NULL) {
 116                 ret = qat_crypto_sym_qp_release(dev, queue_pair_id);
 117                 if (ret < 0)
 118                         return ret;
 119         }
 120
 121         if ((qp_conf->nb_descriptors > ADF_MAX_SYM_DESC) ||
 122                 (qp_conf->nb_descriptors < ADF_MIN_SYM_DESC)) {
 123                 PMD_DRV_LOG(ERR, "Can't create qp for %u descriptors",
 124                                 qp_conf->nb_descriptors);
 125                 return -EINVAL;
 126         }
 127
 128         pci_dev = RTE_DEV_TO_PCI(dev->device);
 129
 130         if (pci_dev->mem_resource[0].addr == NULL) {
 131                 PMD_DRV_LOG(ERR, "Could not find VF config space "
 132                                 "(UIO driver attached?).");
 133                 return -EINVAL;
 134         }
 135
 136         if (queue_pair_id >=
 137                         (ADF_NUM_SYM_QPS_PER_BUNDLE *
 138                                         ADF_NUM_BUNDLES_PER_DEV)) {
 139                 PMD_DRV_LOG(ERR, "qp_id %u invalid for this device",
 140                                 queue_pair_id);
 141                 return -EINVAL;
 142         }
 143         /* Allocate the queue pair data structure. */
 144         qp = rte_zmalloc("qat PMD qp metadata",
 145                         sizeof(*qp), RTE_CACHE_LINE_SIZE);
 146         if (qp == NULL) {
 147                 PMD_DRV_LOG(ERR, "Failed to alloc mem for qp struct");
 148                 return -ENOMEM;
 149         }
 150         qp->nb_descriptors = qp_conf->nb_descriptors;
 151         qp->op_cookies = rte_zmalloc("qat PMD op cookie pointer",
 152                         qp_conf->nb_descriptors * sizeof(*qp->op_cookies),
 153                         RTE_CACHE_LINE_SIZE);
 154         if (qp->op_cookies == NULL) {
 155                 PMD_DRV_LOG(ERR, "Failed to alloc mem for cookie");
 156                 rte_free(qp);
 157                 return -ENOMEM;
 158         }
 159
 160         qp->mmap_bar_addr = pci_dev->mem_resource[0].addr;
 161         qp->inflights16 = 0;
 162
 163         if (qat_tx_queue_create(dev, &(qp->tx_q),
 164                 queue_pair_id, qp_conf->nb_descriptors, socket_id) != 0) {
 165                 PMD_INIT_LOG(ERR, "Tx queue create failed "
 166                                 "queue_pair_id=%u", queue_pair_id);
 167                 goto create_err;
 168         }
 169
 170         if (qat_rx_queue_create(dev, &(qp->rx_q),
 171                 queue_pair_id, qp_conf->nb_descriptors, socket_id) != 0) {
 172                 PMD_DRV_LOG(ERR, "Rx queue create failed "
 173                                 "queue_pair_id=%hu", queue_pair_id);
 174                 qat_queue_delete(&(qp->tx_q));
 175                 goto create_err;
 176         }
 177
 178         adf_configure_queues(qp);
 179         adf_queue_arb_enable(&qp->tx_q, qp->mmap_bar_addr);
 180         snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, "%s_qp_op_%d_%hu",
 181                 pci_dev->driver->driver.name, dev->data->dev_id,
 182                 queue_pair_id);
 183
 184         qp->op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
 185         if (qp->op_cookie_pool == NULL)
 186                 qp->op_cookie_pool = rte_mempool_create(op_cookie_pool_name,
 187                                 qp->nb_descriptors,
 188                                 sizeof(struct qat_crypto_op_cookie), 64, 0,
 189                                 NULL, NULL, NULL, NULL, socket_id,
 190                                 0);
 191         if (!qp->op_cookie_pool) {
 192                 PMD_DRV_LOG(ERR, "QAT PMD Cannot create"
 193                                 " op mempool");
 194                 goto create_err;
 195         }
 196
 197         for (i = 0; i < qp->nb_descriptors; i++) {
 198                 if (rte_mempool_get(qp->op_cookie_pool, &qp->op_cookies[i])) {
 199                         PMD_DRV_LOG(ERR, "QAT PMD Cannot get op_cookie");
 200                         goto create_err;
 201                 }
 202
 203                 struct qat_crypto_op_cookie *sql_cookie =
 204                                 qp->op_cookies[i];
 205
 206                 sql_cookie->qat_sgl_src_phys_addr =
 207                                 rte_mempool_virt2iova(sql_cookie) +
 208                                 offsetof(struct qat_crypto_op_cookie,
 209                                 qat_sgl_list_src);
 210
 211                 sql_cookie->qat_sgl_dst_phys_addr =
 212                                 rte_mempool_virt2iova(sql_cookie) +
 213                                 offsetof(struct qat_crypto_op_cookie,
 214                                 qat_sgl_list_dst);
 215         }
 216
 217         struct qat_pmd_private *internals
 218                 = dev->data->dev_private;
 219         qp->qat_dev_gen = internals->qat_dev_gen;
 220
 221         dev->data->queue_pairs[queue_pair_id] = qp;
 222         return 0;
 223
 224 create_err:
 225         if (qp->op_cookie_pool)
 226                 rte_mempool_free(qp->op_cookie_pool);
 227         rte_free(qp->op_cookies);
 228         rte_free(qp);
 229         return -EFAULT;
 230 }
 231
 232 int qat_crypto_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
 233 {
 234         struct qat_qp *qp =
 235                         (struct qat_qp *)dev->data->queue_pairs[queue_pair_id];
 236         uint32_t i;
 237
 238         PMD_INIT_FUNC_TRACE();
 239         if (qp == NULL) {
 240                 PMD_DRV_LOG(DEBUG, "qp already freed");
 241                 return 0;
 242         }
 243
 244         /* Don't free memory if there are still responses to be processed */
 245         if (qp->inflights16 == 0) {
 246                 qat_queue_delete(&(qp->tx_q));
 247                 qat_queue_delete(&(qp->rx_q));
 248         } else {
 249                 return -EAGAIN;
 250         }
 251
 252         adf_queue_arb_disable(&(qp->tx_q), qp->mmap_bar_addr);
 253
 254         for (i = 0; i < qp->nb_descriptors; i++)
 255                 rte_mempool_put(qp->op_cookie_pool, qp->op_cookies[i]);
 256
 257         if (qp->op_cookie_pool)
 258                 rte_mempool_free(qp->op_cookie_pool);
 259
 260         rte_free(qp->op_cookies);
 261         rte_free(qp);
 262         dev->data->queue_pairs[queue_pair_id] = NULL;
 263         return 0;
 264 }
 265
 266 static int qat_tx_queue_create(struct rte_cryptodev *dev,
 267         struct qat_queue *queue, uint8_t qp_id,
 268         uint32_t nb_desc, int socket_id)
 269 {
 270         PMD_INIT_FUNC_TRACE();
 271         queue->hw_bundle_number = qp_id/ADF_NUM_SYM_QPS_PER_BUNDLE;
 272         queue->hw_queue_number = (qp_id%ADF_NUM_SYM_QPS_PER_BUNDLE) +
 273                                                 ADF_SYM_TX_QUEUE_STARTOFF;
 274         PMD_DRV_LOG(DEBUG, "TX ring for %u msgs: qp_id %d, bundle %u, ring %u",
 275                 nb_desc, qp_id, queue->hw_bundle_number,
 276                 queue->hw_queue_number);
 277
 278         return qat_queue_create(dev, queue, nb_desc,
 279                                 ADF_SYM_TX_RING_DESC_SIZE, socket_id);
 280 }
 281
 282 static int qat_rx_queue_create(struct rte_cryptodev *dev,
 283                 struct qat_queue *queue, uint8_t qp_id, uint32_t nb_desc,
 284                 int socket_id)
 285 {
 286         PMD_INIT_FUNC_TRACE();
 287         queue->hw_bundle_number = qp_id/ADF_NUM_SYM_QPS_PER_BUNDLE;
 288         queue->hw_queue_number = (qp_id%ADF_NUM_SYM_QPS_PER_BUNDLE) +
 289                                                 ADF_SYM_RX_QUEUE_STARTOFF;
 290
 291         PMD_DRV_LOG(DEBUG, "RX ring for %u msgs: qp id %d, bundle %u, ring %u",
 292                 nb_desc, qp_id, queue->hw_bundle_number,
 293                 queue->hw_queue_number);
 294         return qat_queue_create(dev, queue, nb_desc,
 295                                 ADF_SYM_RX_RING_DESC_SIZE, socket_id);
 296 }
 297
 298 static void qat_queue_delete(struct qat_queue *queue)
 299 {
 300         const struct rte_memzone *mz;
 301         int status = 0;
 302
 303         if (queue == NULL) {
 304                 PMD_DRV_LOG(DEBUG, "Invalid queue");
 305                 return;
 306         }
 307         mz = rte_memzone_lookup(queue->memz_name);
 308         if (mz != NULL) {
 309                 /* Write an unused pattern to the queue memory. */
 310                 memset(queue->base_addr, 0x7F, queue->queue_size);
 311                 status = rte_memzone_free(mz);
 312                 if (status != 0)
 313                         PMD_DRV_LOG(ERR, "Error %d on freeing queue %s",
 314                                         status, queue->memz_name);
 315         } else {
 316                 PMD_DRV_LOG(DEBUG, "queue %s doesn't exist",
 317                                 queue->memz_name);
 318         }
 319 }
 320
 321 static int
 322 qat_queue_create(struct rte_cryptodev *dev, struct qat_queue *queue,
 323                 uint32_t nb_desc, uint8_t desc_size, int socket_id)
 324 {
 325         uint64_t queue_base;
 326         void *io_addr;
 327         const struct rte_memzone *qp_mz;
 328         uint32_t queue_size_bytes = nb_desc*desc_size;
 329         struct rte_pci_device *pci_dev;
 330
 331         PMD_INIT_FUNC_TRACE();
 332         if (desc_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) {
 333                 PMD_DRV_LOG(ERR, "Invalid descriptor size %d", desc_size);
 334                 return -EINVAL;
 335         }
 336
 337         pci_dev = RTE_DEV_TO_PCI(dev->device);
 338
 339         /*
 340          * Allocate a memzone for the queue - create a unique name.
 341          */
 342         snprintf(queue->memz_name, sizeof(queue->memz_name), "%s_%s_%d_%d_%d",
 343                 pci_dev->driver->driver.name, "qp_mem", dev->data->dev_id,
 344                 queue->hw_bundle_number, queue->hw_queue_number);
 345         qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
 346                         socket_id);
 347         if (qp_mz == NULL) {
 348                 PMD_DRV_LOG(ERR, "Failed to allocate ring memzone");
 349                 return -ENOMEM;
 350         }
 351
 352         queue->base_addr = (char *)qp_mz->addr;
 353         queue->base_phys_addr = qp_mz->iova;
 354         if (qat_qp_check_queue_alignment(queue->base_phys_addr,
 355                         queue_size_bytes)) {
 356                 PMD_DRV_LOG(ERR, "Invalid alignment on queue create "
 357                                         " 0x%"PRIx64"\n",
 358                                         queue->base_phys_addr);
 359                 return -EFAULT;
 360         }
 361
 362         if (adf_verify_queue_size(desc_size, nb_desc, &(queue->queue_size))
 363                         != 0) {
 364                 PMD_DRV_LOG(ERR, "Invalid num inflights");
 365                 return -EINVAL;
 366         }
 367
 368         queue->max_inflights = ADF_MAX_INFLIGHTS(queue->queue_size,
 369                                         ADF_BYTES_TO_MSG_SIZE(desc_size));
 370         queue->modulo = ADF_RING_SIZE_MODULO(queue->queue_size);
 371         PMD_DRV_LOG(DEBUG, "RING size in CSR: %u, in bytes %u, nb msgs %u,"
 372                                 " msg_size %u, max_inflights %u modulo %u",
 373                                 queue->queue_size, queue_size_bytes,
 374                                 nb_desc, desc_size, queue->max_inflights,
 375                                 queue->modulo);
 376
 377         if (queue->max_inflights < 2) {
 378                 PMD_DRV_LOG(ERR, "Invalid num inflights");
 379                 return -EINVAL;
 380         }
 381         queue->head = 0;
 382         queue->tail = 0;
 383         queue->msg_size = desc_size;
 384
 385         /*
 386          * Write an unused pattern to the queue memory.
 387          */
 388         memset(queue->base_addr, 0x7F, queue_size_bytes);
 389
 390         queue_base = BUILD_RING_BASE_ADDR(queue->base_phys_addr,
 391                                         queue->queue_size);
 392
 393         io_addr = pci_dev->mem_resource[0].addr;
 394
 395         WRITE_CSR_RING_BASE(io_addr, queue->hw_bundle_number,
 396                         queue->hw_queue_number, queue_base);
 397         return 0;
 398 }
 399
 400 static int qat_qp_check_queue_alignment(uint64_t phys_addr,
 401                                         uint32_t queue_size_bytes)
 402 {
 403         PMD_INIT_FUNC_TRACE();
 404         if (((queue_size_bytes - 1) & phys_addr) != 0)
 405                 return -EINVAL;
 406         return 0;
 407 }
 408
 409 static int adf_verify_queue_size(uint32_t msg_size, uint32_t msg_num,
 410         uint32_t *p_queue_size_for_csr)
 411 {
 412         uint8_t i = ADF_MIN_RING_SIZE;
 413
 414         PMD_INIT_FUNC_TRACE();
 415         for (; i <= ADF_MAX_RING_SIZE; i++)
 416                 if ((msg_size * msg_num) ==
 417                                 (uint32_t)ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) {
 418                         *p_queue_size_for_csr = i;
 419                         return 0;
 420                 }
 421         PMD_DRV_LOG(ERR, "Invalid ring size %d", msg_size * msg_num);
 422         return -EINVAL;
 423 }
 424
 425 static void adf_queue_arb_enable(struct qat_queue *txq, void *base_addr)
 426 {
 427         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 428                                         (ADF_ARB_REG_SLOT *
 429                                                         txq->hw_bundle_number);
 430         uint32_t value;
 431
 432         PMD_INIT_FUNC_TRACE();
 433         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 434         value |= (0x01 << txq->hw_queue_number);
 435         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 436 }
 437
 438 static void adf_queue_arb_disable(struct qat_queue *txq, void *base_addr)
 439 {
 440         uint32_t arb_csr_offset =  ADF_ARB_RINGSRVARBEN_OFFSET +
 441                                         (ADF_ARB_REG_SLOT *
 442                                                         txq->hw_bundle_number);
 443         uint32_t value;
 444
 445         PMD_INIT_FUNC_TRACE();
 446         value = ADF_CSR_RD(base_addr, arb_csr_offset);
 447         value ^= (0x01 << txq->hw_queue_number);
 448         ADF_CSR_WR(base_addr, arb_csr_offset, value);
 449 }
 450
 451 static void adf_configure_queues(struct qat_qp *qp)
 452 {
 453         uint32_t queue_config;
 454         struct qat_queue *queue = &qp->tx_q;
 455
 456         PMD_INIT_FUNC_TRACE();
 457         queue_config = BUILD_RING_CONFIG(queue->queue_size);
 458
 459         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 460                         queue->hw_queue_number, queue_config);
 461
 462         queue = &qp->rx_q;
 463         queue_config =
 464                         BUILD_RESP_RING_CONFIG(queue->queue_size,
 465                                         ADF_RING_NEAR_WATERMARK_512,
 466                                         ADF_RING_NEAR_WATERMARK_0);
 467
 468         WRITE_CSR_RING_CONFIG(qp->mmap_bar_addr, queue->hw_bundle_number,
 469                         queue->hw_queue_number, queue_config);
 470 }