drivers/raw/ioat/idxd_pci.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <rte_bus_pci.h>
   6 #include <rte_memzone.h>
   7
   8 #include "ioat_private.h"
   9 #include "ioat_spec.h"
  10
  11 #define IDXD_VENDOR_ID          0x8086
  12 #define IDXD_DEVICE_ID_SPR      0x0B25
  13
  14 #define IDXD_PMD_RAWDEV_NAME_PCI rawdev_idxd_pci
  15
  16 const struct rte_pci_id pci_id_idxd_map[] = {
  17         { RTE_PCI_DEVICE(IDXD_VENDOR_ID, IDXD_DEVICE_ID_SPR) },
  18         { .vendor_id = 0, /* sentinel */ },
  19 };
  20
  21 static inline int
  22 idxd_pci_dev_command(struct idxd_rawdev *idxd, enum rte_idxd_cmds command)
  23 {
  24         uint8_t err_code;
  25         uint16_t qid = idxd->qid;
  26         int i = 0;
  27
  28         if (command >= idxd_disable_wq && command <= idxd_reset_wq)
  29                 qid = (1 << qid);
  30         rte_spinlock_lock(&idxd->u.pci->lk);
  31         idxd->u.pci->regs->cmd = (command << IDXD_CMD_SHIFT) | qid;
  32
  33         do {
  34                 rte_pause();
  35                 err_code = idxd->u.pci->regs->cmdstatus;
  36                 if (++i >= 1000) {
  37                         IOAT_PMD_ERR("Timeout waiting for command response from HW");
  38                         rte_spinlock_unlock(&idxd->u.pci->lk);
  39                         return err_code;
  40                 }
  41         } while (idxd->u.pci->regs->cmdstatus & CMDSTATUS_ACTIVE_MASK);
  42         rte_spinlock_unlock(&idxd->u.pci->lk);
  43
  44         return err_code & CMDSTATUS_ERR_MASK;
  45 }
  46
  47 static int
  48 idxd_is_wq_enabled(struct idxd_rawdev *idxd)
  49 {
  50         uint32_t state = idxd->u.pci->wq_regs[idxd->qid].wqcfg[WQ_STATE_IDX];
  51         return ((state >> WQ_STATE_SHIFT) & WQ_STATE_MASK) == 0x1;
  52 }
  53
  54 static void
  55 idxd_pci_dev_stop(struct rte_rawdev *dev)
  56 {
  57         struct idxd_rawdev *idxd = dev->dev_private;
  58         uint8_t err_code;
  59
  60         if (!idxd_is_wq_enabled(idxd)) {
  61                 IOAT_PMD_ERR("Work queue %d already disabled", idxd->qid);
  62                 return;
  63         }
  64
  65         err_code = idxd_pci_dev_command(idxd, idxd_disable_wq);
  66         if (err_code || idxd_is_wq_enabled(idxd)) {
  67                 IOAT_PMD_ERR("Failed disabling work queue %d, error code: %#x",
  68                                 idxd->qid, err_code);
  69                 return;
  70         }
  71         IOAT_PMD_DEBUG("Work queue %d disabled OK", idxd->qid);
  72 }
  73
  74 static int
  75 idxd_pci_dev_start(struct rte_rawdev *dev)
  76 {
  77         struct idxd_rawdev *idxd = dev->dev_private;
  78         uint8_t err_code;
  79
  80         if (idxd_is_wq_enabled(idxd)) {
  81                 IOAT_PMD_WARN("WQ %d already enabled", idxd->qid);
  82                 return 0;
  83         }
  84
  85         if (idxd->public.batch_ring == NULL) {
  86                 IOAT_PMD_ERR("WQ %d has not been fully configured", idxd->qid);
  87                 return -EINVAL;
  88         }
  89
  90         err_code = idxd_pci_dev_command(idxd, idxd_enable_wq);
  91         if (err_code || !idxd_is_wq_enabled(idxd)) {
  92                 IOAT_PMD_ERR("Failed enabling work queue %d, error code: %#x",
  93                                 idxd->qid, err_code);
  94                 return err_code == 0 ? -1 : err_code;
  95         }
  96
  97         IOAT_PMD_DEBUG("Work queue %d enabled OK", idxd->qid);
  98
  99         return 0;
 100 }
 101
 102 static const struct rte_rawdev_ops idxd_pci_ops = {
 103                 .dev_close = idxd_rawdev_close,
 104                 .dev_selftest = ioat_rawdev_test,
 105                 .dump = idxd_dev_dump,
 106                 .dev_configure = idxd_dev_configure,
 107                 .dev_start = idxd_pci_dev_start,
 108                 .dev_stop = idxd_pci_dev_stop,
 109                 .dev_info_get = idxd_dev_info_get,
 110                 .xstats_get = ioat_xstats_get,
 111                 .xstats_get_names = ioat_xstats_get_names,
 112                 .xstats_reset = ioat_xstats_reset,
 113 };
 114
 115 /* each portal uses 4 x 4k pages */
 116 #define IDXD_PORTAL_SIZE (4096 * 4)
 117
 118 static int
 119 init_pci_device(struct rte_pci_device *dev, struct idxd_rawdev *idxd)
 120 {
 121         struct idxd_pci_common *pci;
 122         uint8_t nb_groups, nb_engines, nb_wqs;
 123         uint16_t grp_offset, wq_offset; /* how far into bar0 the regs are */
 124         uint16_t wq_size, total_wq_size;
 125         uint8_t lg2_max_batch, lg2_max_copy_size;
 126         unsigned int i, err_code;
 127
 128         pci = malloc(sizeof(*pci));
 129         if (pci == NULL) {
 130                 IOAT_PMD_ERR("%s: Can't allocate memory", __func__);
 131                 goto err;
 132         }
 133         rte_spinlock_init(&pci->lk);
 134
 135         /* assign the bar registers, and then configure device */
 136         pci->regs = dev->mem_resource[0].addr;
 137         grp_offset = (uint16_t)pci->regs->offsets[0];
 138         pci->grp_regs = RTE_PTR_ADD(pci->regs, grp_offset * 0x100);
 139         wq_offset = (uint16_t)(pci->regs->offsets[0] >> 16);
 140         pci->wq_regs = RTE_PTR_ADD(pci->regs, wq_offset * 0x100);
 141         pci->portals = dev->mem_resource[2].addr;
 142
 143         /* sanity check device status */
 144         if (pci->regs->gensts & GENSTS_DEV_STATE_MASK) {
 145                 /* need function-level-reset (FLR) or is enabled */
 146                 IOAT_PMD_ERR("Device status is not disabled, cannot init");
 147                 goto err;
 148         }
 149         if (pci->regs->cmdstatus & CMDSTATUS_ACTIVE_MASK) {
 150                 /* command in progress */
 151                 IOAT_PMD_ERR("Device has a command in progress, cannot init");
 152                 goto err;
 153         }
 154
 155         /* read basic info about the hardware for use when configuring */
 156         nb_groups = (uint8_t)pci->regs->grpcap;
 157         nb_engines = (uint8_t)pci->regs->engcap;
 158         nb_wqs = (uint8_t)(pci->regs->wqcap >> 16);
 159         total_wq_size = (uint16_t)pci->regs->wqcap;
 160         lg2_max_copy_size = (uint8_t)(pci->regs->gencap >> 16) & 0x1F;
 161         lg2_max_batch = (uint8_t)(pci->regs->gencap >> 21) & 0x0F;
 162
 163         IOAT_PMD_DEBUG("nb_groups = %u, nb_engines = %u, nb_wqs = %u",
 164                         nb_groups, nb_engines, nb_wqs);
 165
 166         /* zero out any old config */
 167         for (i = 0; i < nb_groups; i++) {
 168                 pci->grp_regs[i].grpengcfg = 0;
 169                 pci->grp_regs[i].grpwqcfg[0] = 0;
 170         }
 171         for (i = 0; i < nb_wqs; i++)
 172                 pci->wq_regs[i].wqcfg[0] = 0;
 173
 174         /* put each engine into a separate group to avoid reordering */
 175         if (nb_groups > nb_engines)
 176                 nb_groups = nb_engines;
 177         if (nb_groups < nb_engines)
 178                 nb_engines = nb_groups;
 179
 180         /* assign engines to groups, round-robin style */
 181         for (i = 0; i < nb_engines; i++) {
 182                 IOAT_PMD_DEBUG("Assigning engine %u to group %u",
 183                                 i, i % nb_groups);
 184                 pci->grp_regs[i % nb_groups].grpengcfg |= (1ULL << i);
 185         }
 186
 187         /* now do the same for queues and give work slots to each queue */
 188         wq_size = total_wq_size / nb_wqs;
 189         IOAT_PMD_DEBUG("Work queue size = %u, max batch = 2^%u, max copy = 2^%u",
 190                         wq_size, lg2_max_batch, lg2_max_copy_size);
 191         for (i = 0; i < nb_wqs; i++) {
 192                 /* add engine "i" to a group */
 193                 IOAT_PMD_DEBUG("Assigning work queue %u to group %u",
 194                                 i, i % nb_groups);
 195                 pci->grp_regs[i % nb_groups].grpwqcfg[0] |= (1ULL << i);
 196                 /* now configure it, in terms of size, max batch, mode */
 197                 pci->wq_regs[i].wqcfg[WQ_SIZE_IDX] = wq_size;
 198                 pci->wq_regs[i].wqcfg[WQ_MODE_IDX] = (1 << WQ_PRIORITY_SHIFT) |
 199                                 WQ_MODE_DEDICATED;
 200                 pci->wq_regs[i].wqcfg[WQ_SIZES_IDX] = lg2_max_copy_size |
 201                                 (lg2_max_batch << WQ_BATCH_SZ_SHIFT);
 202         }
 203
 204         /* dump the group configuration to output */
 205         for (i = 0; i < nb_groups; i++) {
 206                 IOAT_PMD_DEBUG("## Group %d", i);
 207                 IOAT_PMD_DEBUG("    GRPWQCFG: %"PRIx64, pci->grp_regs[i].grpwqcfg[0]);
 208                 IOAT_PMD_DEBUG("    GRPENGCFG: %"PRIx64, pci->grp_regs[i].grpengcfg);
 209                 IOAT_PMD_DEBUG("    GRPFLAGS: %"PRIx32, pci->grp_regs[i].grpflags);
 210         }
 211
 212         idxd->u.pci = pci;
 213         idxd->max_batches = wq_size;
 214
 215         /* enable the device itself */
 216         err_code = idxd_pci_dev_command(idxd, idxd_enable_dev);
 217         if (err_code) {
 218                 IOAT_PMD_ERR("Error enabling device: code %#x", err_code);
 219                 return err_code;
 220         }
 221         IOAT_PMD_DEBUG("IDXD Device enabled OK");
 222
 223         return nb_wqs;
 224
 225 err:
 226         free(pci);
 227         return -1;
 228 }
 229
 230 static int
 231 idxd_rawdev_probe_pci(struct rte_pci_driver *drv, struct rte_pci_device *dev)
 232 {
 233         struct idxd_rawdev idxd = {{0}}; /* Double {} to avoid error on BSD12 */
 234         uint8_t nb_wqs;
 235         int qid, ret = 0;
 236         char name[PCI_PRI_STR_SIZE];
 237
 238         rte_pci_device_name(&dev->addr, name, sizeof(name));
 239         IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node);
 240         dev->device.driver = &drv->driver;
 241
 242         ret = init_pci_device(dev, &idxd);
 243         if (ret < 0) {
 244                 IOAT_PMD_ERR("Error initializing PCI hardware");
 245                 return ret;
 246         }
 247         nb_wqs = (uint8_t)ret;
 248
 249         /* set up one device for each queue */
 250         for (qid = 0; qid < nb_wqs; qid++) {
 251                 char qname[32];
 252
 253                 /* add the queue number to each device name */
 254                 snprintf(qname, sizeof(qname), "%s-q%d", name, qid);
 255                 idxd.qid = qid;
 256                 idxd.public.portal = RTE_PTR_ADD(idxd.u.pci->portals,
 257                                 qid * IDXD_PORTAL_SIZE);
 258                 if (idxd_is_wq_enabled(&idxd))
 259                         IOAT_PMD_ERR("Error, WQ %u seems enabled", qid);
 260                 ret = idxd_rawdev_create(qname, &dev->device,
 261                                 &idxd, &idxd_pci_ops);
 262                 if (ret != 0) {
 263                         IOAT_PMD_ERR("Failed to create rawdev %s", name);
 264                         if (qid == 0) /* if no devices using this, free pci */
 265                                 free(idxd.u.pci);
 266                         return ret;
 267                 }
 268         }
 269
 270         return 0;
 271 }
 272
 273 static int
 274 idxd_rawdev_destroy(const char *name)
 275 {
 276         int ret;
 277         uint8_t err_code;
 278         struct rte_rawdev *rdev;
 279         struct idxd_rawdev *idxd;
 280
 281         if (!name) {
 282                 IOAT_PMD_ERR("Invalid device name");
 283                 return -EINVAL;
 284         }
 285
 286         rdev = rte_rawdev_pmd_get_named_dev(name);
 287         if (!rdev) {
 288                 IOAT_PMD_ERR("Invalid device name (%s)", name);
 289                 return -EINVAL;
 290         }
 291
 292         idxd = rdev->dev_private;
 293
 294         /* disable the device */
 295         err_code = idxd_pci_dev_command(idxd, idxd_disable_dev);
 296         if (err_code) {
 297                 IOAT_PMD_ERR("Error disabling device: code %#x", err_code);
 298                 return err_code;
 299         }
 300         IOAT_PMD_DEBUG("IDXD Device disabled OK");
 301
 302         /* free device memory */
 303         if (rdev->dev_private != NULL) {
 304                 IOAT_PMD_DEBUG("Freeing device driver memory");
 305                 rdev->dev_private = NULL;
 306                 rte_free(idxd->public.batch_ring);
 307                 rte_free(idxd->public.hdl_ring);
 308                 rte_memzone_free(idxd->mz);
 309         }
 310
 311         /* rte_rawdev_close is called by pmd_release */
 312         ret = rte_rawdev_pmd_release(rdev);
 313         if (ret)
 314                 IOAT_PMD_DEBUG("Device cleanup failed");
 315
 316         return 0;
 317 }
 318
 319 static int
 320 idxd_rawdev_remove_pci(struct rte_pci_device *dev)
 321 {
 322         char name[PCI_PRI_STR_SIZE];
 323         int ret = 0;
 324
 325         rte_pci_device_name(&dev->addr, name, sizeof(name));
 326
 327         IOAT_PMD_INFO("Closing %s on NUMA node %d",
 328                         name, dev->device.numa_node);
 329
 330         ret = idxd_rawdev_destroy(name);
 331
 332         return ret;
 333 }
 334
 335 struct rte_pci_driver idxd_pmd_drv_pci = {
 336         .id_table = pci_id_idxd_map,
 337         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 338         .probe = idxd_rawdev_probe_pci,
 339         .remove = idxd_rawdev_remove_pci,
 340 };
 341
 342 RTE_PMD_REGISTER_PCI(IDXD_PMD_RAWDEV_NAME_PCI, idxd_pmd_drv_pci);
 343 RTE_PMD_REGISTER_PCI_TABLE(IDXD_PMD_RAWDEV_NAME_PCI, pci_id_idxd_map);
 344 RTE_PMD_REGISTER_KMOD_DEP(IDXD_PMD_RAWDEV_NAME_PCI,
 345                           "* igb_uio | uio_pci_generic | vfio-pci");