drivers/dma/ioat/ioat_dmadev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2021 Intel Corporation
   3  */
   4
   5 #include <rte_bus_pci.h>
   6 #include <rte_dmadev_pmd.h>
   7 #include <rte_malloc.h>
   8
   9 #include "ioat_internal.h"
  10
  11 static struct rte_pci_driver ioat_pmd_drv;
  12
  13 RTE_LOG_REGISTER_DEFAULT(ioat_pmd_logtype, INFO);
  14
  15 #define IOAT_PMD_NAME dmadev_ioat
  16 #define IOAT_PMD_NAME_STR RTE_STR(IOAT_PMD_NAME)
  17
  18 /* Create a DMA device. */
  19 static int
  20 ioat_dmadev_create(const char *name, struct rte_pci_device *dev)
  21 {
  22         static const struct rte_dma_dev_ops ioat_dmadev_ops = { };
  23
  24         struct rte_dma_dev *dmadev = NULL;
  25         struct ioat_dmadev *ioat = NULL;
  26         int retry = 0;
  27
  28         if (!name) {
  29                 IOAT_PMD_ERR("Invalid name of the device!");
  30                 return -EINVAL;
  31         }
  32
  33         /* Allocate device structure. */
  34         dmadev = rte_dma_pmd_allocate(name, dev->device.numa_node, sizeof(struct ioat_dmadev));
  35         if (dmadev == NULL) {
  36                 IOAT_PMD_ERR("Unable to allocate dma device");
  37                 return -ENOMEM;
  38         }
  39
  40         dmadev->device = &dev->device;
  41
  42         dmadev->fp_obj->dev_private = dmadev->data->dev_private;
  43
  44         dmadev->dev_ops = &ioat_dmadev_ops;
  45
  46         ioat = dmadev->data->dev_private;
  47         ioat->dmadev = dmadev;
  48         ioat->regs = dev->mem_resource[0].addr;
  49         ioat->doorbell = &ioat->regs->dmacount;
  50         ioat->qcfg.nb_desc = 0;
  51         ioat->desc_ring = NULL;
  52         ioat->version = ioat->regs->cbver;
  53
  54         /* Do device initialization - reset and set error behaviour. */
  55         if (ioat->regs->chancnt != 1)
  56                 IOAT_PMD_WARN("%s: Channel count == %d\n", __func__,
  57                                 ioat->regs->chancnt);
  58
  59         /* Locked by someone else. */
  60         if (ioat->regs->chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) {
  61                 IOAT_PMD_WARN("%s: Channel appears locked\n", __func__);
  62                 ioat->regs->chanctrl = 0;
  63         }
  64
  65         /* clear any previous errors */
  66         if (ioat->regs->chanerr != 0) {
  67                 uint32_t val = ioat->regs->chanerr;
  68                 ioat->regs->chanerr = val;
  69         }
  70
  71         ioat->regs->chancmd = IOAT_CHANCMD_SUSPEND;
  72         rte_delay_ms(1);
  73         ioat->regs->chancmd = IOAT_CHANCMD_RESET;
  74         rte_delay_ms(1);
  75         while (ioat->regs->chancmd & IOAT_CHANCMD_RESET) {
  76                 ioat->regs->chainaddr = 0;
  77                 rte_delay_ms(1);
  78                 if (++retry >= 200) {
  79                         IOAT_PMD_ERR("%s: cannot reset device. CHANCMD=%#"PRIx8
  80                                         ", CHANSTS=%#"PRIx64", CHANERR=%#"PRIx32"\n",
  81                                         __func__,
  82                                         ioat->regs->chancmd,
  83                                         ioat->regs->chansts,
  84                                         ioat->regs->chanerr);
  85                         rte_dma_pmd_release(name);
  86                         return -EIO;
  87                 }
  88         }
  89         ioat->regs->chanctrl = IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
  90                         IOAT_CHANCTRL_ERR_COMPLETION_EN;
  91
  92         dmadev->fp_obj->dev_private = ioat;
  93
  94         dmadev->state = RTE_DMA_DEV_READY;
  95
  96         return 0;
  97
  98 }
  99
 100 /* Destroy a DMA device. */
 101 static int
 102 ioat_dmadev_destroy(const char *name)
 103 {
 104         int ret;
 105
 106         if (!name) {
 107                 IOAT_PMD_ERR("Invalid device name");
 108                 return -EINVAL;
 109         }
 110
 111         ret = rte_dma_pmd_release(name);
 112         if (ret)
 113                 IOAT_PMD_DEBUG("Device cleanup failed");
 114
 115         return 0;
 116 }
 117
 118 /* Probe DMA device. */
 119 static int
 120 ioat_dmadev_probe(struct rte_pci_driver *drv, struct rte_pci_device *dev)
 121 {
 122         char name[32];
 123
 124         rte_pci_device_name(&dev->addr, name, sizeof(name));
 125         IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node);
 126
 127         dev->device.driver = &drv->driver;
 128         return ioat_dmadev_create(name, dev);
 129 }
 130
 131 /* Remove DMA device. */
 132 static int
 133 ioat_dmadev_remove(struct rte_pci_device *dev)
 134 {
 135         char name[32];
 136
 137         rte_pci_device_name(&dev->addr, name, sizeof(name));
 138
 139         IOAT_PMD_INFO("Closing %s on NUMA node %d",
 140                         name, dev->device.numa_node);
 141
 142         return ioat_dmadev_destroy(name);
 143 }
 144
 145 static const struct rte_pci_id pci_id_ioat_map[] = {
 146         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_SKX) },
 147         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX0) },
 148         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX1) },
 149         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX2) },
 150         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX3) },
 151         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX4) },
 152         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX5) },
 153         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX6) },
 154         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX7) },
 155         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXE) },
 156         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXF) },
 157         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_ICX) },
 158         { .vendor_id = 0, /* sentinel */ },
 159 };
 160
 161 static struct rte_pci_driver ioat_pmd_drv = {
 162         .id_table = pci_id_ioat_map,
 163         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 164         .probe = ioat_dmadev_probe,
 165         .remove = ioat_dmadev_remove,
 166 };
 167
 168 RTE_PMD_REGISTER_PCI(IOAT_PMD_NAME, ioat_pmd_drv);
 169 RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_NAME, pci_id_ioat_map);
 170 RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_NAME, "* igb_uio | uio_pci_generic | vfio-pci");