drivers/dma/ioat/ioat_dmadev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2021 Intel Corporation
   3  */
   4
   5 #include <rte_bus_pci.h>
   6 #include <rte_dmadev_pmd.h>
   7 #include <rte_malloc.h>
   8
   9 #include "ioat_internal.h"
  10
  11 static struct rte_pci_driver ioat_pmd_drv;
  12
  13 RTE_LOG_REGISTER_DEFAULT(ioat_pmd_logtype, INFO);
  14
  15 #define IOAT_PMD_NAME dmadev_ioat
  16 #define IOAT_PMD_NAME_STR RTE_STR(IOAT_PMD_NAME)
  17
  18 /* Dump DMA device info. */
  19 static int
  20 __dev_dump(void *dev_private, FILE *f)
  21 {
  22         struct ioat_dmadev *ioat = dev_private;
  23         uint64_t chansts_masked = ioat->regs->chansts & IOAT_CHANSTS_STATUS;
  24         uint32_t chanerr = ioat->regs->chanerr;
  25         uint64_t mask = (ioat->qcfg.nb_desc - 1);
  26         char ver = ioat->version;
  27         fprintf(f, "========= IOAT =========\n");
  28         fprintf(f, "  IOAT version: %d.%d\n", ver >> 4, ver & 0xF);
  29         fprintf(f, "  Channel status: %s [0x%"PRIx64"]\n",
  30                         chansts_readable[chansts_masked], chansts_masked);
  31         fprintf(f, "  ChainADDR: 0x%"PRIu64"\n", ioat->regs->chainaddr);
  32         if (chanerr == 0) {
  33                 fprintf(f, "  No Channel Errors\n");
  34         } else {
  35                 fprintf(f, "  ChanERR: 0x%"PRIu32"\n", chanerr);
  36                 if (chanerr & IOAT_CHANERR_INVALID_SRC_ADDR_MASK)
  37                         fprintf(f, "    Invalid Source Address\n");
  38                 if (chanerr & IOAT_CHANERR_INVALID_DST_ADDR_MASK)
  39                         fprintf(f, "    Invalid Destination Address\n");
  40                 if (chanerr & IOAT_CHANERR_INVALID_LENGTH_MASK)
  41                         fprintf(f, "    Invalid Descriptor Length\n");
  42                 if (chanerr & IOAT_CHANERR_DESCRIPTOR_READ_ERROR_MASK)
  43                         fprintf(f, "    Descriptor Read Error\n");
  44                 if ((chanerr & ~(IOAT_CHANERR_INVALID_SRC_ADDR_MASK |
  45                                 IOAT_CHANERR_INVALID_DST_ADDR_MASK |
  46                                 IOAT_CHANERR_INVALID_LENGTH_MASK |
  47                                 IOAT_CHANERR_DESCRIPTOR_READ_ERROR_MASK)) != 0)
  48                         fprintf(f, "    Unknown Error(s)\n");
  49         }
  50         fprintf(f, "== Private Data ==\n");
  51         fprintf(f, "  Config: { ring_size: %u }\n", ioat->qcfg.nb_desc);
  52         fprintf(f, "  Status: 0x%"PRIx64"\n", ioat->status);
  53         fprintf(f, "  Status IOVA: 0x%"PRIx64"\n", ioat->status_addr);
  54         fprintf(f, "  Status ADDR: %p\n", &ioat->status);
  55         fprintf(f, "  Ring IOVA: 0x%"PRIx64"\n", ioat->ring_addr);
  56         fprintf(f, "  Ring ADDR: 0x%"PRIx64"\n", ioat->desc_ring[0].next-64);
  57         fprintf(f, "  Next write: %"PRIu16"\n", ioat->next_write);
  58         fprintf(f, "  Next read: %"PRIu16"\n", ioat->next_read);
  59         struct ioat_dma_hw_desc *desc_ring = &ioat->desc_ring[(ioat->next_write - 1) & mask];
  60         fprintf(f, "  Last Descriptor Written {\n");
  61         fprintf(f, "    Size: %"PRIu32"\n", desc_ring->size);
  62         fprintf(f, "    Control: 0x%"PRIx32"\n", desc_ring->u.control_raw);
  63         fprintf(f, "    Src: 0x%"PRIx64"\n", desc_ring->src_addr);
  64         fprintf(f, "    Dest: 0x%"PRIx64"\n", desc_ring->dest_addr);
  65         fprintf(f, "    Next: 0x%"PRIx64"\n", desc_ring->next);
  66         fprintf(f, "  }\n");
  67         fprintf(f, "  Next Descriptor {\n");
  68         fprintf(f, "    Size: %"PRIu32"\n", ioat->desc_ring[ioat->next_read & mask].size);
  69         fprintf(f, "    Src: 0x%"PRIx64"\n", ioat->desc_ring[ioat->next_read & mask].src_addr);
  70         fprintf(f, "    Dest: 0x%"PRIx64"\n", ioat->desc_ring[ioat->next_read & mask].dest_addr);
  71         fprintf(f, "    Next: 0x%"PRIx64"\n", ioat->desc_ring[ioat->next_read & mask].next);
  72         fprintf(f, "  }\n");
  73
  74         return 0;
  75 }
  76
  77 /* Public wrapper for dump. */
  78 static int
  79 ioat_dev_dump(const struct rte_dma_dev *dev, FILE *f)
  80 {
  81         return __dev_dump(dev->fp_obj->dev_private, f);
  82 }
  83
  84 /* Create a DMA device. */
  85 static int
  86 ioat_dmadev_create(const char *name, struct rte_pci_device *dev)
  87 {
  88         static const struct rte_dma_dev_ops ioat_dmadev_ops = {
  89                 .dev_dump = ioat_dev_dump,
  90         };
  91
  92         struct rte_dma_dev *dmadev = NULL;
  93         struct ioat_dmadev *ioat = NULL;
  94         int retry = 0;
  95
  96         if (!name) {
  97                 IOAT_PMD_ERR("Invalid name of the device!");
  98                 return -EINVAL;
  99         }
 100
 101         /* Allocate device structure. */
 102         dmadev = rte_dma_pmd_allocate(name, dev->device.numa_node, sizeof(struct ioat_dmadev));
 103         if (dmadev == NULL) {
 104                 IOAT_PMD_ERR("Unable to allocate dma device");
 105                 return -ENOMEM;
 106         }
 107
 108         dmadev->device = &dev->device;
 109
 110         dmadev->fp_obj->dev_private = dmadev->data->dev_private;
 111
 112         dmadev->dev_ops = &ioat_dmadev_ops;
 113
 114         ioat = dmadev->data->dev_private;
 115         ioat->dmadev = dmadev;
 116         ioat->regs = dev->mem_resource[0].addr;
 117         ioat->doorbell = &ioat->regs->dmacount;
 118         ioat->qcfg.nb_desc = 0;
 119         ioat->desc_ring = NULL;
 120         ioat->version = ioat->regs->cbver;
 121
 122         /* Do device initialization - reset and set error behaviour. */
 123         if (ioat->regs->chancnt != 1)
 124                 IOAT_PMD_WARN("%s: Channel count == %d\n", __func__,
 125                                 ioat->regs->chancnt);
 126
 127         /* Locked by someone else. */
 128         if (ioat->regs->chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) {
 129                 IOAT_PMD_WARN("%s: Channel appears locked\n", __func__);
 130                 ioat->regs->chanctrl = 0;
 131         }
 132
 133         /* clear any previous errors */
 134         if (ioat->regs->chanerr != 0) {
 135                 uint32_t val = ioat->regs->chanerr;
 136                 ioat->regs->chanerr = val;
 137         }
 138
 139         ioat->regs->chancmd = IOAT_CHANCMD_SUSPEND;
 140         rte_delay_ms(1);
 141         ioat->regs->chancmd = IOAT_CHANCMD_RESET;
 142         rte_delay_ms(1);
 143         while (ioat->regs->chancmd & IOAT_CHANCMD_RESET) {
 144                 ioat->regs->chainaddr = 0;
 145                 rte_delay_ms(1);
 146                 if (++retry >= 200) {
 147                         IOAT_PMD_ERR("%s: cannot reset device. CHANCMD=%#"PRIx8
 148                                         ", CHANSTS=%#"PRIx64", CHANERR=%#"PRIx32"\n",
 149                                         __func__,
 150                                         ioat->regs->chancmd,
 151                                         ioat->regs->chansts,
 152                                         ioat->regs->chanerr);
 153                         rte_dma_pmd_release(name);
 154                         return -EIO;
 155                 }
 156         }
 157         ioat->regs->chanctrl = IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 158                         IOAT_CHANCTRL_ERR_COMPLETION_EN;
 159
 160         dmadev->fp_obj->dev_private = ioat;
 161
 162         dmadev->state = RTE_DMA_DEV_READY;
 163
 164         return 0;
 165
 166 }
 167
 168 /* Destroy a DMA device. */
 169 static int
 170 ioat_dmadev_destroy(const char *name)
 171 {
 172         int ret;
 173
 174         if (!name) {
 175                 IOAT_PMD_ERR("Invalid device name");
 176                 return -EINVAL;
 177         }
 178
 179         ret = rte_dma_pmd_release(name);
 180         if (ret)
 181                 IOAT_PMD_DEBUG("Device cleanup failed");
 182
 183         return 0;
 184 }
 185
 186 /* Probe DMA device. */
 187 static int
 188 ioat_dmadev_probe(struct rte_pci_driver *drv, struct rte_pci_device *dev)
 189 {
 190         char name[32];
 191
 192         rte_pci_device_name(&dev->addr, name, sizeof(name));
 193         IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node);
 194
 195         dev->device.driver = &drv->driver;
 196         return ioat_dmadev_create(name, dev);
 197 }
 198
 199 /* Remove DMA device. */
 200 static int
 201 ioat_dmadev_remove(struct rte_pci_device *dev)
 202 {
 203         char name[32];
 204
 205         rte_pci_device_name(&dev->addr, name, sizeof(name));
 206
 207         IOAT_PMD_INFO("Closing %s on NUMA node %d",
 208                         name, dev->device.numa_node);
 209
 210         return ioat_dmadev_destroy(name);
 211 }
 212
 213 static const struct rte_pci_id pci_id_ioat_map[] = {
 214         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_SKX) },
 215         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX0) },
 216         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX1) },
 217         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX2) },
 218         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX3) },
 219         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX4) },
 220         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX5) },
 221         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX6) },
 222         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX7) },
 223         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXE) },
 224         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXF) },
 225         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_ICX) },
 226         { .vendor_id = 0, /* sentinel */ },
 227 };
 228
 229 static struct rte_pci_driver ioat_pmd_drv = {
 230         .id_table = pci_id_ioat_map,
 231         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 232         .probe = ioat_dmadev_probe,
 233         .remove = ioat_dmadev_remove,
 234 };
 235
 236 RTE_PMD_REGISTER_PCI(IOAT_PMD_NAME, ioat_pmd_drv);
 237 RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_NAME, pci_id_ioat_map);
 238 RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_NAME, "* igb_uio | uio_pci_generic | vfio-pci");