drivers/raw/ioat/ioat_rawdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <rte_cycles.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_memzone.h>
   8 #include <rte_string_fns.h>
   9 #include <rte_rawdev_pmd.h>
  10
  11 #include "rte_ioat_rawdev.h"
  12 #include "ioat_spec.h"
  13 #include "ioat_private.h"
  14
  15 static struct rte_pci_driver ioat_pmd_drv;
  16
  17 #define IOAT_VENDOR_ID          0x8086
  18 #define IOAT_DEVICE_ID_SKX      0x2021
  19 #define IOAT_DEVICE_ID_BDX0     0x6f20
  20 #define IOAT_DEVICE_ID_BDX1     0x6f21
  21 #define IOAT_DEVICE_ID_BDX2     0x6f22
  22 #define IOAT_DEVICE_ID_BDX3     0x6f23
  23 #define IOAT_DEVICE_ID_BDX4     0x6f24
  24 #define IOAT_DEVICE_ID_BDX5     0x6f25
  25 #define IOAT_DEVICE_ID_BDX6     0x6f26
  26 #define IOAT_DEVICE_ID_BDX7     0x6f27
  27 #define IOAT_DEVICE_ID_BDXE     0x6f2E
  28 #define IOAT_DEVICE_ID_BDXF     0x6f2F
  29 #define IOAT_DEVICE_ID_ICX      0x0b00
  30
  31 RTE_LOG_REGISTER(ioat_pmd_logtype, rawdev.ioat, INFO);
  32
  33 #define DESC_SZ sizeof(struct rte_ioat_generic_hw_desc)
  34 #define COMPLETION_SZ sizeof(__m128i)
  35
  36 static int
  37 ioat_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config,
  38                 size_t config_size)
  39 {
  40         struct rte_ioat_rawdev_config *params = config;
  41         struct rte_ioat_rawdev *ioat = dev->dev_private;
  42         char mz_name[RTE_MEMZONE_NAMESIZE];
  43         unsigned short i;
  44
  45         if (dev->started)
  46                 return -EBUSY;
  47
  48         if (params == NULL || config_size != sizeof(*params))
  49                 return -EINVAL;
  50
  51         if (params->ring_size > 4096 || params->ring_size < 64 ||
  52                         !rte_is_power_of_2(params->ring_size))
  53                 return -EINVAL;
  54
  55         ioat->ring_size = params->ring_size;
  56         ioat->hdls_disable = params->hdls_disable;
  57         if (ioat->desc_ring != NULL) {
  58                 rte_memzone_free(ioat->desc_mz);
  59                 ioat->desc_ring = NULL;
  60                 ioat->desc_mz = NULL;
  61         }
  62
  63         /* allocate one block of memory for both descriptors
  64          * and completion handles.
  65          */
  66         snprintf(mz_name, sizeof(mz_name), "rawdev%u_desc_ring", dev->dev_id);
  67         ioat->desc_mz = rte_memzone_reserve(mz_name,
  68                         (DESC_SZ + COMPLETION_SZ) * ioat->ring_size,
  69                         dev->device->numa_node, RTE_MEMZONE_IOVA_CONTIG);
  70         if (ioat->desc_mz == NULL)
  71                 return -ENOMEM;
  72         ioat->desc_ring = ioat->desc_mz->addr;
  73         ioat->hdls = (void *)&ioat->desc_ring[ioat->ring_size];
  74
  75         ioat->ring_addr = ioat->desc_mz->iova;
  76
  77         /* configure descriptor ring - each one points to next */
  78         for (i = 0; i < ioat->ring_size; i++) {
  79                 ioat->desc_ring[i].next = ioat->ring_addr +
  80                                 (((i + 1) % ioat->ring_size) * DESC_SZ);
  81         }
  82
  83         return 0;
  84 }
  85
  86 static int
  87 ioat_dev_start(struct rte_rawdev *dev)
  88 {
  89         struct rte_ioat_rawdev *ioat = dev->dev_private;
  90
  91         if (ioat->ring_size == 0 || ioat->desc_ring == NULL)
  92                 return -EBUSY;
  93
  94         /* inform hardware of where the descriptor ring is */
  95         ioat->regs->chainaddr = ioat->ring_addr;
  96         /* inform hardware of where to write the status/completions */
  97         ioat->regs->chancmp = ioat->status_addr;
  98
  99         /* prime the status register to be set to the last element */
 100         ioat->status =  ioat->ring_addr + ((ioat->ring_size - 1) * DESC_SZ);
 101         return 0;
 102 }
 103
 104 static void
 105 ioat_dev_stop(struct rte_rawdev *dev)
 106 {
 107         RTE_SET_USED(dev);
 108 }
 109
 110 static int
 111 ioat_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info,
 112                 size_t dev_info_size)
 113 {
 114         struct rte_ioat_rawdev_config *cfg = dev_info;
 115         struct rte_ioat_rawdev *ioat = dev->dev_private;
 116
 117         if (dev_info == NULL || dev_info_size != sizeof(*cfg))
 118                 return -EINVAL;
 119
 120         cfg->ring_size = ioat->ring_size;
 121         cfg->hdls_disable = ioat->hdls_disable;
 122         return 0;
 123 }
 124
 125 static const char * const xstat_names[] = {
 126                 "failed_enqueues", "successful_enqueues",
 127                 "copies_started", "copies_completed"
 128 };
 129
 130 static int
 131 ioat_xstats_get(const struct rte_rawdev *dev, const unsigned int ids[],
 132                 uint64_t values[], unsigned int n)
 133 {
 134         const struct rte_ioat_rawdev *ioat = dev->dev_private;
 135         unsigned int i;
 136
 137         for (i = 0; i < n; i++) {
 138                 switch (ids[i]) {
 139                 case 0: values[i] = ioat->enqueue_failed; break;
 140                 case 1: values[i] = ioat->enqueued; break;
 141                 case 2: values[i] = ioat->started; break;
 142                 case 3: values[i] = ioat->completed; break;
 143                 default: values[i] = 0; break;
 144                 }
 145         }
 146         return n;
 147 }
 148
 149 static int
 150 ioat_xstats_get_names(const struct rte_rawdev *dev,
 151                 struct rte_rawdev_xstats_name *names,
 152                 unsigned int size)
 153 {
 154         unsigned int i;
 155
 156         RTE_SET_USED(dev);
 157         if (size < RTE_DIM(xstat_names))
 158                 return RTE_DIM(xstat_names);
 159
 160         for (i = 0; i < RTE_DIM(xstat_names); i++)
 161                 strlcpy(names[i].name, xstat_names[i], sizeof(names[i]));
 162
 163         return RTE_DIM(xstat_names);
 164 }
 165
 166 static int
 167 ioat_xstats_reset(struct rte_rawdev *dev, const uint32_t *ids, uint32_t nb_ids)
 168 {
 169         struct rte_ioat_rawdev *ioat = dev->dev_private;
 170         unsigned int i;
 171
 172         if (!ids) {
 173                 ioat->enqueue_failed = 0;
 174                 ioat->enqueued = 0;
 175                 ioat->started = 0;
 176                 ioat->completed = 0;
 177                 return 0;
 178         }
 179
 180         for (i = 0; i < nb_ids; i++) {
 181                 switch (ids[i]) {
 182                 case 0:
 183                         ioat->enqueue_failed = 0;
 184                         break;
 185                 case 1:
 186                         ioat->enqueued = 0;
 187                         break;
 188                 case 2:
 189                         ioat->started = 0;
 190                         break;
 191                 case 3:
 192                         ioat->completed = 0;
 193                         break;
 194                 default:
 195                         IOAT_PMD_WARN("Invalid xstat id - cannot reset value");
 196                         break;
 197                 }
 198         }
 199
 200         return 0;
 201 }
 202
 203 static int
 204 ioat_dev_close(struct rte_rawdev *dev __rte_unused)
 205 {
 206         return 0;
 207 }
 208
 209 extern int ioat_rawdev_test(uint16_t dev_id);
 210
 211 static int
 212 ioat_rawdev_create(const char *name, struct rte_pci_device *dev)
 213 {
 214         static const struct rte_rawdev_ops ioat_rawdev_ops = {
 215                         .dev_configure = ioat_dev_configure,
 216                         .dev_start = ioat_dev_start,
 217                         .dev_stop = ioat_dev_stop,
 218                         .dev_close = ioat_dev_close,
 219                         .dev_info_get = ioat_dev_info_get,
 220                         .xstats_get = ioat_xstats_get,
 221                         .xstats_get_names = ioat_xstats_get_names,
 222                         .xstats_reset = ioat_xstats_reset,
 223                         .dev_selftest = ioat_rawdev_test,
 224         };
 225
 226         struct rte_rawdev *rawdev = NULL;
 227         struct rte_ioat_rawdev *ioat = NULL;
 228         const struct rte_memzone *mz = NULL;
 229         char mz_name[RTE_MEMZONE_NAMESIZE];
 230         int ret = 0;
 231         int retry = 0;
 232
 233         if (!name) {
 234                 IOAT_PMD_ERR("Invalid name of the device!");
 235                 ret = -EINVAL;
 236                 goto cleanup;
 237         }
 238
 239         /* Allocate device structure */
 240         rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct rte_ioat_rawdev),
 241                                          dev->device.numa_node);
 242         if (rawdev == NULL) {
 243                 IOAT_PMD_ERR("Unable to allocate raw device");
 244                 ret = -ENOMEM;
 245                 goto cleanup;
 246         }
 247
 248         snprintf(mz_name, sizeof(mz_name), "rawdev%u_private", rawdev->dev_id);
 249         mz = rte_memzone_reserve(mz_name, sizeof(struct rte_ioat_rawdev),
 250                         dev->device.numa_node, RTE_MEMZONE_IOVA_CONTIG);
 251         if (mz == NULL) {
 252                 IOAT_PMD_ERR("Unable to reserve memzone for private data\n");
 253                 ret = -ENOMEM;
 254                 goto cleanup;
 255         }
 256
 257         rawdev->dev_private = mz->addr;
 258         rawdev->dev_ops = &ioat_rawdev_ops;
 259         rawdev->device = &dev->device;
 260         rawdev->driver_name = dev->device.driver->name;
 261
 262         ioat = rawdev->dev_private;
 263         ioat->type = RTE_IOAT_DEV;
 264         ioat->rawdev = rawdev;
 265         ioat->mz = mz;
 266         ioat->regs = dev->mem_resource[0].addr;
 267         ioat->doorbell = &ioat->regs->dmacount;
 268         ioat->ring_size = 0;
 269         ioat->desc_ring = NULL;
 270         ioat->status_addr = ioat->mz->iova +
 271                         offsetof(struct rte_ioat_rawdev, status);
 272
 273         /* do device initialization - reset and set error behaviour */
 274         if (ioat->regs->chancnt != 1)
 275                 IOAT_PMD_ERR("%s: Channel count == %d\n", __func__,
 276                                 ioat->regs->chancnt);
 277
 278         if (ioat->regs->chanctrl & 0x100) { /* locked by someone else */
 279                 IOAT_PMD_WARN("%s: Channel appears locked\n", __func__);
 280                 ioat->regs->chanctrl = 0;
 281         }
 282
 283         ioat->regs->chancmd = RTE_IOAT_CHANCMD_SUSPEND;
 284         rte_delay_ms(1);
 285         ioat->regs->chancmd = RTE_IOAT_CHANCMD_RESET;
 286         rte_delay_ms(1);
 287         while (ioat->regs->chancmd & RTE_IOAT_CHANCMD_RESET) {
 288                 ioat->regs->chainaddr = 0;
 289                 rte_delay_ms(1);
 290                 if (++retry >= 200) {
 291                         IOAT_PMD_ERR("%s: cannot reset device. CHANCMD=0x%"PRIx8", CHANSTS=0x%"PRIx64", CHANERR=0x%"PRIx32"\n",
 292                                         __func__,
 293                                         ioat->regs->chancmd,
 294                                         ioat->regs->chansts,
 295                                         ioat->regs->chanerr);
 296                         ret = -EIO;
 297                 }
 298         }
 299         ioat->regs->chanctrl = RTE_IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 300                         RTE_IOAT_CHANCTRL_ERR_COMPLETION_EN;
 301
 302         return 0;
 303
 304 cleanup:
 305         if (rawdev)
 306                 rte_rawdev_pmd_release(rawdev);
 307
 308         return ret;
 309 }
 310
 311 static int
 312 ioat_rawdev_destroy(const char *name)
 313 {
 314         int ret;
 315         struct rte_rawdev *rdev;
 316
 317         if (!name) {
 318                 IOAT_PMD_ERR("Invalid device name");
 319                 return -EINVAL;
 320         }
 321
 322         rdev = rte_rawdev_pmd_get_named_dev(name);
 323         if (!rdev) {
 324                 IOAT_PMD_ERR("Invalid device name (%s)", name);
 325                 return -EINVAL;
 326         }
 327
 328         if (rdev->dev_private != NULL) {
 329                 struct rte_ioat_rawdev *ioat = rdev->dev_private;
 330                 rdev->dev_private = NULL;
 331                 rte_memzone_free(ioat->desc_mz);
 332                 rte_memzone_free(ioat->mz);
 333         }
 334
 335         /* rte_rawdev_close is called by pmd_release */
 336         ret = rte_rawdev_pmd_release(rdev);
 337         if (ret)
 338                 IOAT_PMD_DEBUG("Device cleanup failed");
 339
 340         return 0;
 341 }
 342
 343 static int
 344 ioat_rawdev_probe(struct rte_pci_driver *drv, struct rte_pci_device *dev)
 345 {
 346         char name[32];
 347         int ret = 0;
 348
 349
 350         rte_pci_device_name(&dev->addr, name, sizeof(name));
 351         IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node);
 352
 353         dev->device.driver = &drv->driver;
 354         ret = ioat_rawdev_create(name, dev);
 355         return ret;
 356 }
 357
 358 static int
 359 ioat_rawdev_remove(struct rte_pci_device *dev)
 360 {
 361         char name[32];
 362         int ret;
 363
 364         rte_pci_device_name(&dev->addr, name, sizeof(name));
 365
 366         IOAT_PMD_INFO("Closing %s on NUMA node %d",
 367                         name, dev->device.numa_node);
 368
 369         ret = ioat_rawdev_destroy(name);
 370         return ret;
 371 }
 372
 373 static const struct rte_pci_id pci_id_ioat_map[] = {
 374         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_SKX) },
 375         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX0) },
 376         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX1) },
 377         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX2) },
 378         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX3) },
 379         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX4) },
 380         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX5) },
 381         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX6) },
 382         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX7) },
 383         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXE) },
 384         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXF) },
 385         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_ICX) },
 386         { .vendor_id = 0, /* sentinel */ },
 387 };
 388
 389 static struct rte_pci_driver ioat_pmd_drv = {
 390         .id_table = pci_id_ioat_map,
 391         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 392         .probe = ioat_rawdev_probe,
 393         .remove = ioat_rawdev_remove,
 394 };
 395
 396 RTE_PMD_REGISTER_PCI(IOAT_PMD_RAWDEV_NAME, ioat_pmd_drv);
 397 RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_RAWDEV_NAME, pci_id_ioat_map);
 398 RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_RAWDEV_NAME, "* igb_uio | uio_pci_generic");