drivers/raw/ioat/ioat_rawdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <rte_cycles.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_string_fns.h>
   8 #include <rte_rawdev_pmd.h>
   9
  10 #include "rte_ioat_rawdev.h"
  11
  12 static struct rte_pci_driver ioat_pmd_drv;
  13
  14 #define IOAT_VENDOR_ID          0x8086
  15 #define IOAT_DEVICE_ID_SKX      0x2021
  16 #define IOAT_DEVICE_ID_BDX0     0x6f20
  17 #define IOAT_DEVICE_ID_BDX1     0x6f21
  18 #define IOAT_DEVICE_ID_BDX2     0x6f22
  19 #define IOAT_DEVICE_ID_BDX3     0x6f23
  20 #define IOAT_DEVICE_ID_BDX4     0x6f24
  21 #define IOAT_DEVICE_ID_BDX5     0x6f25
  22 #define IOAT_DEVICE_ID_BDX6     0x6f26
  23 #define IOAT_DEVICE_ID_BDX7     0x6f27
  24 #define IOAT_DEVICE_ID_BDXE     0x6f2E
  25 #define IOAT_DEVICE_ID_BDXF     0x6f2F
  26 #define IOAT_DEVICE_ID_ICX      0x0b00
  27
  28 RTE_LOG_REGISTER(ioat_pmd_logtype, rawdev.ioat, INFO);
  29
  30 #define IOAT_PMD_LOG(level, fmt, args...) rte_log(RTE_LOG_ ## level, \
  31         ioat_pmd_logtype, "%s(): " fmt "\n", __func__, ##args)
  32
  33 #define IOAT_PMD_DEBUG(fmt, args...)  IOAT_PMD_LOG(DEBUG, fmt, ## args)
  34 #define IOAT_PMD_INFO(fmt, args...)   IOAT_PMD_LOG(INFO, fmt, ## args)
  35 #define IOAT_PMD_ERR(fmt, args...)    IOAT_PMD_LOG(ERR, fmt, ## args)
  36 #define IOAT_PMD_WARN(fmt, args...)   IOAT_PMD_LOG(WARNING, fmt, ## args)
  37
  38 #define DESC_SZ sizeof(struct rte_ioat_generic_hw_desc)
  39 #define COMPLETION_SZ sizeof(__m128i)
  40
  41 static int
  42 ioat_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config,
  43                 size_t config_size)
  44 {
  45         struct rte_ioat_rawdev_config *params = config;
  46         struct rte_ioat_rawdev *ioat = dev->dev_private;
  47         char mz_name[RTE_MEMZONE_NAMESIZE];
  48         unsigned short i;
  49
  50         if (dev->started)
  51                 return -EBUSY;
  52
  53         if (params == NULL || config_size != sizeof(*params))
  54                 return -EINVAL;
  55
  56         if (params->ring_size > 4096 || params->ring_size < 64 ||
  57                         !rte_is_power_of_2(params->ring_size))
  58                 return -EINVAL;
  59
  60         ioat->ring_size = params->ring_size;
  61         if (ioat->desc_ring != NULL) {
  62                 rte_memzone_free(ioat->desc_mz);
  63                 ioat->desc_ring = NULL;
  64                 ioat->desc_mz = NULL;
  65         }
  66
  67         /* allocate one block of memory for both descriptors
  68          * and completion handles.
  69          */
  70         snprintf(mz_name, sizeof(mz_name), "rawdev%u_desc_ring", dev->dev_id);
  71         ioat->desc_mz = rte_memzone_reserve(mz_name,
  72                         (DESC_SZ + COMPLETION_SZ) * ioat->ring_size,
  73                         dev->device->numa_node, RTE_MEMZONE_IOVA_CONTIG);
  74         if (ioat->desc_mz == NULL)
  75                 return -ENOMEM;
  76         ioat->desc_ring = ioat->desc_mz->addr;
  77         ioat->hdls = (void *)&ioat->desc_ring[ioat->ring_size];
  78
  79         ioat->ring_addr = ioat->desc_mz->iova;
  80
  81         /* configure descriptor ring - each one points to next */
  82         for (i = 0; i < ioat->ring_size; i++) {
  83                 ioat->desc_ring[i].next = ioat->ring_addr +
  84                                 (((i + 1) % ioat->ring_size) * DESC_SZ);
  85         }
  86
  87         return 0;
  88 }
  89
  90 static int
  91 ioat_dev_start(struct rte_rawdev *dev)
  92 {
  93         struct rte_ioat_rawdev *ioat = dev->dev_private;
  94
  95         if (ioat->ring_size == 0 || ioat->desc_ring == NULL)
  96                 return -EBUSY;
  97
  98         /* inform hardware of where the descriptor ring is */
  99         ioat->regs->chainaddr = ioat->ring_addr;
 100         /* inform hardware of where to write the status/completions */
 101         ioat->regs->chancmp = ioat->status_addr;
 102
 103         /* prime the status register to be set to the last element */
 104         ioat->status =  ioat->ring_addr + ((ioat->ring_size - 1) * DESC_SZ);
 105         return 0;
 106 }
 107
 108 static void
 109 ioat_dev_stop(struct rte_rawdev *dev)
 110 {
 111         RTE_SET_USED(dev);
 112 }
 113
 114 static int
 115 ioat_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info,
 116                 size_t dev_info_size)
 117 {
 118         struct rte_ioat_rawdev_config *cfg = dev_info;
 119         struct rte_ioat_rawdev *ioat = dev->dev_private;
 120
 121         if (dev_info == NULL || dev_info_size != sizeof(*cfg))
 122                 return -EINVAL;
 123
 124         cfg->ring_size = ioat->ring_size;
 125         return 0;
 126 }
 127
 128 static const char * const xstat_names[] = {
 129                 "failed_enqueues", "successful_enqueues",
 130                 "copies_started", "copies_completed"
 131 };
 132
 133 static int
 134 ioat_xstats_get(const struct rte_rawdev *dev, const unsigned int ids[],
 135                 uint64_t values[], unsigned int n)
 136 {
 137         const struct rte_ioat_rawdev *ioat = dev->dev_private;
 138         unsigned int i;
 139
 140         for (i = 0; i < n; i++) {
 141                 switch (ids[i]) {
 142                 case 0: values[i] = ioat->enqueue_failed; break;
 143                 case 1: values[i] = ioat->enqueued; break;
 144                 case 2: values[i] = ioat->started; break;
 145                 case 3: values[i] = ioat->completed; break;
 146                 default: values[i] = 0; break;
 147                 }
 148         }
 149         return n;
 150 }
 151
 152 static int
 153 ioat_xstats_get_names(const struct rte_rawdev *dev,
 154                 struct rte_rawdev_xstats_name *names,
 155                 unsigned int size)
 156 {
 157         unsigned int i;
 158
 159         RTE_SET_USED(dev);
 160         if (size < RTE_DIM(xstat_names))
 161                 return RTE_DIM(xstat_names);
 162
 163         for (i = 0; i < RTE_DIM(xstat_names); i++)
 164                 strlcpy(names[i].name, xstat_names[i], sizeof(names[i]));
 165
 166         return RTE_DIM(xstat_names);
 167 }
 168
 169 static int
 170 ioat_xstats_reset(struct rte_rawdev *dev, const uint32_t *ids, uint32_t nb_ids)
 171 {
 172         struct rte_ioat_rawdev *ioat = dev->dev_private;
 173         unsigned int i;
 174
 175         if (!ids) {
 176                 ioat->enqueue_failed = 0;
 177                 ioat->enqueued = 0;
 178                 ioat->started = 0;
 179                 ioat->completed = 0;
 180                 return 0;
 181         }
 182
 183         for (i = 0; i < nb_ids; i++) {
 184                 switch (ids[i]) {
 185                 case 0:
 186                         ioat->enqueue_failed = 0;
 187                         break;
 188                 case 1:
 189                         ioat->enqueued = 0;
 190                         break;
 191                 case 2:
 192                         ioat->started = 0;
 193                         break;
 194                 case 3:
 195                         ioat->completed = 0;
 196                         break;
 197                 default:
 198                         IOAT_PMD_WARN("Invalid xstat id - cannot reset value");
 199                         break;
 200                 }
 201         }
 202
 203         return 0;
 204 }
 205
 206 extern int ioat_rawdev_test(uint16_t dev_id);
 207
 208 static int
 209 ioat_rawdev_create(const char *name, struct rte_pci_device *dev)
 210 {
 211         static const struct rte_rawdev_ops ioat_rawdev_ops = {
 212                         .dev_configure = ioat_dev_configure,
 213                         .dev_start = ioat_dev_start,
 214                         .dev_stop = ioat_dev_stop,
 215                         .dev_info_get = ioat_dev_info_get,
 216                         .xstats_get = ioat_xstats_get,
 217                         .xstats_get_names = ioat_xstats_get_names,
 218                         .xstats_reset = ioat_xstats_reset,
 219                         .dev_selftest = ioat_rawdev_test,
 220         };
 221
 222         struct rte_rawdev *rawdev = NULL;
 223         struct rte_ioat_rawdev *ioat = NULL;
 224         const struct rte_memzone *mz = NULL;
 225         char mz_name[RTE_MEMZONE_NAMESIZE];
 226         int ret = 0;
 227         int retry = 0;
 228
 229         if (!name) {
 230                 IOAT_PMD_ERR("Invalid name of the device!");
 231                 ret = -EINVAL;
 232                 goto cleanup;
 233         }
 234
 235         /* Allocate device structure */
 236         rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct rte_ioat_rawdev),
 237                                          dev->device.numa_node);
 238         if (rawdev == NULL) {
 239                 IOAT_PMD_ERR("Unable to allocate raw device");
 240                 ret = -ENOMEM;
 241                 goto cleanup;
 242         }
 243
 244         snprintf(mz_name, sizeof(mz_name), "rawdev%u_private", rawdev->dev_id);
 245         mz = rte_memzone_reserve(mz_name, sizeof(struct rte_ioat_rawdev),
 246                         dev->device.numa_node, RTE_MEMZONE_IOVA_CONTIG);
 247         if (mz == NULL) {
 248                 IOAT_PMD_ERR("Unable to reserve memzone for private data\n");
 249                 ret = -ENOMEM;
 250                 goto cleanup;
 251         }
 252
 253         rawdev->dev_private = mz->addr;
 254         rawdev->dev_ops = &ioat_rawdev_ops;
 255         rawdev->device = &dev->device;
 256         rawdev->driver_name = dev->device.driver->name;
 257
 258         ioat = rawdev->dev_private;
 259         ioat->rawdev = rawdev;
 260         ioat->mz = mz;
 261         ioat->regs = dev->mem_resource[0].addr;
 262         ioat->ring_size = 0;
 263         ioat->desc_ring = NULL;
 264         ioat->status_addr = ioat->mz->iova +
 265                         offsetof(struct rte_ioat_rawdev, status);
 266
 267         /* do device initialization - reset and set error behaviour */
 268         if (ioat->regs->chancnt != 1)
 269                 IOAT_PMD_ERR("%s: Channel count == %d\n", __func__,
 270                                 ioat->regs->chancnt);
 271
 272         if (ioat->regs->chanctrl & 0x100) { /* locked by someone else */
 273                 IOAT_PMD_WARN("%s: Channel appears locked\n", __func__);
 274                 ioat->regs->chanctrl = 0;
 275         }
 276
 277         ioat->regs->chancmd = RTE_IOAT_CHANCMD_SUSPEND;
 278         rte_delay_ms(1);
 279         ioat->regs->chancmd = RTE_IOAT_CHANCMD_RESET;
 280         rte_delay_ms(1);
 281         while (ioat->regs->chancmd & RTE_IOAT_CHANCMD_RESET) {
 282                 ioat->regs->chainaddr = 0;
 283                 rte_delay_ms(1);
 284                 if (++retry >= 200) {
 285                         IOAT_PMD_ERR("%s: cannot reset device. CHANCMD=0x%"PRIx8", CHANSTS=0x%"PRIx64", CHANERR=0x%"PRIx32"\n",
 286                                         __func__,
 287                                         ioat->regs->chancmd,
 288                                         ioat->regs->chansts,
 289                                         ioat->regs->chanerr);
 290                         ret = -EIO;
 291                 }
 292         }
 293         ioat->regs->chanctrl = RTE_IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 294                         RTE_IOAT_CHANCTRL_ERR_COMPLETION_EN;
 295
 296         return 0;
 297
 298 cleanup:
 299         if (rawdev)
 300                 rte_rawdev_pmd_release(rawdev);
 301
 302         return ret;
 303 }
 304
 305 static int
 306 ioat_rawdev_destroy(const char *name)
 307 {
 308         int ret;
 309         struct rte_rawdev *rdev;
 310
 311         if (!name) {
 312                 IOAT_PMD_ERR("Invalid device name");
 313                 return -EINVAL;
 314         }
 315
 316         rdev = rte_rawdev_pmd_get_named_dev(name);
 317         if (!rdev) {
 318                 IOAT_PMD_ERR("Invalid device name (%s)", name);
 319                 return -EINVAL;
 320         }
 321
 322         if (rdev->dev_private != NULL) {
 323                 struct rte_ioat_rawdev *ioat = rdev->dev_private;
 324                 rdev->dev_private = NULL;
 325                 rte_memzone_free(ioat->desc_mz);
 326                 rte_memzone_free(ioat->mz);
 327         }
 328
 329         /* rte_rawdev_close is called by pmd_release */
 330         ret = rte_rawdev_pmd_release(rdev);
 331         if (ret)
 332                 IOAT_PMD_DEBUG("Device cleanup failed");
 333
 334         return 0;
 335 }
 336
 337 static int
 338 ioat_rawdev_probe(struct rte_pci_driver *drv, struct rte_pci_device *dev)
 339 {
 340         char name[32];
 341         int ret = 0;
 342
 343
 344         rte_pci_device_name(&dev->addr, name, sizeof(name));
 345         IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node);
 346
 347         dev->device.driver = &drv->driver;
 348         ret = ioat_rawdev_create(name, dev);
 349         return ret;
 350 }
 351
 352 static int
 353 ioat_rawdev_remove(struct rte_pci_device *dev)
 354 {
 355         char name[32];
 356         int ret;
 357
 358         rte_pci_device_name(&dev->addr, name, sizeof(name));
 359
 360         IOAT_PMD_INFO("Closing %s on NUMA node %d",
 361                         name, dev->device.numa_node);
 362
 363         ret = ioat_rawdev_destroy(name);
 364         return ret;
 365 }
 366
 367 static const struct rte_pci_id pci_id_ioat_map[] = {
 368         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_SKX) },
 369         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX0) },
 370         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX1) },
 371         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX2) },
 372         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX3) },
 373         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX4) },
 374         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX5) },
 375         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX6) },
 376         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX7) },
 377         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXE) },
 378         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXF) },
 379         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_ICX) },
 380         { .vendor_id = 0, /* sentinel */ },
 381 };
 382
 383 static struct rte_pci_driver ioat_pmd_drv = {
 384         .id_table = pci_id_ioat_map,
 385         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 386         .probe = ioat_rawdev_probe,
 387         .remove = ioat_rawdev_remove,
 388 };
 389
 390 RTE_PMD_REGISTER_PCI(IOAT_PMD_RAWDEV_NAME, ioat_pmd_drv);
 391 RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_RAWDEV_NAME, pci_id_ioat_map);
 392 RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_RAWDEV_NAME, "* igb_uio | uio_pci_generic");