drivers/raw/ioat/ioat_rawdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <rte_cycles.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_string_fns.h>
   8 #include <rte_rawdev_pmd.h>
   9
  10 #include "rte_ioat_rawdev.h"
  11
  12 static struct rte_pci_driver ioat_pmd_drv;
  13
  14 #define IOAT_VENDOR_ID          0x8086
  15 #define IOAT_DEVICE_ID_SKX      0x2021
  16 #define IOAT_DEVICE_ID_BDX0     0x6f20
  17 #define IOAT_DEVICE_ID_BDX1     0x6f21
  18 #define IOAT_DEVICE_ID_BDX2     0x6f22
  19 #define IOAT_DEVICE_ID_BDX3     0x6f23
  20 #define IOAT_DEVICE_ID_BDX4     0x6f24
  21 #define IOAT_DEVICE_ID_BDX5     0x6f25
  22 #define IOAT_DEVICE_ID_BDX6     0x6f26
  23 #define IOAT_DEVICE_ID_BDX7     0x6f27
  24 #define IOAT_DEVICE_ID_BDXE     0x6f2E
  25 #define IOAT_DEVICE_ID_BDXF     0x6f2F
  26 #define IOAT_DEVICE_ID_ICX      0x0b00
  27
  28 RTE_LOG_REGISTER(ioat_pmd_logtype, rawdev.ioat, INFO);
  29
  30 #define IOAT_PMD_LOG(level, fmt, args...) rte_log(RTE_LOG_ ## level, \
  31         ioat_pmd_logtype, "%s(): " fmt "\n", __func__, ##args)
  32
  33 #define IOAT_PMD_DEBUG(fmt, args...)  IOAT_PMD_LOG(DEBUG, fmt, ## args)
  34 #define IOAT_PMD_INFO(fmt, args...)   IOAT_PMD_LOG(INFO, fmt, ## args)
  35 #define IOAT_PMD_ERR(fmt, args...)    IOAT_PMD_LOG(ERR, fmt, ## args)
  36 #define IOAT_PMD_WARN(fmt, args...)   IOAT_PMD_LOG(WARNING, fmt, ## args)
  37
  38 #define DESC_SZ sizeof(struct rte_ioat_generic_hw_desc)
  39 #define COMPLETION_SZ sizeof(__m128i)
  40
  41 static int
  42 ioat_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config,
  43                 size_t config_size)
  44 {
  45         struct rte_ioat_rawdev_config *params = config;
  46         struct rte_ioat_rawdev *ioat = dev->dev_private;
  47         char mz_name[RTE_MEMZONE_NAMESIZE];
  48         unsigned short i;
  49
  50         if (dev->started)
  51                 return -EBUSY;
  52
  53         if (params == NULL || config_size != sizeof(*params))
  54                 return -EINVAL;
  55
  56         if (params->ring_size > 4096 || params->ring_size < 64 ||
  57                         !rte_is_power_of_2(params->ring_size))
  58                 return -EINVAL;
  59
  60         ioat->ring_size = params->ring_size;
  61         ioat->hdls_disable = params->hdls_disable;
  62         if (ioat->desc_ring != NULL) {
  63                 rte_memzone_free(ioat->desc_mz);
  64                 ioat->desc_ring = NULL;
  65                 ioat->desc_mz = NULL;
  66         }
  67
  68         /* allocate one block of memory for both descriptors
  69          * and completion handles.
  70          */
  71         snprintf(mz_name, sizeof(mz_name), "rawdev%u_desc_ring", dev->dev_id);
  72         ioat->desc_mz = rte_memzone_reserve(mz_name,
  73                         (DESC_SZ + COMPLETION_SZ) * ioat->ring_size,
  74                         dev->device->numa_node, RTE_MEMZONE_IOVA_CONTIG);
  75         if (ioat->desc_mz == NULL)
  76                 return -ENOMEM;
  77         ioat->desc_ring = ioat->desc_mz->addr;
  78         ioat->hdls = (void *)&ioat->desc_ring[ioat->ring_size];
  79
  80         ioat->ring_addr = ioat->desc_mz->iova;
  81
  82         /* configure descriptor ring - each one points to next */
  83         for (i = 0; i < ioat->ring_size; i++) {
  84                 ioat->desc_ring[i].next = ioat->ring_addr +
  85                                 (((i + 1) % ioat->ring_size) * DESC_SZ);
  86         }
  87
  88         return 0;
  89 }
  90
  91 static int
  92 ioat_dev_start(struct rte_rawdev *dev)
  93 {
  94         struct rte_ioat_rawdev *ioat = dev->dev_private;
  95
  96         if (ioat->ring_size == 0 || ioat->desc_ring == NULL)
  97                 return -EBUSY;
  98
  99         /* inform hardware of where the descriptor ring is */
 100         ioat->regs->chainaddr = ioat->ring_addr;
 101         /* inform hardware of where to write the status/completions */
 102         ioat->regs->chancmp = ioat->status_addr;
 103
 104         /* prime the status register to be set to the last element */
 105         ioat->status =  ioat->ring_addr + ((ioat->ring_size - 1) * DESC_SZ);
 106         return 0;
 107 }
 108
 109 static void
 110 ioat_dev_stop(struct rte_rawdev *dev)
 111 {
 112         RTE_SET_USED(dev);
 113 }
 114
 115 static int
 116 ioat_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info,
 117                 size_t dev_info_size)
 118 {
 119         struct rte_ioat_rawdev_config *cfg = dev_info;
 120         struct rte_ioat_rawdev *ioat = dev->dev_private;
 121
 122         if (dev_info == NULL || dev_info_size != sizeof(*cfg))
 123                 return -EINVAL;
 124
 125         cfg->ring_size = ioat->ring_size;
 126         cfg->hdls_disable = ioat->hdls_disable;
 127         return 0;
 128 }
 129
 130 static const char * const xstat_names[] = {
 131                 "failed_enqueues", "successful_enqueues",
 132                 "copies_started", "copies_completed"
 133 };
 134
 135 static int
 136 ioat_xstats_get(const struct rte_rawdev *dev, const unsigned int ids[],
 137                 uint64_t values[], unsigned int n)
 138 {
 139         const struct rte_ioat_rawdev *ioat = dev->dev_private;
 140         unsigned int i;
 141
 142         for (i = 0; i < n; i++) {
 143                 switch (ids[i]) {
 144                 case 0: values[i] = ioat->enqueue_failed; break;
 145                 case 1: values[i] = ioat->enqueued; break;
 146                 case 2: values[i] = ioat->started; break;
 147                 case 3: values[i] = ioat->completed; break;
 148                 default: values[i] = 0; break;
 149                 }
 150         }
 151         return n;
 152 }
 153
 154 static int
 155 ioat_xstats_get_names(const struct rte_rawdev *dev,
 156                 struct rte_rawdev_xstats_name *names,
 157                 unsigned int size)
 158 {
 159         unsigned int i;
 160
 161         RTE_SET_USED(dev);
 162         if (size < RTE_DIM(xstat_names))
 163                 return RTE_DIM(xstat_names);
 164
 165         for (i = 0; i < RTE_DIM(xstat_names); i++)
 166                 strlcpy(names[i].name, xstat_names[i], sizeof(names[i]));
 167
 168         return RTE_DIM(xstat_names);
 169 }
 170
 171 static int
 172 ioat_xstats_reset(struct rte_rawdev *dev, const uint32_t *ids, uint32_t nb_ids)
 173 {
 174         struct rte_ioat_rawdev *ioat = dev->dev_private;
 175         unsigned int i;
 176
 177         if (!ids) {
 178                 ioat->enqueue_failed = 0;
 179                 ioat->enqueued = 0;
 180                 ioat->started = 0;
 181                 ioat->completed = 0;
 182                 return 0;
 183         }
 184
 185         for (i = 0; i < nb_ids; i++) {
 186                 switch (ids[i]) {
 187                 case 0:
 188                         ioat->enqueue_failed = 0;
 189                         break;
 190                 case 1:
 191                         ioat->enqueued = 0;
 192                         break;
 193                 case 2:
 194                         ioat->started = 0;
 195                         break;
 196                 case 3:
 197                         ioat->completed = 0;
 198                         break;
 199                 default:
 200                         IOAT_PMD_WARN("Invalid xstat id - cannot reset value");
 201                         break;
 202                 }
 203         }
 204
 205         return 0;
 206 }
 207
 208 static int
 209 ioat_dev_close(struct rte_rawdev *dev __rte_unused)
 210 {
 211         return 0;
 212 }
 213
 214 extern int ioat_rawdev_test(uint16_t dev_id);
 215
 216 static int
 217 ioat_rawdev_create(const char *name, struct rte_pci_device *dev)
 218 {
 219         static const struct rte_rawdev_ops ioat_rawdev_ops = {
 220                         .dev_configure = ioat_dev_configure,
 221                         .dev_start = ioat_dev_start,
 222                         .dev_stop = ioat_dev_stop,
 223                         .dev_close = ioat_dev_close,
 224                         .dev_info_get = ioat_dev_info_get,
 225                         .xstats_get = ioat_xstats_get,
 226                         .xstats_get_names = ioat_xstats_get_names,
 227                         .xstats_reset = ioat_xstats_reset,
 228                         .dev_selftest = ioat_rawdev_test,
 229         };
 230
 231         struct rte_rawdev *rawdev = NULL;
 232         struct rte_ioat_rawdev *ioat = NULL;
 233         const struct rte_memzone *mz = NULL;
 234         char mz_name[RTE_MEMZONE_NAMESIZE];
 235         int ret = 0;
 236         int retry = 0;
 237
 238         if (!name) {
 239                 IOAT_PMD_ERR("Invalid name of the device!");
 240                 ret = -EINVAL;
 241                 goto cleanup;
 242         }
 243
 244         /* Allocate device structure */
 245         rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct rte_ioat_rawdev),
 246                                          dev->device.numa_node);
 247         if (rawdev == NULL) {
 248                 IOAT_PMD_ERR("Unable to allocate raw device");
 249                 ret = -ENOMEM;
 250                 goto cleanup;
 251         }
 252
 253         snprintf(mz_name, sizeof(mz_name), "rawdev%u_private", rawdev->dev_id);
 254         mz = rte_memzone_reserve(mz_name, sizeof(struct rte_ioat_rawdev),
 255                         dev->device.numa_node, RTE_MEMZONE_IOVA_CONTIG);
 256         if (mz == NULL) {
 257                 IOAT_PMD_ERR("Unable to reserve memzone for private data\n");
 258                 ret = -ENOMEM;
 259                 goto cleanup;
 260         }
 261
 262         rawdev->dev_private = mz->addr;
 263         rawdev->dev_ops = &ioat_rawdev_ops;
 264         rawdev->device = &dev->device;
 265         rawdev->driver_name = dev->device.driver->name;
 266
 267         ioat = rawdev->dev_private;
 268         ioat->rawdev = rawdev;
 269         ioat->mz = mz;
 270         ioat->regs = dev->mem_resource[0].addr;
 271         ioat->ring_size = 0;
 272         ioat->desc_ring = NULL;
 273         ioat->status_addr = ioat->mz->iova +
 274                         offsetof(struct rte_ioat_rawdev, status);
 275
 276         /* do device initialization - reset and set error behaviour */
 277         if (ioat->regs->chancnt != 1)
 278                 IOAT_PMD_ERR("%s: Channel count == %d\n", __func__,
 279                                 ioat->regs->chancnt);
 280
 281         if (ioat->regs->chanctrl & 0x100) { /* locked by someone else */
 282                 IOAT_PMD_WARN("%s: Channel appears locked\n", __func__);
 283                 ioat->regs->chanctrl = 0;
 284         }
 285
 286         ioat->regs->chancmd = RTE_IOAT_CHANCMD_SUSPEND;
 287         rte_delay_ms(1);
 288         ioat->regs->chancmd = RTE_IOAT_CHANCMD_RESET;
 289         rte_delay_ms(1);
 290         while (ioat->regs->chancmd & RTE_IOAT_CHANCMD_RESET) {
 291                 ioat->regs->chainaddr = 0;
 292                 rte_delay_ms(1);
 293                 if (++retry >= 200) {
 294                         IOAT_PMD_ERR("%s: cannot reset device. CHANCMD=0x%"PRIx8", CHANSTS=0x%"PRIx64", CHANERR=0x%"PRIx32"\n",
 295                                         __func__,
 296                                         ioat->regs->chancmd,
 297                                         ioat->regs->chansts,
 298                                         ioat->regs->chanerr);
 299                         ret = -EIO;
 300                 }
 301         }
 302         ioat->regs->chanctrl = RTE_IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 303                         RTE_IOAT_CHANCTRL_ERR_COMPLETION_EN;
 304
 305         return 0;
 306
 307 cleanup:
 308         if (rawdev)
 309                 rte_rawdev_pmd_release(rawdev);
 310
 311         return ret;
 312 }
 313
 314 static int
 315 ioat_rawdev_destroy(const char *name)
 316 {
 317         int ret;
 318         struct rte_rawdev *rdev;
 319
 320         if (!name) {
 321                 IOAT_PMD_ERR("Invalid device name");
 322                 return -EINVAL;
 323         }
 324
 325         rdev = rte_rawdev_pmd_get_named_dev(name);
 326         if (!rdev) {
 327                 IOAT_PMD_ERR("Invalid device name (%s)", name);
 328                 return -EINVAL;
 329         }
 330
 331         if (rdev->dev_private != NULL) {
 332                 struct rte_ioat_rawdev *ioat = rdev->dev_private;
 333                 rdev->dev_private = NULL;
 334                 rte_memzone_free(ioat->desc_mz);
 335                 rte_memzone_free(ioat->mz);
 336         }
 337
 338         /* rte_rawdev_close is called by pmd_release */
 339         ret = rte_rawdev_pmd_release(rdev);
 340         if (ret)
 341                 IOAT_PMD_DEBUG("Device cleanup failed");
 342
 343         return 0;
 344 }
 345
 346 static int
 347 ioat_rawdev_probe(struct rte_pci_driver *drv, struct rte_pci_device *dev)
 348 {
 349         char name[32];
 350         int ret = 0;
 351
 352
 353         rte_pci_device_name(&dev->addr, name, sizeof(name));
 354         IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node);
 355
 356         dev->device.driver = &drv->driver;
 357         ret = ioat_rawdev_create(name, dev);
 358         return ret;
 359 }
 360
 361 static int
 362 ioat_rawdev_remove(struct rte_pci_device *dev)
 363 {
 364         char name[32];
 365         int ret;
 366
 367         rte_pci_device_name(&dev->addr, name, sizeof(name));
 368
 369         IOAT_PMD_INFO("Closing %s on NUMA node %d",
 370                         name, dev->device.numa_node);
 371
 372         ret = ioat_rawdev_destroy(name);
 373         return ret;
 374 }
 375
 376 static const struct rte_pci_id pci_id_ioat_map[] = {
 377         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_SKX) },
 378         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX0) },
 379         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX1) },
 380         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX2) },
 381         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX3) },
 382         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX4) },
 383         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX5) },
 384         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX6) },
 385         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDX7) },
 386         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXE) },
 387         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_BDXF) },
 388         { RTE_PCI_DEVICE(IOAT_VENDOR_ID, IOAT_DEVICE_ID_ICX) },
 389         { .vendor_id = 0, /* sentinel */ },
 390 };
 391
 392 static struct rte_pci_driver ioat_pmd_drv = {
 393         .id_table = pci_id_ioat_map,
 394         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 395         .probe = ioat_rawdev_probe,
 396         .remove = ioat_rawdev_remove,
 397 };
 398
 399 RTE_PMD_REGISTER_PCI(IOAT_PMD_RAWDEV_NAME, ioat_pmd_drv);
 400 RTE_PMD_REGISTER_PCI_TABLE(IOAT_PMD_RAWDEV_NAME, pci_id_ioat_map);
 401 RTE_PMD_REGISTER_KMOD_DEP(IOAT_PMD_RAWDEV_NAME, "* igb_uio | uio_pci_generic");