1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
5 #include <rte_bus_pci.h>
6 #include <rte_memzone.h>
8 #include "ioat_private.h"
11 #define IDXD_VENDOR_ID 0x8086
12 #define IDXD_DEVICE_ID_SPR 0x0B25
14 #define IDXD_PMD_RAWDEV_NAME_PCI rawdev_idxd_pci
16 const struct rte_pci_id pci_id_idxd_map[] = {
17 { RTE_PCI_DEVICE(IDXD_VENDOR_ID, IDXD_DEVICE_ID_SPR) },
18 { .vendor_id = 0, /* sentinel */ },
22 idxd_pci_dev_command(struct idxd_rawdev *idxd, enum rte_idxd_cmds command)
25 uint16_t qid = idxd->qid;
28 if (command >= idxd_disable_wq && command <= idxd_reset_wq)
30 rte_spinlock_lock(&idxd->u.pci->lk);
31 idxd->u.pci->regs->cmd = (command << IDXD_CMD_SHIFT) | qid;
35 err_code = idxd->u.pci->regs->cmdstatus;
37 IOAT_PMD_ERR("Timeout waiting for command response from HW");
38 rte_spinlock_unlock(&idxd->u.pci->lk);
41 } while (idxd->u.pci->regs->cmdstatus & CMDSTATUS_ACTIVE_MASK);
42 rte_spinlock_unlock(&idxd->u.pci->lk);
44 return err_code & CMDSTATUS_ERR_MASK;
48 idxd_is_wq_enabled(struct idxd_rawdev *idxd)
50 uint32_t state = idxd->u.pci->wq_regs[idxd->qid].wqcfg[WQ_STATE_IDX];
51 return ((state >> WQ_STATE_SHIFT) & WQ_STATE_MASK) == 0x1;
55 idxd_pci_dev_stop(struct rte_rawdev *dev)
57 struct idxd_rawdev *idxd = dev->dev_private;
60 if (!idxd_is_wq_enabled(idxd)) {
61 IOAT_PMD_ERR("Work queue %d already disabled", idxd->qid);
65 err_code = idxd_pci_dev_command(idxd, idxd_disable_wq);
66 if (err_code || idxd_is_wq_enabled(idxd)) {
67 IOAT_PMD_ERR("Failed disabling work queue %d, error code: %#x",
71 IOAT_PMD_DEBUG("Work queue %d disabled OK", idxd->qid);
75 idxd_pci_dev_start(struct rte_rawdev *dev)
77 struct idxd_rawdev *idxd = dev->dev_private;
80 if (idxd_is_wq_enabled(idxd)) {
81 IOAT_PMD_WARN("WQ %d already enabled", idxd->qid);
85 if (idxd->public.batch_ring == NULL) {
86 IOAT_PMD_ERR("WQ %d has not been fully configured", idxd->qid);
90 err_code = idxd_pci_dev_command(idxd, idxd_enable_wq);
91 if (err_code || !idxd_is_wq_enabled(idxd)) {
92 IOAT_PMD_ERR("Failed enabling work queue %d, error code: %#x",
94 return err_code == 0 ? -1 : err_code;
97 IOAT_PMD_DEBUG("Work queue %d enabled OK", idxd->qid);
102 static const struct rte_rawdev_ops idxd_pci_ops = {
103 .dev_close = idxd_rawdev_close,
104 .dev_selftest = idxd_rawdev_test,
105 .dump = idxd_dev_dump,
106 .dev_configure = idxd_dev_configure,
107 .dev_start = idxd_pci_dev_start,
108 .dev_stop = idxd_pci_dev_stop,
109 .dev_info_get = idxd_dev_info_get,
110 .xstats_get = ioat_xstats_get,
111 .xstats_get_names = ioat_xstats_get_names,
112 .xstats_reset = ioat_xstats_reset,
115 /* each portal uses 4 x 4k pages */
116 #define IDXD_PORTAL_SIZE (4096 * 4)
119 init_pci_device(struct rte_pci_device *dev, struct idxd_rawdev *idxd)
121 struct idxd_pci_common *pci;
122 uint8_t nb_groups, nb_engines, nb_wqs;
123 uint16_t grp_offset, wq_offset; /* how far into bar0 the regs are */
124 uint16_t wq_size, total_wq_size;
125 uint8_t lg2_max_batch, lg2_max_copy_size;
126 unsigned int i, err_code;
128 pci = malloc(sizeof(*pci));
130 IOAT_PMD_ERR("%s: Can't allocate memory", __func__);
133 rte_spinlock_init(&pci->lk);
135 /* assign the bar registers, and then configure device */
136 pci->regs = dev->mem_resource[0].addr;
137 grp_offset = (uint16_t)pci->regs->offsets[0];
138 pci->grp_regs = RTE_PTR_ADD(pci->regs, grp_offset * 0x100);
139 wq_offset = (uint16_t)(pci->regs->offsets[0] >> 16);
140 pci->wq_regs = RTE_PTR_ADD(pci->regs, wq_offset * 0x100);
141 pci->portals = dev->mem_resource[2].addr;
143 /* sanity check device status */
144 if (pci->regs->gensts & GENSTS_DEV_STATE_MASK) {
145 /* need function-level-reset (FLR) or is enabled */
146 IOAT_PMD_ERR("Device status is not disabled, cannot init");
149 if (pci->regs->cmdstatus & CMDSTATUS_ACTIVE_MASK) {
150 /* command in progress */
151 IOAT_PMD_ERR("Device has a command in progress, cannot init");
155 /* read basic info about the hardware for use when configuring */
156 nb_groups = (uint8_t)pci->regs->grpcap;
157 nb_engines = (uint8_t)pci->regs->engcap;
158 nb_wqs = (uint8_t)(pci->regs->wqcap >> 16);
159 total_wq_size = (uint16_t)pci->regs->wqcap;
160 lg2_max_copy_size = (uint8_t)(pci->regs->gencap >> 16) & 0x1F;
161 lg2_max_batch = (uint8_t)(pci->regs->gencap >> 21) & 0x0F;
163 IOAT_PMD_DEBUG("nb_groups = %u, nb_engines = %u, nb_wqs = %u",
164 nb_groups, nb_engines, nb_wqs);
166 /* zero out any old config */
167 for (i = 0; i < nb_groups; i++) {
168 pci->grp_regs[i].grpengcfg = 0;
169 pci->grp_regs[i].grpwqcfg[0] = 0;
171 for (i = 0; i < nb_wqs; i++)
172 pci->wq_regs[i].wqcfg[0] = 0;
174 /* put each engine into a separate group to avoid reordering */
175 if (nb_groups > nb_engines)
176 nb_groups = nb_engines;
177 if (nb_groups < nb_engines)
178 nb_engines = nb_groups;
180 /* assign engines to groups, round-robin style */
181 for (i = 0; i < nb_engines; i++) {
182 IOAT_PMD_DEBUG("Assigning engine %u to group %u",
184 pci->grp_regs[i % nb_groups].grpengcfg |= (1ULL << i);
187 /* now do the same for queues and give work slots to each queue */
188 wq_size = total_wq_size / nb_wqs;
189 IOAT_PMD_DEBUG("Work queue size = %u, max batch = 2^%u, max copy = 2^%u",
190 wq_size, lg2_max_batch, lg2_max_copy_size);
191 for (i = 0; i < nb_wqs; i++) {
192 /* add engine "i" to a group */
193 IOAT_PMD_DEBUG("Assigning work queue %u to group %u",
195 pci->grp_regs[i % nb_groups].grpwqcfg[0] |= (1ULL << i);
196 /* now configure it, in terms of size, max batch, mode */
197 pci->wq_regs[i].wqcfg[WQ_SIZE_IDX] = wq_size;
198 pci->wq_regs[i].wqcfg[WQ_MODE_IDX] = (1 << WQ_PRIORITY_SHIFT) |
200 pci->wq_regs[i].wqcfg[WQ_SIZES_IDX] = lg2_max_copy_size |
201 (lg2_max_batch << WQ_BATCH_SZ_SHIFT);
204 /* dump the group configuration to output */
205 for (i = 0; i < nb_groups; i++) {
206 IOAT_PMD_DEBUG("## Group %d", i);
207 IOAT_PMD_DEBUG(" GRPWQCFG: %"PRIx64, pci->grp_regs[i].grpwqcfg[0]);
208 IOAT_PMD_DEBUG(" GRPENGCFG: %"PRIx64, pci->grp_regs[i].grpengcfg);
209 IOAT_PMD_DEBUG(" GRPFLAGS: %"PRIx32, pci->grp_regs[i].grpflags);
213 idxd->max_batches = wq_size;
215 /* enable the device itself */
216 err_code = idxd_pci_dev_command(idxd, idxd_enable_dev);
218 IOAT_PMD_ERR("Error enabling device: code %#x", err_code);
221 IOAT_PMD_DEBUG("IDXD Device enabled OK");
231 idxd_rawdev_probe_pci(struct rte_pci_driver *drv, struct rte_pci_device *dev)
233 struct idxd_rawdev idxd = {{0}}; /* Double {} to avoid error on BSD12 */
236 char name[PCI_PRI_STR_SIZE];
238 rte_pci_device_name(&dev->addr, name, sizeof(name));
239 IOAT_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node);
240 dev->device.driver = &drv->driver;
242 ret = init_pci_device(dev, &idxd);
244 IOAT_PMD_ERR("Error initializing PCI hardware");
247 nb_wqs = (uint8_t)ret;
249 /* set up one device for each queue */
250 for (qid = 0; qid < nb_wqs; qid++) {
253 /* add the queue number to each device name */
254 snprintf(qname, sizeof(qname), "%s-q%d", name, qid);
256 idxd.public.portal = RTE_PTR_ADD(idxd.u.pci->portals,
257 qid * IDXD_PORTAL_SIZE);
258 if (idxd_is_wq_enabled(&idxd))
259 IOAT_PMD_ERR("Error, WQ %u seems enabled", qid);
260 ret = idxd_rawdev_create(qname, &dev->device,
261 &idxd, &idxd_pci_ops);
263 IOAT_PMD_ERR("Failed to create rawdev %s", name);
264 if (qid == 0) /* if no devices using this, free pci */
274 idxd_rawdev_destroy(const char *name)
278 struct rte_rawdev *rdev;
279 struct idxd_rawdev *idxd;
282 IOAT_PMD_ERR("Invalid device name");
286 rdev = rte_rawdev_pmd_get_named_dev(name);
288 IOAT_PMD_ERR("Invalid device name (%s)", name);
292 idxd = rdev->dev_private;
294 /* disable the device */
295 err_code = idxd_pci_dev_command(idxd, idxd_disable_dev);
297 IOAT_PMD_ERR("Error disabling device: code %#x", err_code);
300 IOAT_PMD_DEBUG("IDXD Device disabled OK");
302 /* free device memory */
303 if (rdev->dev_private != NULL) {
304 IOAT_PMD_DEBUG("Freeing device driver memory");
305 rdev->dev_private = NULL;
306 rte_free(idxd->public.batch_ring);
307 rte_free(idxd->public.hdl_ring);
308 rte_memzone_free(idxd->mz);
311 /* rte_rawdev_close is called by pmd_release */
312 ret = rte_rawdev_pmd_release(rdev);
314 IOAT_PMD_DEBUG("Device cleanup failed");
320 idxd_rawdev_remove_pci(struct rte_pci_device *dev)
322 char name[PCI_PRI_STR_SIZE];
325 rte_pci_device_name(&dev->addr, name, sizeof(name));
327 IOAT_PMD_INFO("Closing %s on NUMA node %d",
328 name, dev->device.numa_node);
330 ret = idxd_rawdev_destroy(name);
335 struct rte_pci_driver idxd_pmd_drv_pci = {
336 .id_table = pci_id_idxd_map,
337 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
338 .probe = idxd_rawdev_probe_pci,
339 .remove = idxd_rawdev_remove_pci,
342 RTE_PMD_REGISTER_PCI(IDXD_PMD_RAWDEV_NAME_PCI, idxd_pmd_drv_pci);
343 RTE_PMD_REGISTER_PCI_TABLE(IDXD_PMD_RAWDEV_NAME_PCI, pci_id_idxd_map);
344 RTE_PMD_REGISTER_KMOD_DEP(IDXD_PMD_RAWDEV_NAME_PCI,
345 "* igb_uio | uio_pci_generic | vfio-pci");