1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
5 #include <rte_rawdev_pmd.h>
6 #include <rte_memzone.h>
7 #include <rte_common.h>
8 #include <rte_string_fns.h>
10 #include "ioat_private.h"
12 static const char * const xstat_names[] = {
13 "failed_enqueues", "successful_enqueues",
14 "copies_started", "copies_completed"
18 ioat_xstats_get(const struct rte_rawdev *dev, const unsigned int ids[],
19 uint64_t values[], unsigned int n)
21 const struct rte_ioat_rawdev *ioat = dev->dev_private;
22 const uint64_t *stats = (const void *)&ioat->xstats;
25 for (i = 0; i < n; i++) {
26 if (ids[i] > sizeof(ioat->xstats)/sizeof(*stats))
29 values[i] = stats[ids[i]];
35 ioat_xstats_get_names(const struct rte_rawdev *dev,
36 struct rte_rawdev_xstats_name *names,
42 if (size < RTE_DIM(xstat_names))
43 return RTE_DIM(xstat_names);
45 for (i = 0; i < RTE_DIM(xstat_names); i++)
46 strlcpy(names[i].name, xstat_names[i], sizeof(names[i]));
48 return RTE_DIM(xstat_names);
52 ioat_xstats_reset(struct rte_rawdev *dev, const uint32_t *ids, uint32_t nb_ids)
54 struct rte_ioat_rawdev *ioat = dev->dev_private;
55 uint64_t *stats = (void *)&ioat->xstats;
59 memset(&ioat->xstats, 0, sizeof(ioat->xstats));
63 for (i = 0; i < nb_ids; i++)
64 if (ids[i] < sizeof(ioat->xstats)/sizeof(*stats))
71 idxd_rawdev_close(struct rte_rawdev *dev __rte_unused)
77 idxd_dev_dump(struct rte_rawdev *dev, FILE *f)
79 struct idxd_rawdev *idxd = dev->dev_private;
80 struct rte_idxd_rawdev *rte_idxd = &idxd->public;
83 fprintf(f, "Raw Device #%d\n", dev->dev_id);
84 fprintf(f, "Driver: %s\n\n", dev->driver_name);
86 fprintf(f, "Portal: %p\n", rte_idxd->portal);
87 fprintf(f, "Config: {ring_size: %u, hdls_disable: %u}\n\n",
88 rte_idxd->cfg.ring_size, rte_idxd->cfg.hdls_disable);
90 fprintf(f, "max batches: %u\n", rte_idxd->max_batches);
91 fprintf(f, "batch idx read: %u\n", rte_idxd->batch_idx_read);
92 fprintf(f, "batch idx write: %u\n", rte_idxd->batch_idx_write);
93 fprintf(f, "batch idxes:");
94 for (i = 0; i < rte_idxd->max_batches + 1; i++)
95 fprintf(f, "%u ", rte_idxd->batch_idx_ring[i]);
98 fprintf(f, "hdls read: %u\n", rte_idxd->max_batches);
99 fprintf(f, "hdls avail: %u\n", rte_idxd->hdls_avail);
100 fprintf(f, "batch start: %u\n", rte_idxd->batch_start);
101 fprintf(f, "batch size: %u\n", rte_idxd->batch_size);
107 idxd_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info,
110 struct rte_ioat_rawdev_config *cfg = dev_info;
111 struct idxd_rawdev *idxd = dev->dev_private;
112 struct rte_idxd_rawdev *rte_idxd = &idxd->public;
114 if (info_size != sizeof(*cfg))
118 *cfg = rte_idxd->cfg;
123 idxd_dev_configure(const struct rte_rawdev *dev,
124 rte_rawdev_obj_t config, size_t config_size)
126 struct idxd_rawdev *idxd = dev->dev_private;
127 struct rte_idxd_rawdev *rte_idxd = &idxd->public;
128 struct rte_ioat_rawdev_config *cfg = config;
129 uint16_t max_desc = cfg->ring_size;
131 if (config_size != sizeof(*cfg))
135 IOAT_PMD_ERR("%s: Error, device is started.", __func__);
139 rte_idxd->cfg = *cfg;
141 if (!rte_is_power_of_2(max_desc))
142 max_desc = rte_align32pow2(max_desc);
143 IOAT_PMD_DEBUG("Rawdev %u using %u descriptors",
144 dev->dev_id, max_desc);
145 rte_idxd->desc_ring_mask = max_desc - 1;
147 /* in case we are reconfiguring a device, free any existing memory */
148 rte_free(rte_idxd->desc_ring);
149 rte_free(rte_idxd->hdl_ring);
150 rte_free(rte_idxd->hdl_ring_flags);
152 /* allocate the descriptor ring at 2x size as batches can't wrap */
153 rte_idxd->desc_ring = rte_zmalloc(NULL,
154 sizeof(*rte_idxd->desc_ring) * max_desc * 2, 0);
155 if (rte_idxd->desc_ring == NULL)
157 rte_idxd->desc_iova = rte_mem_virt2iova(rte_idxd->desc_ring);
159 rte_idxd->hdl_ring = rte_zmalloc(NULL,
160 sizeof(*rte_idxd->hdl_ring) * max_desc, 0);
161 if (rte_idxd->hdl_ring == NULL) {
162 rte_free(rte_idxd->desc_ring);
163 rte_idxd->desc_ring = NULL;
166 rte_idxd->hdl_ring_flags = rte_zmalloc(NULL,
167 sizeof(*rte_idxd->hdl_ring_flags) * max_desc, 0);
168 if (rte_idxd->hdl_ring_flags == NULL) {
169 rte_free(rte_idxd->desc_ring);
170 rte_free(rte_idxd->hdl_ring);
171 rte_idxd->desc_ring = NULL;
172 rte_idxd->hdl_ring = NULL;
175 rte_idxd->hdls_read = rte_idxd->batch_start = 0;
176 rte_idxd->batch_size = 0;
182 idxd_rawdev_create(const char *name, struct rte_device *dev,
183 const struct idxd_rawdev *base_idxd,
184 const struct rte_rawdev_ops *ops)
186 struct idxd_rawdev *idxd;
187 struct rte_idxd_rawdev *public;
188 struct rte_rawdev *rawdev = NULL;
189 const struct rte_memzone *mz = NULL;
190 char mz_name[RTE_MEMZONE_NAMESIZE];
193 RTE_BUILD_BUG_ON(sizeof(struct rte_idxd_hw_desc) != 64);
194 RTE_BUILD_BUG_ON(offsetof(struct rte_idxd_hw_desc, size) != 32);
195 RTE_BUILD_BUG_ON(sizeof(struct rte_idxd_completion) != 32);
198 IOAT_PMD_ERR("Invalid name of the device!");
203 /* Allocate device structure */
204 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct idxd_rawdev),
206 if (rawdev == NULL) {
207 IOAT_PMD_ERR("Unable to allocate raw device");
212 /* Allocate memory for the primary process or else return the memory
213 * of primary memzone for the secondary process.
215 snprintf(mz_name, sizeof(mz_name), "rawdev%u_private", rawdev->dev_id);
216 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
217 mz = rte_memzone_lookup(mz_name);
219 IOAT_PMD_ERR("Unable lookup memzone for private data\n");
223 rawdev->dev_private = mz->addr;
224 rawdev->dev_ops = ops;
225 rawdev->device = dev;
228 mz = rte_memzone_reserve(mz_name, sizeof(struct idxd_rawdev),
229 dev->numa_node, RTE_MEMZONE_IOVA_CONTIG);
231 IOAT_PMD_ERR("Unable to reserve memzone for private data\n");
235 rawdev->dev_private = mz->addr;
236 rawdev->dev_ops = ops;
237 rawdev->device = dev;
238 rawdev->driver_name = IOAT_PMD_RAWDEV_NAME_STR;
240 idxd = rawdev->dev_private;
241 *idxd = *base_idxd; /* copy over the main fields already passed in */
242 idxd->rawdev = rawdev;
245 public = &idxd->public;
246 public->type = RTE_IDXD_DEV;
247 public->max_batches = idxd->max_batches;
248 public->batch_idx_read = 0;
249 public->batch_idx_write = 0;
250 /* allocate batch index ring. The +1 is because we can never fully use
251 * the ring, otherwise read == write means both full and empty.
253 public->batch_idx_ring = rte_zmalloc(NULL,
254 sizeof(uint16_t) * (idxd->max_batches + 1), 0);
255 if (public->batch_idx_ring == NULL) {
256 IOAT_PMD_ERR("Unable to reserve memory for batch data\n");
265 rte_memzone_free(mz);
267 rte_rawdev_pmd_release(rawdev);