1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
8 * Device specific vhost lib
12 #include <sys/queue.h>
14 #include <rte_class.h>
15 #include <rte_malloc.h>
16 #include <rte_spinlock.h>
17 #include <rte_tailq.h>
20 #include "rte_vdpa_dev.h"
23 /** Double linked list of vDPA devices. */
24 TAILQ_HEAD(vdpa_device_list, rte_vdpa_device);
26 static struct vdpa_device_list vdpa_device_list =
27 TAILQ_HEAD_INITIALIZER(vdpa_device_list);
28 static rte_spinlock_t vdpa_device_list_lock = RTE_SPINLOCK_INITIALIZER;
31 /* Unsafe, needs to be called with vdpa_device_list_lock held */
32 static struct rte_vdpa_device *
33 __vdpa_find_device_by_name(const char *name)
35 struct rte_vdpa_device *dev, *ret = NULL;
40 TAILQ_FOREACH(dev, &vdpa_device_list, next) {
41 if (!strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)) {
50 struct rte_vdpa_device *
51 rte_vdpa_find_device_by_name(const char *name)
53 struct rte_vdpa_device *dev;
55 rte_spinlock_lock(&vdpa_device_list_lock);
56 dev = __vdpa_find_device_by_name(name);
57 rte_spinlock_unlock(&vdpa_device_list_lock);
63 rte_vdpa_get_rte_device(struct rte_vdpa_device *vdpa_dev)
68 return vdpa_dev->device;
71 struct rte_vdpa_device *
72 rte_vdpa_register_device(struct rte_device *rte_dev,
73 struct rte_vdpa_dev_ops *ops)
75 struct rte_vdpa_device *dev;
80 /* Check mandatory ops are implemented */
81 if (!ops->get_queue_num || !ops->get_features ||
82 !ops->get_protocol_features || !ops->dev_conf ||
83 !ops->dev_close || !ops->set_vring_state ||
86 "Some mandatory vDPA ops aren't implemented\n");
90 rte_spinlock_lock(&vdpa_device_list_lock);
91 /* Check the device hasn't been register already */
92 dev = __vdpa_find_device_by_name(rte_dev->name);
98 dev = rte_zmalloc(NULL, sizeof(*dev), 0);
102 dev->device = rte_dev;
104 TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
106 rte_spinlock_unlock(&vdpa_device_list_lock);
112 rte_vdpa_unregister_device(struct rte_vdpa_device *dev)
114 struct rte_vdpa_device *cur_dev, *tmp_dev;
117 rte_spinlock_lock(&vdpa_device_list_lock);
118 TAILQ_FOREACH_SAFE(cur_dev, &vdpa_device_list, next, tmp_dev) {
122 TAILQ_REMOVE(&vdpa_device_list, dev, next);
127 rte_spinlock_unlock(&vdpa_device_list_lock);
133 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
135 struct virtio_net *dev = get_device(vid);
136 uint16_t idx, idx_m, desc_id;
137 struct vhost_virtqueue *vq;
138 struct vring_desc desc;
139 struct vring_desc *desc_ring;
140 struct vring_desc *idesc = NULL;
141 struct vring *s_vring;
146 if (!dev || !vring_m)
149 if (qid >= dev->nr_vring)
152 if (vq_is_packed(dev))
155 s_vring = (struct vring *)vring_m;
156 vq = dev->virtqueue[qid];
158 idx_m = s_vring->used->idx;
159 ret = (uint16_t)(idx_m - idx);
161 while (idx != idx_m) {
162 /* copy used entry, used ring logging is not covered here */
163 vq->used->ring[idx & (vq->size - 1)] =
164 s_vring->used->ring[idx & (vq->size - 1)];
166 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
167 desc_ring = vq->desc;
170 if (unlikely(desc_id >= vq->size))
173 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
174 dlen = vq->desc[desc_id].len;
175 nr_descs = dlen / sizeof(struct vring_desc);
176 if (unlikely(nr_descs > vq->size))
179 desc_ring = (struct vring_desc *)(uintptr_t)
180 vhost_iova_to_vva(dev, vq,
181 vq->desc[desc_id].addr, &dlen,
183 if (unlikely(!desc_ring))
186 if (unlikely(dlen < vq->desc[desc_id].len)) {
187 idesc = vhost_alloc_copy_ind_table(dev, vq,
188 vq->desc[desc_id].addr,
189 vq->desc[desc_id].len);
190 if (unlikely(!idesc))
199 /* dirty page logging for DMA writeable buffer */
201 if (unlikely(desc_id >= vq->size))
203 if (unlikely(nr_descs-- == 0))
205 desc = desc_ring[desc_id];
206 if (desc.flags & VRING_DESC_F_WRITE)
207 vhost_log_write_iova(dev, vq, desc.addr,
210 } while (desc.flags & VRING_DESC_F_NEXT);
212 if (unlikely(idesc)) {
213 free_ind_table(idesc);
220 /* used idx is the synchronization point for the split vring */
221 __atomic_store_n(&vq->used->idx, idx_m, __ATOMIC_RELEASE);
223 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
224 vring_used_event(s_vring) = idx_m;
230 free_ind_table(idesc);
235 rte_vdpa_get_queue_num(struct rte_vdpa_device *dev, uint32_t *queue_num)
237 if (dev == NULL || dev->ops == NULL || dev->ops->get_queue_num == NULL)
240 return dev->ops->get_queue_num(dev, queue_num);
244 rte_vdpa_get_features(struct rte_vdpa_device *dev, uint64_t *features)
246 if (dev == NULL || dev->ops == NULL || dev->ops->get_features == NULL)
249 return dev->ops->get_features(dev, features);
253 rte_vdpa_get_protocol_features(struct rte_vdpa_device *dev, uint64_t *features)
255 if (dev == NULL || dev->ops == NULL ||
256 dev->ops->get_protocol_features == NULL)
259 return dev->ops->get_protocol_features(dev, features);
263 rte_vdpa_get_stats_names(struct rte_vdpa_device *dev,
264 struct rte_vdpa_stat_name *stats_names,
270 RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats_names, -ENOTSUP);
272 return dev->ops->get_stats_names(dev, stats_names, size);
276 rte_vdpa_get_stats(struct rte_vdpa_device *dev, uint16_t qid,
277 struct rte_vdpa_stat *stats, unsigned int n)
279 if (!dev || !stats || !n)
282 RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats, -ENOTSUP);
284 return dev->ops->get_stats(dev, qid, stats, n);
288 rte_vdpa_reset_stats(struct rte_vdpa_device *dev, uint16_t qid)
293 RTE_FUNC_PTR_OR_ERR_RET(dev->ops->reset_stats, -ENOTSUP);
295 return dev->ops->reset_stats(dev, qid);
299 vdpa_dev_match(struct rte_vdpa_device *dev,
300 const struct rte_device *rte_dev)
302 if (dev->device == rte_dev)
308 /* Generic rte_vdpa_dev comparison function. */
309 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *,
310 const struct rte_device *rte_dev);
312 static struct rte_vdpa_device *
313 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp,
314 struct rte_device *rte_dev)
316 struct rte_vdpa_device *dev;
318 rte_spinlock_lock(&vdpa_device_list_lock);
320 dev = TAILQ_FIRST(&vdpa_device_list);
322 dev = TAILQ_NEXT(start, next);
324 while (dev != NULL) {
325 if (cmp(dev, rte_dev) == 0)
328 dev = TAILQ_NEXT(dev, next);
330 rte_spinlock_unlock(&vdpa_device_list_lock);
336 vdpa_dev_iterate(const void *start,
338 const struct rte_dev_iterator *it)
340 struct rte_vdpa_device *vdpa_dev = NULL;
344 vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device);
349 static struct rte_class rte_class_vdpa = {
350 .dev_iterate = vdpa_dev_iterate,
353 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa);