1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
8 * Device specific vhost lib
13 #include <rte_malloc.h>
17 static struct rte_vdpa_device *vdpa_devices[MAX_VHOST_DEVICE];
18 static uint32_t vdpa_device_num;
21 is_same_vdpa_device(struct rte_vdpa_dev_addr *a,
22 struct rte_vdpa_dev_addr *b)
26 if (a->type != b->type)
31 if (a->pci_addr.domain != b->pci_addr.domain ||
32 a->pci_addr.bus != b->pci_addr.bus ||
33 a->pci_addr.devid != b->pci_addr.devid ||
34 a->pci_addr.function != b->pci_addr.function)
45 rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr,
46 struct rte_vdpa_dev_ops *ops)
48 struct rte_vdpa_device *dev;
49 char device_name[MAX_VDPA_NAME_LEN];
52 if (vdpa_device_num >= MAX_VHOST_DEVICE)
55 for (i = 0; i < MAX_VHOST_DEVICE; i++) {
56 dev = vdpa_devices[i];
57 if (dev && is_same_vdpa_device(&dev->addr, addr))
61 for (i = 0; i < MAX_VHOST_DEVICE; i++) {
62 if (vdpa_devices[i] == NULL)
66 if (i == MAX_VHOST_DEVICE)
69 sprintf(device_name, "vdpa-dev-%d", i);
70 dev = rte_zmalloc(device_name, sizeof(struct rte_vdpa_device),
75 memcpy(&dev->addr, addr, sizeof(struct rte_vdpa_dev_addr));
77 vdpa_devices[i] = dev;
84 rte_vdpa_unregister_device(int did)
86 if (did < 0 || did >= MAX_VHOST_DEVICE || vdpa_devices[did] == NULL)
89 rte_free(vdpa_devices[did]);
90 vdpa_devices[did] = NULL;
97 rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)
99 struct rte_vdpa_device *dev;
102 for (i = 0; i < MAX_VHOST_DEVICE; ++i) {
103 dev = vdpa_devices[i];
104 if (dev && is_same_vdpa_device(&dev->addr, addr))
111 struct rte_vdpa_device *
112 rte_vdpa_get_device(int did)
114 if (did < 0 || did >= MAX_VHOST_DEVICE)
117 return vdpa_devices[did];
121 rte_vdpa_get_device_num(void)
123 return vdpa_device_num;
127 invalid_desc_check(struct virtio_net *dev, struct vhost_virtqueue *vq,
128 uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
130 uint64_t desc_addr, desc_chunck_len;
133 desc_chunck_len = desc_len;
134 desc_addr = vhost_iova_to_vva(dev, vq,
142 desc_len -= desc_chunck_len;
143 desc_iova += desc_chunck_len;
149 int __rte_experimental
150 rte_vdpa_relay_vring_avail(int vid, uint16_t qid, void *vring_m)
152 struct virtio_net *dev = get_device(vid);
153 uint16_t idx, idx_m, desc_id;
154 struct vring_desc desc;
155 struct vhost_virtqueue *vq;
156 struct vring_desc *desc_ring;
157 struct vring_desc *idesc = NULL;
158 struct vring *s_vring;
163 if (!dev || !vring_m)
166 if (qid >= dev->nr_vring)
169 if (vq_is_packed(dev))
172 s_vring = (struct vring *)vring_m;
173 vq = dev->virtqueue[qid];
174 idx = vq->avail->idx;
175 idx_m = s_vring->avail->idx;
176 ret = (uint16_t)(idx - idx_m);
178 while (idx_m != idx) {
179 /* avail entry copy */
180 desc_id = vq->avail->ring[idx_m & (vq->size - 1)];
181 if (unlikely(desc_id >= vq->size))
184 s_vring->avail->ring[idx_m & (vq->size - 1)] = desc_id;
185 desc_ring = vq->desc;
187 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
188 dlen = vq->desc[desc_id].len;
189 desc_ring = (struct vring_desc *)(uintptr_t)
190 vhost_iova_to_vva(dev, vq,
191 vq->desc[desc_id].addr, &dlen,
193 if (unlikely(!desc_ring))
196 if (unlikely(dlen < vq->desc[desc_id].len)) {
197 idesc = alloc_copy_ind_table(dev, vq,
198 vq->desc[desc_id].addr,
199 vq->desc[desc_id].len);
200 if (unlikely(!idesc))
209 /* check if the buf addr is within the guest memory */
211 if (unlikely(desc_id >= vq->size))
213 desc = desc_ring[desc_id];
214 perm = desc.flags & VRING_DESC_F_WRITE ?
215 VHOST_ACCESS_WO : VHOST_ACCESS_RO;
216 if (invalid_desc_check(dev, vq, desc.addr, desc.len,
220 } while (desc.flags & VRING_DESC_F_NEXT);
222 if (unlikely(idesc)) {
223 free_ind_table(idesc);
231 s_vring->avail->idx = idx;
233 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
234 vhost_avail_event(vq) = idx;
240 free_ind_table(idesc);
244 int __rte_experimental
245 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
247 struct virtio_net *dev = get_device(vid);
248 uint16_t idx, idx_m, desc_id;
249 struct vhost_virtqueue *vq;
250 struct vring_desc desc;
251 struct vring_desc *desc_ring;
252 struct vring_desc *idesc = NULL;
253 struct vring *s_vring;
257 if (!dev || !vring_m)
260 if (qid >= dev->nr_vring)
263 if (vq_is_packed(dev))
266 s_vring = (struct vring *)vring_m;
267 vq = dev->virtqueue[qid];
269 idx_m = s_vring->used->idx;
270 ret = (uint16_t)(idx_m - idx);
272 while (idx != idx_m) {
273 /* copy used entry, used ring logging is not covered here */
274 vq->used->ring[idx & (vq->size - 1)] =
275 s_vring->used->ring[idx & (vq->size - 1)];
277 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
278 desc_ring = vq->desc;
280 if (unlikely(desc_id >= vq->size))
283 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
284 dlen = vq->desc[desc_id].len;
285 desc_ring = (struct vring_desc *)(uintptr_t)
286 vhost_iova_to_vva(dev, vq,
287 vq->desc[desc_id].addr, &dlen,
289 if (unlikely(!desc_ring))
292 if (unlikely(dlen < vq->desc[desc_id].len)) {
293 idesc = alloc_copy_ind_table(dev, vq,
294 vq->desc[desc_id].addr,
295 vq->desc[desc_id].len);
296 if (unlikely(!idesc))
305 /* dirty page logging for DMA writeable buffer */
307 if (unlikely(desc_id >= vq->size))
309 desc = desc_ring[desc_id];
310 if (desc.flags & VRING_DESC_F_WRITE)
311 vhost_log_write(dev, desc.addr, desc.len);
313 } while (desc.flags & VRING_DESC_F_NEXT);
315 if (unlikely(idesc)) {
316 free_ind_table(idesc);
324 vq->used->idx = idx_m;
326 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
327 vring_used_event(s_vring) = idx_m;
333 free_ind_table(idesc);