lib/librte_vhost/vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 /**
   6  * @file
   7  *
   8  * Device specific vhost lib
   9  */
  10
  11 #include <stdbool.h>
  12 #include <sys/queue.h>
  13
  14 #include <rte_class.h>
  15 #include <rte_malloc.h>
  16 #include <rte_spinlock.h>
  17 #include <rte_tailq.h>
  18
  19 #include "rte_vdpa.h"
  20 #include "vhost.h"
  21
  22 /** Double linked list of vDPA devices. */
  23 TAILQ_HEAD(vdpa_device_list, rte_vdpa_device);
  24
  25 static struct vdpa_device_list vdpa_device_list =
  26                 TAILQ_HEAD_INITIALIZER(vdpa_device_list);
  27 static rte_spinlock_t vdpa_device_list_lock = RTE_SPINLOCK_INITIALIZER;
  28 static uint32_t vdpa_device_num;
  29
  30
  31 /* Unsafe, needs to be called with vdpa_device_list_lock held */
  32 static struct rte_vdpa_device *
  33 __vdpa_find_device_by_name(const char *name)
  34 {
  35         struct rte_vdpa_device *dev, *ret = NULL;
  36
  37         if (name == NULL)
  38                 return NULL;
  39
  40         TAILQ_FOREACH(dev, &vdpa_device_list, next) {
  41                 if (!strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)) {
  42                         ret = dev;
  43                         break;
  44                 }
  45         }
  46
  47         return ret;
  48 }
  49
  50 struct rte_vdpa_device *
  51 rte_vdpa_find_device_by_name(const char *name)
  52 {
  53         struct rte_vdpa_device *dev;
  54
  55         rte_spinlock_lock(&vdpa_device_list_lock);
  56         dev = __vdpa_find_device_by_name(name);
  57         rte_spinlock_unlock(&vdpa_device_list_lock);
  58
  59         return dev;
  60 }
  61
  62 struct rte_device *
  63 rte_vdpa_get_rte_device(struct rte_vdpa_device *vdpa_dev)
  64 {
  65         if (vdpa_dev == NULL)
  66                 return NULL;
  67
  68         return vdpa_dev->device;
  69 }
  70
  71 struct rte_vdpa_device *
  72 rte_vdpa_register_device(struct rte_device *rte_dev,
  73                 struct rte_vdpa_dev_ops *ops)
  74 {
  75         struct rte_vdpa_device *dev;
  76
  77         if (ops == NULL)
  78                 return NULL;
  79
  80         rte_spinlock_lock(&vdpa_device_list_lock);
  81         /* Check the device hasn't been register already */
  82         dev = __vdpa_find_device_by_name(rte_dev->name);
  83         if (dev) {
  84                 dev = NULL;
  85                 goto out_unlock;
  86         }
  87
  88         dev = rte_zmalloc(NULL, sizeof(*dev), 0);
  89         if (!dev)
  90                 goto out_unlock;
  91
  92         dev->device = rte_dev;
  93         dev->ops = ops;
  94         TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
  95         vdpa_device_num++;
  96 out_unlock:
  97         rte_spinlock_unlock(&vdpa_device_list_lock);
  98
  99         return dev;
 100 }
 101
 102 int
 103 rte_vdpa_unregister_device(struct rte_vdpa_device *dev)
 104 {
 105         struct rte_vdpa_device *cur_dev, *tmp_dev;
 106         int ret = -1;
 107
 108         rte_spinlock_lock(&vdpa_device_list_lock);
 109         TAILQ_FOREACH_SAFE(cur_dev, &vdpa_device_list, next, tmp_dev) {
 110                 if (dev != cur_dev)
 111                         continue;
 112
 113                 TAILQ_REMOVE(&vdpa_device_list, dev, next);
 114                 rte_free(dev);
 115                 vdpa_device_num--;
 116                 ret = 0;
 117                 break;
 118         }
 119         rte_spinlock_unlock(&vdpa_device_list_lock);
 120
 121         return ret;
 122 }
 123
 124 int
 125 rte_vdpa_get_device_num(void)
 126 {
 127         return vdpa_device_num;
 128 }
 129
 130 int
 131 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
 132 {
 133         struct virtio_net *dev = get_device(vid);
 134         uint16_t idx, idx_m, desc_id;
 135         struct vhost_virtqueue *vq;
 136         struct vring_desc desc;
 137         struct vring_desc *desc_ring;
 138         struct vring_desc *idesc = NULL;
 139         struct vring *s_vring;
 140         uint64_t dlen;
 141         uint32_t nr_descs;
 142         int ret;
 143
 144         if (!dev || !vring_m)
 145                 return -1;
 146
 147         if (qid >= dev->nr_vring)
 148                 return -1;
 149
 150         if (vq_is_packed(dev))
 151                 return -1;
 152
 153         s_vring = (struct vring *)vring_m;
 154         vq = dev->virtqueue[qid];
 155         idx = vq->used->idx;
 156         idx_m = s_vring->used->idx;
 157         ret = (uint16_t)(idx_m - idx);
 158
 159         while (idx != idx_m) {
 160                 /* copy used entry, used ring logging is not covered here */
 161                 vq->used->ring[idx & (vq->size - 1)] =
 162                         s_vring->used->ring[idx & (vq->size - 1)];
 163
 164                 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
 165                 desc_ring = vq->desc;
 166                 nr_descs = vq->size;
 167
 168                 if (unlikely(desc_id >= vq->size))
 169                         return -1;
 170
 171                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 172                         dlen = vq->desc[desc_id].len;
 173                         nr_descs = dlen / sizeof(struct vring_desc);
 174                         if (unlikely(nr_descs > vq->size))
 175                                 return -1;
 176
 177                         desc_ring = (struct vring_desc *)(uintptr_t)
 178                                 vhost_iova_to_vva(dev, vq,
 179                                                 vq->desc[desc_id].addr, &dlen,
 180                                                 VHOST_ACCESS_RO);
 181                         if (unlikely(!desc_ring))
 182                                 return -1;
 183
 184                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 185                                 idesc = vhost_alloc_copy_ind_table(dev, vq,
 186                                                 vq->desc[desc_id].addr,
 187                                                 vq->desc[desc_id].len);
 188                                 if (unlikely(!idesc))
 189                                         return -1;
 190
 191                                 desc_ring = idesc;
 192                         }
 193
 194                         desc_id = 0;
 195                 }
 196
 197                 /* dirty page logging for DMA writeable buffer */
 198                 do {
 199                         if (unlikely(desc_id >= vq->size))
 200                                 goto fail;
 201                         if (unlikely(nr_descs-- == 0))
 202                                 goto fail;
 203                         desc = desc_ring[desc_id];
 204                         if (desc.flags & VRING_DESC_F_WRITE)
 205                                 vhost_log_write_iova(dev, vq, desc.addr,
 206                                                      desc.len);
 207                         desc_id = desc.next;
 208                 } while (desc.flags & VRING_DESC_F_NEXT);
 209
 210                 if (unlikely(idesc)) {
 211                         free_ind_table(idesc);
 212                         idesc = NULL;
 213                 }
 214
 215                 idx++;
 216         }
 217
 218         rte_smp_wmb();
 219         vq->used->idx = idx_m;
 220
 221         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 222                 vring_used_event(s_vring) = idx_m;
 223
 224         return ret;
 225
 226 fail:
 227         if (unlikely(idesc))
 228                 free_ind_table(idesc);
 229         return -1;
 230 }
 231
 232 int
 233 rte_vdpa_get_stats_names(struct rte_vdpa_device *dev,
 234                 struct rte_vdpa_stat_name *stats_names,
 235                 unsigned int size)
 236 {
 237         if (!dev)
 238                 return -EINVAL;
 239
 240         RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats_names, -ENOTSUP);
 241
 242         return dev->ops->get_stats_names(dev, stats_names, size);
 243 }
 244
 245 int
 246 rte_vdpa_get_stats(struct rte_vdpa_device *dev, uint16_t qid,
 247                 struct rte_vdpa_stat *stats, unsigned int n)
 248 {
 249         if (!dev || !stats || !n)
 250                 return -EINVAL;
 251
 252         RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats, -ENOTSUP);
 253
 254         return dev->ops->get_stats(dev, qid, stats, n);
 255 }
 256
 257 int
 258 rte_vdpa_reset_stats(struct rte_vdpa_device *dev, uint16_t qid)
 259 {
 260         if (!dev)
 261                 return -EINVAL;
 262
 263         RTE_FUNC_PTR_OR_ERR_RET(dev->ops->reset_stats, -ENOTSUP);
 264
 265         return dev->ops->reset_stats(dev, qid);
 266 }
 267
 268 static int
 269 vdpa_dev_match(struct rte_vdpa_device *dev,
 270               const struct rte_device *rte_dev)
 271 {
 272         if (dev->device == rte_dev)
 273                 return 0;
 274
 275         return -1;
 276 }
 277
 278 /* Generic rte_vdpa_dev comparison function. */
 279 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *,
 280                 const struct rte_device *rte_dev);
 281
 282 static struct rte_vdpa_device *
 283 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp,
 284                 struct rte_device *rte_dev)
 285 {
 286         struct rte_vdpa_device *dev;
 287
 288         rte_spinlock_lock(&vdpa_device_list_lock);
 289         if (start == NULL)
 290                 dev = TAILQ_FIRST(&vdpa_device_list);
 291         else
 292                 dev = TAILQ_NEXT(start, next);
 293
 294         while (dev != NULL) {
 295                 if (cmp(dev, rte_dev) == 0)
 296                         break;
 297
 298                 dev = TAILQ_NEXT(dev, next);
 299         }
 300         rte_spinlock_unlock(&vdpa_device_list_lock);
 301
 302         return dev;
 303 }
 304
 305 static void *
 306 vdpa_dev_iterate(const void *start,
 307                 const char *str,
 308                 const struct rte_dev_iterator *it)
 309 {
 310         struct rte_vdpa_device *vdpa_dev = NULL;
 311
 312         RTE_SET_USED(str);
 313
 314         vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device);
 315
 316         return vdpa_dev;
 317 }
 318
 319 static struct rte_class rte_class_vdpa = {
 320         .dev_iterate = vdpa_dev_iterate,
 321 };
 322
 323 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa);