lib/librte_vhost/vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 /**
   6  * @file
   7  *
   8  * Device specific vhost lib
   9  */
  10
  11 #include <stdbool.h>
  12 #include <sys/queue.h>
  13
  14 #include <rte_class.h>
  15 #include <rte_malloc.h>
  16 #include <rte_spinlock.h>
  17 #include <rte_tailq.h>
  18
  19 #include "rte_vdpa.h"
  20 #include "rte_vdpa_dev.h"
  21 #include "vhost.h"
  22
  23 /** Double linked list of vDPA devices. */
  24 TAILQ_HEAD(vdpa_device_list, rte_vdpa_device);
  25
  26 static struct vdpa_device_list vdpa_device_list =
  27                 TAILQ_HEAD_INITIALIZER(vdpa_device_list);
  28 static rte_spinlock_t vdpa_device_list_lock = RTE_SPINLOCK_INITIALIZER;
  29
  30
  31 /* Unsafe, needs to be called with vdpa_device_list_lock held */
  32 static struct rte_vdpa_device *
  33 __vdpa_find_device_by_name(const char *name)
  34 {
  35         struct rte_vdpa_device *dev, *ret = NULL;
  36
  37         if (name == NULL)
  38                 return NULL;
  39
  40         TAILQ_FOREACH(dev, &vdpa_device_list, next) {
  41                 if (!strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)) {
  42                         ret = dev;
  43                         break;
  44                 }
  45         }
  46
  47         return ret;
  48 }
  49
  50 struct rte_vdpa_device *
  51 rte_vdpa_find_device_by_name(const char *name)
  52 {
  53         struct rte_vdpa_device *dev;
  54
  55         rte_spinlock_lock(&vdpa_device_list_lock);
  56         dev = __vdpa_find_device_by_name(name);
  57         rte_spinlock_unlock(&vdpa_device_list_lock);
  58
  59         return dev;
  60 }
  61
  62 struct rte_device *
  63 rte_vdpa_get_rte_device(struct rte_vdpa_device *vdpa_dev)
  64 {
  65         if (vdpa_dev == NULL)
  66                 return NULL;
  67
  68         return vdpa_dev->device;
  69 }
  70
  71 struct rte_vdpa_device *
  72 rte_vdpa_register_device(struct rte_device *rte_dev,
  73                 struct rte_vdpa_dev_ops *ops)
  74 {
  75         struct rte_vdpa_device *dev;
  76
  77         if (ops == NULL)
  78                 return NULL;
  79
  80         rte_spinlock_lock(&vdpa_device_list_lock);
  81         /* Check the device hasn't been register already */
  82         dev = __vdpa_find_device_by_name(rte_dev->name);
  83         if (dev) {
  84                 dev = NULL;
  85                 goto out_unlock;
  86         }
  87
  88         dev = rte_zmalloc(NULL, sizeof(*dev), 0);
  89         if (!dev)
  90                 goto out_unlock;
  91
  92         dev->device = rte_dev;
  93         dev->ops = ops;
  94         TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
  95 out_unlock:
  96         rte_spinlock_unlock(&vdpa_device_list_lock);
  97
  98         return dev;
  99 }
 100
 101 int
 102 rte_vdpa_unregister_device(struct rte_vdpa_device *dev)
 103 {
 104         struct rte_vdpa_device *cur_dev, *tmp_dev;
 105         int ret = -1;
 106
 107         rte_spinlock_lock(&vdpa_device_list_lock);
 108         TAILQ_FOREACH_SAFE(cur_dev, &vdpa_device_list, next, tmp_dev) {
 109                 if (dev != cur_dev)
 110                         continue;
 111
 112                 TAILQ_REMOVE(&vdpa_device_list, dev, next);
 113                 rte_free(dev);
 114                 ret = 0;
 115                 break;
 116         }
 117         rte_spinlock_unlock(&vdpa_device_list_lock);
 118
 119         return ret;
 120 }
 121
 122 int
 123 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
 124 {
 125         struct virtio_net *dev = get_device(vid);
 126         uint16_t idx, idx_m, desc_id;
 127         struct vhost_virtqueue *vq;
 128         struct vring_desc desc;
 129         struct vring_desc *desc_ring;
 130         struct vring_desc *idesc = NULL;
 131         struct vring *s_vring;
 132         uint64_t dlen;
 133         uint32_t nr_descs;
 134         int ret;
 135
 136         if (!dev || !vring_m)
 137                 return -1;
 138
 139         if (qid >= dev->nr_vring)
 140                 return -1;
 141
 142         if (vq_is_packed(dev))
 143                 return -1;
 144
 145         s_vring = (struct vring *)vring_m;
 146         vq = dev->virtqueue[qid];
 147         idx = vq->used->idx;
 148         idx_m = s_vring->used->idx;
 149         ret = (uint16_t)(idx_m - idx);
 150
 151         while (idx != idx_m) {
 152                 /* copy used entry, used ring logging is not covered here */
 153                 vq->used->ring[idx & (vq->size - 1)] =
 154                         s_vring->used->ring[idx & (vq->size - 1)];
 155
 156                 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
 157                 desc_ring = vq->desc;
 158                 nr_descs = vq->size;
 159
 160                 if (unlikely(desc_id >= vq->size))
 161                         return -1;
 162
 163                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 164                         dlen = vq->desc[desc_id].len;
 165                         nr_descs = dlen / sizeof(struct vring_desc);
 166                         if (unlikely(nr_descs > vq->size))
 167                                 return -1;
 168
 169                         desc_ring = (struct vring_desc *)(uintptr_t)
 170                                 vhost_iova_to_vva(dev, vq,
 171                                                 vq->desc[desc_id].addr, &dlen,
 172                                                 VHOST_ACCESS_RO);
 173                         if (unlikely(!desc_ring))
 174                                 return -1;
 175
 176                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 177                                 idesc = vhost_alloc_copy_ind_table(dev, vq,
 178                                                 vq->desc[desc_id].addr,
 179                                                 vq->desc[desc_id].len);
 180                                 if (unlikely(!idesc))
 181                                         return -1;
 182
 183                                 desc_ring = idesc;
 184                         }
 185
 186                         desc_id = 0;
 187                 }
 188
 189                 /* dirty page logging for DMA writeable buffer */
 190                 do {
 191                         if (unlikely(desc_id >= vq->size))
 192                                 goto fail;
 193                         if (unlikely(nr_descs-- == 0))
 194                                 goto fail;
 195                         desc = desc_ring[desc_id];
 196                         if (desc.flags & VRING_DESC_F_WRITE)
 197                                 vhost_log_write_iova(dev, vq, desc.addr,
 198                                                      desc.len);
 199                         desc_id = desc.next;
 200                 } while (desc.flags & VRING_DESC_F_NEXT);
 201
 202                 if (unlikely(idesc)) {
 203                         free_ind_table(idesc);
 204                         idesc = NULL;
 205                 }
 206
 207                 idx++;
 208         }
 209
 210         rte_smp_wmb();
 211         vq->used->idx = idx_m;
 212
 213         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 214                 vring_used_event(s_vring) = idx_m;
 215
 216         return ret;
 217
 218 fail:
 219         if (unlikely(idesc))
 220                 free_ind_table(idesc);
 221         return -1;
 222 }
 223
 224 int
 225 rte_vdpa_get_queue_num(struct rte_vdpa_device *dev, uint32_t *queue_num)
 226 {
 227         if (dev == NULL || dev->ops == NULL || dev->ops->get_queue_num == NULL)
 228                 return -1;
 229
 230         return dev->ops->get_queue_num(dev, queue_num);
 231 }
 232
 233 int
 234 rte_vdpa_get_features(struct rte_vdpa_device *dev, uint64_t *features)
 235 {
 236         if (dev == NULL || dev->ops == NULL || dev->ops->get_features == NULL)
 237                 return -1;
 238
 239         return dev->ops->get_features(dev, features);
 240 }
 241
 242 int
 243 rte_vdpa_get_protocol_features(struct rte_vdpa_device *dev, uint64_t *features)
 244 {
 245         if (dev == NULL || dev->ops == NULL ||
 246                         dev->ops->get_protocol_features == NULL)
 247                 return -1;
 248
 249         return dev->ops->get_protocol_features(dev, features);
 250 }
 251
 252 int
 253 rte_vdpa_get_stats_names(struct rte_vdpa_device *dev,
 254                 struct rte_vdpa_stat_name *stats_names,
 255                 unsigned int size)
 256 {
 257         if (!dev)
 258                 return -EINVAL;
 259
 260         RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats_names, -ENOTSUP);
 261
 262         return dev->ops->get_stats_names(dev, stats_names, size);
 263 }
 264
 265 int
 266 rte_vdpa_get_stats(struct rte_vdpa_device *dev, uint16_t qid,
 267                 struct rte_vdpa_stat *stats, unsigned int n)
 268 {
 269         if (!dev || !stats || !n)
 270                 return -EINVAL;
 271
 272         RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats, -ENOTSUP);
 273
 274         return dev->ops->get_stats(dev, qid, stats, n);
 275 }
 276
 277 int
 278 rte_vdpa_reset_stats(struct rte_vdpa_device *dev, uint16_t qid)
 279 {
 280         if (!dev)
 281                 return -EINVAL;
 282
 283         RTE_FUNC_PTR_OR_ERR_RET(dev->ops->reset_stats, -ENOTSUP);
 284
 285         return dev->ops->reset_stats(dev, qid);
 286 }
 287
 288 static int
 289 vdpa_dev_match(struct rte_vdpa_device *dev,
 290               const struct rte_device *rte_dev)
 291 {
 292         if (dev->device == rte_dev)
 293                 return 0;
 294
 295         return -1;
 296 }
 297
 298 /* Generic rte_vdpa_dev comparison function. */
 299 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *,
 300                 const struct rte_device *rte_dev);
 301
 302 static struct rte_vdpa_device *
 303 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp,
 304                 struct rte_device *rte_dev)
 305 {
 306         struct rte_vdpa_device *dev;
 307
 308         rte_spinlock_lock(&vdpa_device_list_lock);
 309         if (start == NULL)
 310                 dev = TAILQ_FIRST(&vdpa_device_list);
 311         else
 312                 dev = TAILQ_NEXT(start, next);
 313
 314         while (dev != NULL) {
 315                 if (cmp(dev, rte_dev) == 0)
 316                         break;
 317
 318                 dev = TAILQ_NEXT(dev, next);
 319         }
 320         rte_spinlock_unlock(&vdpa_device_list_lock);
 321
 322         return dev;
 323 }
 324
 325 static void *
 326 vdpa_dev_iterate(const void *start,
 327                 const char *str,
 328                 const struct rte_dev_iterator *it)
 329 {
 330         struct rte_vdpa_device *vdpa_dev = NULL;
 331
 332         RTE_SET_USED(str);
 333
 334         vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device);
 335
 336         return vdpa_dev;
 337 }
 338
 339 static struct rte_class rte_class_vdpa = {
 340         .dev_iterate = vdpa_dev_iterate,
 341 };
 342
 343 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa);