lib/vhost/vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 /**
   6  * @file
   7  *
   8  * Device specific vhost lib
   9  */
  10
  11 #include <sys/queue.h>
  12
  13 #include <rte_class.h>
  14 #include <rte_malloc.h>
  15 #include <rte_spinlock.h>
  16 #include <rte_tailq.h>
  17
  18 #include "rte_vdpa.h"
  19 #include "vdpa_driver.h"
  20 #include "vhost.h"
  21
  22 /** Double linked list of vDPA devices. */
  23 TAILQ_HEAD(vdpa_device_list, rte_vdpa_device);
  24
  25 static struct vdpa_device_list vdpa_device_list =
  26                 TAILQ_HEAD_INITIALIZER(vdpa_device_list);
  27 static rte_spinlock_t vdpa_device_list_lock = RTE_SPINLOCK_INITIALIZER;
  28
  29
  30 /* Unsafe, needs to be called with vdpa_device_list_lock held */
  31 static struct rte_vdpa_device *
  32 __vdpa_find_device_by_name(const char *name)
  33 {
  34         struct rte_vdpa_device *dev, *ret = NULL;
  35
  36         if (name == NULL)
  37                 return NULL;
  38
  39         TAILQ_FOREACH(dev, &vdpa_device_list, next) {
  40                 if (!strncmp(dev->device->name, name, RTE_DEV_NAME_MAX_LEN)) {
  41                         ret = dev;
  42                         break;
  43                 }
  44         }
  45
  46         return ret;
  47 }
  48
  49 struct rte_vdpa_device *
  50 rte_vdpa_find_device_by_name(const char *name)
  51 {
  52         struct rte_vdpa_device *dev;
  53
  54         rte_spinlock_lock(&vdpa_device_list_lock);
  55         dev = __vdpa_find_device_by_name(name);
  56         rte_spinlock_unlock(&vdpa_device_list_lock);
  57
  58         return dev;
  59 }
  60
  61 struct rte_device *
  62 rte_vdpa_get_rte_device(struct rte_vdpa_device *vdpa_dev)
  63 {
  64         if (vdpa_dev == NULL)
  65                 return NULL;
  66
  67         return vdpa_dev->device;
  68 }
  69
  70 struct rte_vdpa_device *
  71 rte_vdpa_register_device(struct rte_device *rte_dev,
  72                 struct rte_vdpa_dev_ops *ops)
  73 {
  74         struct rte_vdpa_device *dev;
  75
  76         if (ops == NULL)
  77                 return NULL;
  78
  79         /* Check mandatory ops are implemented */
  80         if (!ops->get_queue_num || !ops->get_features ||
  81                         !ops->get_protocol_features || !ops->dev_conf ||
  82                         !ops->dev_close || !ops->set_vring_state ||
  83                         !ops->set_features) {
  84                 VHOST_LOG_CONFIG(ERR, "(%s) Some mandatory vDPA ops aren't implemented\n",
  85                                 rte_dev->name);
  86                 return NULL;
  87         }
  88
  89         rte_spinlock_lock(&vdpa_device_list_lock);
  90         /* Check the device hasn't been register already */
  91         dev = __vdpa_find_device_by_name(rte_dev->name);
  92         if (dev) {
  93                 dev = NULL;
  94                 goto out_unlock;
  95         }
  96
  97         dev = rte_zmalloc(NULL, sizeof(*dev), 0);
  98         if (!dev)
  99                 goto out_unlock;
 100
 101         dev->device = rte_dev;
 102         dev->ops = ops;
 103         TAILQ_INSERT_TAIL(&vdpa_device_list, dev, next);
 104 out_unlock:
 105         rte_spinlock_unlock(&vdpa_device_list_lock);
 106
 107         return dev;
 108 }
 109
 110 int
 111 rte_vdpa_unregister_device(struct rte_vdpa_device *dev)
 112 {
 113         struct rte_vdpa_device *cur_dev, *tmp_dev;
 114         int ret = -1;
 115
 116         rte_spinlock_lock(&vdpa_device_list_lock);
 117         RTE_TAILQ_FOREACH_SAFE(cur_dev, &vdpa_device_list, next, tmp_dev) {
 118                 if (dev != cur_dev)
 119                         continue;
 120
 121                 TAILQ_REMOVE(&vdpa_device_list, dev, next);
 122                 rte_free(dev);
 123                 ret = 0;
 124                 break;
 125         }
 126         rte_spinlock_unlock(&vdpa_device_list_lock);
 127
 128         return ret;
 129 }
 130
 131 int
 132 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
 133 {
 134         struct virtio_net *dev = get_device(vid);
 135         uint16_t idx, idx_m, desc_id;
 136         struct vhost_virtqueue *vq;
 137         struct vring_desc desc;
 138         struct vring_desc *desc_ring;
 139         struct vring_desc *idesc = NULL;
 140         struct vring *s_vring;
 141         uint64_t dlen;
 142         uint32_t nr_descs;
 143         int ret;
 144
 145         if (!dev || !vring_m)
 146                 return -1;
 147
 148         if (qid >= dev->nr_vring)
 149                 return -1;
 150
 151         if (vq_is_packed(dev))
 152                 return -1;
 153
 154         s_vring = (struct vring *)vring_m;
 155         vq = dev->virtqueue[qid];
 156         idx = vq->used->idx;
 157         idx_m = s_vring->used->idx;
 158         ret = (uint16_t)(idx_m - idx);
 159
 160         while (idx != idx_m) {
 161                 /* copy used entry, used ring logging is not covered here */
 162                 vq->used->ring[idx & (vq->size - 1)] =
 163                         s_vring->used->ring[idx & (vq->size - 1)];
 164
 165                 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
 166                 desc_ring = vq->desc;
 167                 nr_descs = vq->size;
 168
 169                 if (unlikely(desc_id >= vq->size))
 170                         return -1;
 171
 172                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 173                         dlen = vq->desc[desc_id].len;
 174                         nr_descs = dlen / sizeof(struct vring_desc);
 175                         if (unlikely(nr_descs > vq->size))
 176                                 return -1;
 177
 178                         desc_ring = (struct vring_desc *)(uintptr_t)
 179                                 vhost_iova_to_vva(dev, vq,
 180                                                 vq->desc[desc_id].addr, &dlen,
 181                                                 VHOST_ACCESS_RO);
 182                         if (unlikely(!desc_ring))
 183                                 return -1;
 184
 185                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 186                                 idesc = vhost_alloc_copy_ind_table(dev, vq,
 187                                                 vq->desc[desc_id].addr,
 188                                                 vq->desc[desc_id].len);
 189                                 if (unlikely(!idesc))
 190                                         return -1;
 191
 192                                 desc_ring = idesc;
 193                         }
 194
 195                         desc_id = 0;
 196                 }
 197
 198                 /* dirty page logging for DMA writeable buffer */
 199                 do {
 200                         if (unlikely(desc_id >= vq->size))
 201                                 goto fail;
 202                         if (unlikely(nr_descs-- == 0))
 203                                 goto fail;
 204                         desc = desc_ring[desc_id];
 205                         if (desc.flags & VRING_DESC_F_WRITE)
 206                                 vhost_log_write_iova(dev, vq, desc.addr,
 207                                                      desc.len);
 208                         desc_id = desc.next;
 209                 } while (desc.flags & VRING_DESC_F_NEXT);
 210
 211                 if (unlikely(idesc)) {
 212                         free_ind_table(idesc);
 213                         idesc = NULL;
 214                 }
 215
 216                 idx++;
 217         }
 218
 219         /* used idx is the synchronization point for the split vring */
 220         __atomic_store_n(&vq->used->idx, idx_m, __ATOMIC_RELEASE);
 221
 222         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 223                 vring_used_event(s_vring) = idx_m;
 224
 225         return ret;
 226
 227 fail:
 228         if (unlikely(idesc))
 229                 free_ind_table(idesc);
 230         return -1;
 231 }
 232
 233 int
 234 rte_vdpa_get_queue_num(struct rte_vdpa_device *dev, uint32_t *queue_num)
 235 {
 236         if (dev == NULL || dev->ops == NULL || dev->ops->get_queue_num == NULL)
 237                 return -1;
 238
 239         return dev->ops->get_queue_num(dev, queue_num);
 240 }
 241
 242 int
 243 rte_vdpa_get_features(struct rte_vdpa_device *dev, uint64_t *features)
 244 {
 245         if (dev == NULL || dev->ops == NULL || dev->ops->get_features == NULL)
 246                 return -1;
 247
 248         return dev->ops->get_features(dev, features);
 249 }
 250
 251 int
 252 rte_vdpa_get_protocol_features(struct rte_vdpa_device *dev, uint64_t *features)
 253 {
 254         if (dev == NULL || dev->ops == NULL ||
 255                         dev->ops->get_protocol_features == NULL)
 256                 return -1;
 257
 258         return dev->ops->get_protocol_features(dev, features);
 259 }
 260
 261 int
 262 rte_vdpa_get_stats_names(struct rte_vdpa_device *dev,
 263                 struct rte_vdpa_stat_name *stats_names,
 264                 unsigned int size)
 265 {
 266         if (!dev)
 267                 return -EINVAL;
 268
 269         RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats_names, -ENOTSUP);
 270
 271         return dev->ops->get_stats_names(dev, stats_names, size);
 272 }
 273
 274 int
 275 rte_vdpa_get_stats(struct rte_vdpa_device *dev, uint16_t qid,
 276                 struct rte_vdpa_stat *stats, unsigned int n)
 277 {
 278         if (!dev || !stats || !n)
 279                 return -EINVAL;
 280
 281         RTE_FUNC_PTR_OR_ERR_RET(dev->ops->get_stats, -ENOTSUP);
 282
 283         return dev->ops->get_stats(dev, qid, stats, n);
 284 }
 285
 286 int
 287 rte_vdpa_reset_stats(struct rte_vdpa_device *dev, uint16_t qid)
 288 {
 289         if (!dev)
 290                 return -EINVAL;
 291
 292         RTE_FUNC_PTR_OR_ERR_RET(dev->ops->reset_stats, -ENOTSUP);
 293
 294         return dev->ops->reset_stats(dev, qid);
 295 }
 296
 297 static int
 298 vdpa_dev_match(struct rte_vdpa_device *dev,
 299               const struct rte_device *rte_dev)
 300 {
 301         if (dev->device == rte_dev)
 302                 return 0;
 303
 304         return -1;
 305 }
 306
 307 /* Generic rte_vdpa_dev comparison function. */
 308 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *,
 309                 const struct rte_device *rte_dev);
 310
 311 static struct rte_vdpa_device *
 312 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp,
 313                 struct rte_device *rte_dev)
 314 {
 315         struct rte_vdpa_device *dev;
 316
 317         rte_spinlock_lock(&vdpa_device_list_lock);
 318         if (start == NULL)
 319                 dev = TAILQ_FIRST(&vdpa_device_list);
 320         else
 321                 dev = TAILQ_NEXT(start, next);
 322
 323         while (dev != NULL) {
 324                 if (cmp(dev, rte_dev) == 0)
 325                         break;
 326
 327                 dev = TAILQ_NEXT(dev, next);
 328         }
 329         rte_spinlock_unlock(&vdpa_device_list_lock);
 330
 331         return dev;
 332 }
 333
 334 static void *
 335 vdpa_dev_iterate(const void *start,
 336                 const char *str,
 337                 const struct rte_dev_iterator *it)
 338 {
 339         struct rte_vdpa_device *vdpa_dev = NULL;
 340
 341         RTE_SET_USED(str);
 342
 343         vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device);
 344
 345         return vdpa_dev;
 346 }
 347
 348 static struct rte_class rte_class_vdpa = {
 349         .dev_iterate = vdpa_dev_iterate,
 350 };
 351
 352 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa);