lib/librte_vhost/vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 /**
   6  * @file
   7  *
   8  * Device specific vhost lib
   9  */
  10
  11 #include <stdbool.h>
  12
  13 #include <rte_class.h>
  14 #include <rte_malloc.h>
  15 #include <rte_pci.h>
  16 #include "rte_vdpa.h"
  17 #include "vhost.h"
  18
  19 static struct rte_vdpa_device vdpa_devices[MAX_VHOST_DEVICE];
  20 static uint32_t vdpa_device_num;
  21
  22 static bool
  23 is_same_vdpa_device(struct rte_vdpa_dev_addr *a,
  24                 struct rte_vdpa_dev_addr *b)
  25 {
  26         bool ret = true;
  27
  28         if (a->type != b->type)
  29                 return false;
  30
  31         switch (a->type) {
  32         case VDPA_ADDR_PCI:
  33                 if (a->pci_addr.domain != b->pci_addr.domain ||
  34                                 a->pci_addr.bus != b->pci_addr.bus ||
  35                                 a->pci_addr.devid != b->pci_addr.devid ||
  36                                 a->pci_addr.function != b->pci_addr.function)
  37                         ret = false;
  38                 break;
  39         default:
  40                 break;
  41         }
  42
  43         return ret;
  44 }
  45
  46 int
  47 rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr,
  48                 struct rte_vdpa_dev_ops *ops)
  49 {
  50         struct rte_vdpa_device *dev;
  51         int i;
  52
  53         if (vdpa_device_num >= MAX_VHOST_DEVICE || addr == NULL || ops == NULL)
  54                 return -1;
  55
  56         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
  57                 dev = &vdpa_devices[i];
  58                 if (dev->ops && is_same_vdpa_device(&dev->addr, addr))
  59                         return -1;
  60         }
  61
  62         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
  63                 if (vdpa_devices[i].ops == NULL)
  64                         break;
  65         }
  66
  67         if (i == MAX_VHOST_DEVICE)
  68                 return -1;
  69
  70         dev = &vdpa_devices[i];
  71         memcpy(&dev->addr, addr, sizeof(struct rte_vdpa_dev_addr));
  72         dev->ops = ops;
  73         vdpa_device_num++;
  74
  75         return i;
  76 }
  77
  78 int
  79 rte_vdpa_unregister_device(int did)
  80 {
  81         if (did < 0 || did >= MAX_VHOST_DEVICE || vdpa_devices[did].ops == NULL)
  82                 return -1;
  83
  84         memset(&vdpa_devices[did], 0, sizeof(struct rte_vdpa_device));
  85         vdpa_device_num--;
  86
  87         return did;
  88 }
  89
  90 int
  91 rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)
  92 {
  93         struct rte_vdpa_device *dev;
  94         int i;
  95
  96         if (addr == NULL)
  97                 return -1;
  98
  99         for (i = 0; i < MAX_VHOST_DEVICE; ++i) {
 100                 dev = &vdpa_devices[i];
 101                 if (dev->ops == NULL)
 102                         continue;
 103
 104                 if (is_same_vdpa_device(&dev->addr, addr))
 105                         return i;
 106         }
 107
 108         return -1;
 109 }
 110
 111 struct rte_vdpa_device *
 112 rte_vdpa_get_device(int did)
 113 {
 114         if (did < 0 || did >= MAX_VHOST_DEVICE)
 115                 return NULL;
 116
 117         return &vdpa_devices[did];
 118 }
 119
 120 int
 121 rte_vdpa_get_device_num(void)
 122 {
 123         return vdpa_device_num;
 124 }
 125
 126 int
 127 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
 128 {
 129         struct virtio_net *dev = get_device(vid);
 130         uint16_t idx, idx_m, desc_id;
 131         struct vhost_virtqueue *vq;
 132         struct vring_desc desc;
 133         struct vring_desc *desc_ring;
 134         struct vring_desc *idesc = NULL;
 135         struct vring *s_vring;
 136         uint64_t dlen;
 137         uint32_t nr_descs;
 138         int ret;
 139
 140         if (!dev || !vring_m)
 141                 return -1;
 142
 143         if (qid >= dev->nr_vring)
 144                 return -1;
 145
 146         if (vq_is_packed(dev))
 147                 return -1;
 148
 149         s_vring = (struct vring *)vring_m;
 150         vq = dev->virtqueue[qid];
 151         idx = vq->used->idx;
 152         idx_m = s_vring->used->idx;
 153         ret = (uint16_t)(idx_m - idx);
 154
 155         while (idx != idx_m) {
 156                 /* copy used entry, used ring logging is not covered here */
 157                 vq->used->ring[idx & (vq->size - 1)] =
 158                         s_vring->used->ring[idx & (vq->size - 1)];
 159
 160                 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
 161                 desc_ring = vq->desc;
 162                 nr_descs = vq->size;
 163
 164                 if (unlikely(desc_id >= vq->size))
 165                         return -1;
 166
 167                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 168                         dlen = vq->desc[desc_id].len;
 169                         nr_descs = dlen / sizeof(struct vring_desc);
 170                         if (unlikely(nr_descs > vq->size))
 171                                 return -1;
 172
 173                         desc_ring = (struct vring_desc *)(uintptr_t)
 174                                 vhost_iova_to_vva(dev, vq,
 175                                                 vq->desc[desc_id].addr, &dlen,
 176                                                 VHOST_ACCESS_RO);
 177                         if (unlikely(!desc_ring))
 178                                 return -1;
 179
 180                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 181                                 idesc = vhost_alloc_copy_ind_table(dev, vq,
 182                                                 vq->desc[desc_id].addr,
 183                                                 vq->desc[desc_id].len);
 184                                 if (unlikely(!idesc))
 185                                         return -1;
 186
 187                                 desc_ring = idesc;
 188                         }
 189
 190                         desc_id = 0;
 191                 }
 192
 193                 /* dirty page logging for DMA writeable buffer */
 194                 do {
 195                         if (unlikely(desc_id >= vq->size))
 196                                 goto fail;
 197                         if (unlikely(nr_descs-- == 0))
 198                                 goto fail;
 199                         desc = desc_ring[desc_id];
 200                         if (desc.flags & VRING_DESC_F_WRITE)
 201                                 vhost_log_write_iova(dev, vq, desc.addr,
 202                                                      desc.len);
 203                         desc_id = desc.next;
 204                 } while (desc.flags & VRING_DESC_F_NEXT);
 205
 206                 if (unlikely(idesc)) {
 207                         free_ind_table(idesc);
 208                         idesc = NULL;
 209                 }
 210
 211                 idx++;
 212         }
 213
 214         rte_smp_wmb();
 215         vq->used->idx = idx_m;
 216
 217         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 218                 vring_used_event(s_vring) = idx_m;
 219
 220         return ret;
 221
 222 fail:
 223         if (unlikely(idesc))
 224                 free_ind_table(idesc);
 225         return -1;
 226 }
 227
 228 int
 229 rte_vdpa_get_stats_names(int did, struct rte_vdpa_stat_name *stats_names,
 230                          unsigned int size)
 231 {
 232         struct rte_vdpa_device *vdpa_dev;
 233
 234         vdpa_dev = rte_vdpa_get_device(did);
 235         if (!vdpa_dev)
 236                 return -ENODEV;
 237
 238         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_stats_names, -ENOTSUP);
 239
 240         return vdpa_dev->ops->get_stats_names(did, stats_names, size);
 241 }
 242
 243 int
 244 rte_vdpa_get_stats(int did, uint16_t qid, struct rte_vdpa_stat *stats,
 245                    unsigned int n)
 246 {
 247         struct rte_vdpa_device *vdpa_dev;
 248
 249         vdpa_dev = rte_vdpa_get_device(did);
 250         if (!vdpa_dev)
 251                 return -ENODEV;
 252
 253         if (!stats || !n)
 254                 return -EINVAL;
 255
 256         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_stats, -ENOTSUP);
 257
 258         return vdpa_dev->ops->get_stats(did, qid, stats, n);
 259 }
 260
 261 int
 262 rte_vdpa_reset_stats(int did, uint16_t qid)
 263 {
 264         struct rte_vdpa_device *vdpa_dev;
 265
 266         vdpa_dev = rte_vdpa_get_device(did);
 267         if (!vdpa_dev)
 268                 return -ENODEV;
 269
 270         RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->reset_stats, -ENOTSUP);
 271
 272         return vdpa_dev->ops->reset_stats(did, qid);
 273 }
 274
 275 static uint16_t
 276 vdpa_dev_to_id(const struct rte_vdpa_device *dev)
 277 {
 278         if (dev == NULL)
 279                 return MAX_VHOST_DEVICE;
 280
 281         if (dev < &vdpa_devices[0] ||
 282                         dev >= &vdpa_devices[MAX_VHOST_DEVICE])
 283                 return MAX_VHOST_DEVICE;
 284
 285         return (uint16_t)(dev - vdpa_devices);
 286 }
 287
 288 static int
 289 vdpa_dev_match(struct rte_vdpa_device *dev,
 290               const struct rte_device *rte_dev)
 291 {
 292         struct rte_vdpa_dev_addr addr;
 293
 294         /*  Only PCI bus supported for now */
 295         if (strcmp(rte_dev->bus->name, "pci") != 0)
 296                 return -1;
 297
 298         addr.type = VDPA_ADDR_PCI;
 299
 300         if (rte_pci_addr_parse(rte_dev->name, &addr.pci_addr) != 0)
 301                 return -1;
 302
 303         if (!is_same_vdpa_device(&dev->addr, &addr))
 304                 return -1;
 305
 306         return 0;
 307 }
 308
 309 /* Generic rte_vdpa_dev comparison function. */
 310 typedef int (*rte_vdpa_cmp_t)(struct rte_vdpa_device *,
 311                 const struct rte_device *rte_dev);
 312
 313 static struct rte_vdpa_device *
 314 vdpa_find_device(const struct rte_vdpa_device *start, rte_vdpa_cmp_t cmp,
 315                 struct rte_device *rte_dev)
 316 {
 317         struct rte_vdpa_device *dev;
 318         uint16_t idx;
 319
 320         if (start != NULL)
 321                 idx = vdpa_dev_to_id(start) + 1;
 322         else
 323                 idx = 0;
 324         for (; idx < MAX_VHOST_DEVICE; idx++) {
 325                 dev = &vdpa_devices[idx];
 326                 /*
 327                  * ToDo: Certainly better to introduce a state field,
 328                  * but rely on ops being set for now.
 329                  */
 330                 if (dev->ops == NULL)
 331                         continue;
 332                 if (cmp(dev, rte_dev) == 0)
 333                         return dev;
 334         }
 335         return NULL;
 336 }
 337
 338 static void *
 339 vdpa_dev_iterate(const void *start,
 340                 const char *str,
 341                 const struct rte_dev_iterator *it)
 342 {
 343         struct rte_vdpa_device *vdpa_dev = NULL;
 344
 345         RTE_SET_USED(str);
 346
 347         vdpa_dev = vdpa_find_device(start, vdpa_dev_match, it->device);
 348
 349         return vdpa_dev;
 350 }
 351
 352 static struct rte_class rte_class_vdpa = {
 353         .dev_iterate = vdpa_dev_iterate,
 354 };
 355
 356 RTE_REGISTER_CLASS(vdpa, rte_class_vdpa);