lib/librte_vhost/vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 /**
   6  * @file
   7  *
   8  * Device specific vhost lib
   9  */
  10
  11 #include <stdbool.h>
  12
  13 #include <rte_malloc.h>
  14 #include "rte_vdpa.h"
  15 #include "vhost.h"
  16
  17 static struct rte_vdpa_device *vdpa_devices[MAX_VHOST_DEVICE];
  18 static uint32_t vdpa_device_num;
  19
  20 static bool
  21 is_same_vdpa_device(struct rte_vdpa_dev_addr *a,
  22                 struct rte_vdpa_dev_addr *b)
  23 {
  24         bool ret = true;
  25
  26         if (a->type != b->type)
  27                 return false;
  28
  29         switch (a->type) {
  30         case PCI_ADDR:
  31                 if (a->pci_addr.domain != b->pci_addr.domain ||
  32                                 a->pci_addr.bus != b->pci_addr.bus ||
  33                                 a->pci_addr.devid != b->pci_addr.devid ||
  34                                 a->pci_addr.function != b->pci_addr.function)
  35                         ret = false;
  36                 break;
  37         default:
  38                 break;
  39         }
  40
  41         return ret;
  42 }
  43
  44 int
  45 rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr,
  46                 struct rte_vdpa_dev_ops *ops)
  47 {
  48         struct rte_vdpa_device *dev;
  49         char device_name[MAX_VDPA_NAME_LEN];
  50         int i;
  51
  52         if (vdpa_device_num >= MAX_VHOST_DEVICE)
  53                 return -1;
  54
  55         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
  56                 dev = vdpa_devices[i];
  57                 if (dev && is_same_vdpa_device(&dev->addr, addr))
  58                         return -1;
  59         }
  60
  61         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
  62                 if (vdpa_devices[i] == NULL)
  63                         break;
  64         }
  65
  66         if (i == MAX_VHOST_DEVICE)
  67                 return -1;
  68
  69         sprintf(device_name, "vdpa-dev-%d", i);
  70         dev = rte_zmalloc(device_name, sizeof(struct rte_vdpa_device),
  71                         RTE_CACHE_LINE_SIZE);
  72         if (!dev)
  73                 return -1;
  74
  75         memcpy(&dev->addr, addr, sizeof(struct rte_vdpa_dev_addr));
  76         dev->ops = ops;
  77         vdpa_devices[i] = dev;
  78         vdpa_device_num++;
  79
  80         return i;
  81 }
  82
  83 int
  84 rte_vdpa_unregister_device(int did)
  85 {
  86         if (did < 0 || did >= MAX_VHOST_DEVICE || vdpa_devices[did] == NULL)
  87                 return -1;
  88
  89         rte_free(vdpa_devices[did]);
  90         vdpa_devices[did] = NULL;
  91         vdpa_device_num--;
  92
  93         return did;
  94 }
  95
  96 int
  97 rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)
  98 {
  99         struct rte_vdpa_device *dev;
 100         int i;
 101
 102         for (i = 0; i < MAX_VHOST_DEVICE; ++i) {
 103                 dev = vdpa_devices[i];
 104                 if (dev && is_same_vdpa_device(&dev->addr, addr))
 105                         return i;
 106         }
 107
 108         return -1;
 109 }
 110
 111 struct rte_vdpa_device *
 112 rte_vdpa_get_device(int did)
 113 {
 114         if (did < 0 || did >= MAX_VHOST_DEVICE)
 115                 return NULL;
 116
 117         return vdpa_devices[did];
 118 }
 119
 120 int
 121 rte_vdpa_get_device_num(void)
 122 {
 123         return vdpa_device_num;
 124 }
 125
 126 static bool
 127 invalid_desc_check(struct virtio_net *dev, struct vhost_virtqueue *vq,
 128                 uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
 129 {
 130         uint64_t desc_addr, desc_chunck_len;
 131
 132         while (desc_len) {
 133                 desc_chunck_len = desc_len;
 134                 desc_addr = vhost_iova_to_vva(dev, vq,
 135                                 desc_iova,
 136                                 &desc_chunck_len,
 137                                 perm);
 138
 139                 if (!desc_addr)
 140                         return true;
 141
 142                 desc_len -= desc_chunck_len;
 143                 desc_iova += desc_chunck_len;
 144         }
 145
 146         return false;
 147 }
 148
 149 int __rte_experimental
 150 rte_vdpa_relay_vring_avail(int vid, uint16_t qid, void *vring_m)
 151 {
 152         struct virtio_net *dev = get_device(vid);
 153         uint16_t idx, idx_m, desc_id;
 154         struct vring_desc desc;
 155         struct vhost_virtqueue *vq;
 156         struct vring_desc *desc_ring;
 157         struct vring_desc *idesc = NULL;
 158         struct vring *s_vring;
 159         uint64_t dlen;
 160         uint32_t nr_descs;
 161         int ret;
 162         uint8_t perm;
 163
 164         if (!dev || !vring_m)
 165                 return -1;
 166
 167         if (qid >= dev->nr_vring)
 168                 return -1;
 169
 170         if (vq_is_packed(dev))
 171                 return -1;
 172
 173         s_vring = (struct vring *)vring_m;
 174         vq = dev->virtqueue[qid];
 175         idx = vq->avail->idx;
 176         idx_m = s_vring->avail->idx;
 177         ret = (uint16_t)(idx - idx_m);
 178
 179         while (idx_m != idx) {
 180                 /* avail entry copy */
 181                 desc_id = vq->avail->ring[idx_m & (vq->size - 1)];
 182                 if (unlikely(desc_id >= vq->size))
 183                         return -1;
 184
 185                 s_vring->avail->ring[idx_m & (vq->size - 1)] = desc_id;
 186                 desc_ring = vq->desc;
 187                 nr_descs = vq->size;
 188
 189                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 190                         dlen = vq->desc[desc_id].len;
 191                         nr_descs = dlen / sizeof(struct vring_desc);
 192                         if (unlikely(nr_descs > vq->size))
 193                                 return -1;
 194
 195                         desc_ring = (struct vring_desc *)(uintptr_t)
 196                                 vhost_iova_to_vva(dev, vq,
 197                                                 vq->desc[desc_id].addr, &dlen,
 198                                                 VHOST_ACCESS_RO);
 199                         if (unlikely(!desc_ring))
 200                                 return -1;
 201
 202                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 203                                 idesc = alloc_copy_ind_table(dev, vq,
 204                                                 vq->desc[desc_id].addr,
 205                                                 vq->desc[desc_id].len);
 206                                 if (unlikely(!idesc))
 207                                         return -1;
 208
 209                                 desc_ring = idesc;
 210                         }
 211
 212                         desc_id = 0;
 213                 }
 214
 215                 /* check if the buf addr is within the guest memory */
 216                 do {
 217                         if (unlikely(desc_id >= vq->size))
 218                                 goto fail;
 219                         if (unlikely(nr_descs-- == 0))
 220                                 goto fail;
 221                         desc = desc_ring[desc_id];
 222                         perm = desc.flags & VRING_DESC_F_WRITE ?
 223                                 VHOST_ACCESS_WO : VHOST_ACCESS_RO;
 224                         if (invalid_desc_check(dev, vq, desc.addr, desc.len,
 225                                                 perm))
 226                                 goto fail;
 227                         desc_id = desc.next;
 228                 } while (desc.flags & VRING_DESC_F_NEXT);
 229
 230                 if (unlikely(idesc)) {
 231                         free_ind_table(idesc);
 232                         idesc = NULL;
 233                 }
 234
 235                 idx_m++;
 236         }
 237
 238         rte_smp_wmb();
 239         s_vring->avail->idx = idx;
 240
 241         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 242                 vhost_avail_event(vq) = idx;
 243
 244         return ret;
 245
 246 fail:
 247         if (unlikely(idesc))
 248                 free_ind_table(idesc);
 249         return -1;
 250 }
 251
 252 int __rte_experimental
 253 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
 254 {
 255         struct virtio_net *dev = get_device(vid);
 256         uint16_t idx, idx_m, desc_id;
 257         struct vhost_virtqueue *vq;
 258         struct vring_desc desc;
 259         struct vring_desc *desc_ring;
 260         struct vring_desc *idesc = NULL;
 261         struct vring *s_vring;
 262         uint64_t dlen;
 263         uint32_t nr_descs;
 264         int ret;
 265
 266         if (!dev || !vring_m)
 267                 return -1;
 268
 269         if (qid >= dev->nr_vring)
 270                 return -1;
 271
 272         if (vq_is_packed(dev))
 273                 return -1;
 274
 275         s_vring = (struct vring *)vring_m;
 276         vq = dev->virtqueue[qid];
 277         idx = vq->used->idx;
 278         idx_m = s_vring->used->idx;
 279         ret = (uint16_t)(idx_m - idx);
 280
 281         while (idx != idx_m) {
 282                 /* copy used entry, used ring logging is not covered here */
 283                 vq->used->ring[idx & (vq->size - 1)] =
 284                         s_vring->used->ring[idx & (vq->size - 1)];
 285
 286                 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
 287                 desc_ring = vq->desc;
 288                 nr_descs = vq->size;
 289
 290                 if (unlikely(desc_id >= vq->size))
 291                         return -1;
 292
 293                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 294                         dlen = vq->desc[desc_id].len;
 295                         nr_descs = dlen / sizeof(struct vring_desc);
 296                         if (unlikely(nr_descs > vq->size))
 297                                 return -1;
 298
 299                         desc_ring = (struct vring_desc *)(uintptr_t)
 300                                 vhost_iova_to_vva(dev, vq,
 301                                                 vq->desc[desc_id].addr, &dlen,
 302                                                 VHOST_ACCESS_RO);
 303                         if (unlikely(!desc_ring))
 304                                 return -1;
 305
 306                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 307                                 idesc = alloc_copy_ind_table(dev, vq,
 308                                                 vq->desc[desc_id].addr,
 309                                                 vq->desc[desc_id].len);
 310                                 if (unlikely(!idesc))
 311                                         return -1;
 312
 313                                 desc_ring = idesc;
 314                         }
 315
 316                         desc_id = 0;
 317                 }
 318
 319                 /* dirty page logging for DMA writeable buffer */
 320                 do {
 321                         if (unlikely(desc_id >= vq->size))
 322                                 goto fail;
 323                         if (unlikely(nr_descs-- == 0))
 324                                 goto fail;
 325                         desc = desc_ring[desc_id];
 326                         if (desc.flags & VRING_DESC_F_WRITE)
 327                                 vhost_log_write(dev, desc.addr, desc.len);
 328                         desc_id = desc.next;
 329                 } while (desc.flags & VRING_DESC_F_NEXT);
 330
 331                 if (unlikely(idesc)) {
 332                         free_ind_table(idesc);
 333                         idesc = NULL;
 334                 }
 335
 336                 idx++;
 337         }
 338
 339         rte_smp_wmb();
 340         vq->used->idx = idx_m;
 341
 342         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 343                 vring_used_event(s_vring) = idx_m;
 344
 345         return ret;
 346
 347 fail:
 348         if (unlikely(idesc))
 349                 free_ind_table(idesc);
 350         return -1;
 351 }