lib/librte_vhost/vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 /**
   6  * @file
   7  *
   8  * Device specific vhost lib
   9  */
  10
  11 #include <stdbool.h>
  12
  13 #include <rte_malloc.h>
  14 #include "rte_vdpa.h"
  15 #include "vhost.h"
  16
  17 static struct rte_vdpa_device *vdpa_devices[MAX_VHOST_DEVICE];
  18 static uint32_t vdpa_device_num;
  19
  20 static bool
  21 is_same_vdpa_device(struct rte_vdpa_dev_addr *a,
  22                 struct rte_vdpa_dev_addr *b)
  23 {
  24         bool ret = true;
  25
  26         if (a->type != b->type)
  27                 return false;
  28
  29         switch (a->type) {
  30         case PCI_ADDR:
  31                 if (a->pci_addr.domain != b->pci_addr.domain ||
  32                                 a->pci_addr.bus != b->pci_addr.bus ||
  33                                 a->pci_addr.devid != b->pci_addr.devid ||
  34                                 a->pci_addr.function != b->pci_addr.function)
  35                         ret = false;
  36                 break;
  37         default:
  38                 break;
  39         }
  40
  41         return ret;
  42 }
  43
  44 int
  45 rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr,
  46                 struct rte_vdpa_dev_ops *ops)
  47 {
  48         struct rte_vdpa_device *dev;
  49         char device_name[MAX_VDPA_NAME_LEN];
  50         int i;
  51
  52         if (vdpa_device_num >= MAX_VHOST_DEVICE)
  53                 return -1;
  54
  55         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
  56                 dev = vdpa_devices[i];
  57                 if (dev && is_same_vdpa_device(&dev->addr, addr))
  58                         return -1;
  59         }
  60
  61         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
  62                 if (vdpa_devices[i] == NULL)
  63                         break;
  64         }
  65
  66         if (i == MAX_VHOST_DEVICE)
  67                 return -1;
  68
  69         sprintf(device_name, "vdpa-dev-%d", i);
  70         dev = rte_zmalloc(device_name, sizeof(struct rte_vdpa_device),
  71                         RTE_CACHE_LINE_SIZE);
  72         if (!dev)
  73                 return -1;
  74
  75         memcpy(&dev->addr, addr, sizeof(struct rte_vdpa_dev_addr));
  76         dev->ops = ops;
  77         vdpa_devices[i] = dev;
  78         vdpa_device_num++;
  79
  80         return i;
  81 }
  82
  83 int
  84 rte_vdpa_unregister_device(int did)
  85 {
  86         if (did < 0 || did >= MAX_VHOST_DEVICE || vdpa_devices[did] == NULL)
  87                 return -1;
  88
  89         rte_free(vdpa_devices[did]);
  90         vdpa_devices[did] = NULL;
  91         vdpa_device_num--;
  92
  93         return did;
  94 }
  95
  96 int
  97 rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)
  98 {
  99         struct rte_vdpa_device *dev;
 100         int i;
 101
 102         for (i = 0; i < MAX_VHOST_DEVICE; ++i) {
 103                 dev = vdpa_devices[i];
 104                 if (dev && is_same_vdpa_device(&dev->addr, addr))
 105                         return i;
 106         }
 107
 108         return -1;
 109 }
 110
 111 struct rte_vdpa_device *
 112 rte_vdpa_get_device(int did)
 113 {
 114         if (did < 0 || did >= MAX_VHOST_DEVICE)
 115                 return NULL;
 116
 117         return vdpa_devices[did];
 118 }
 119
 120 int
 121 rte_vdpa_get_device_num(void)
 122 {
 123         return vdpa_device_num;
 124 }
 125
 126 static bool
 127 invalid_desc_check(struct virtio_net *dev, struct vhost_virtqueue *vq,
 128                 uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
 129 {
 130         uint64_t desc_addr, desc_chunck_len;
 131
 132         while (desc_len) {
 133                 desc_chunck_len = desc_len;
 134                 desc_addr = vhost_iova_to_vva(dev, vq,
 135                                 desc_iova,
 136                                 &desc_chunck_len,
 137                                 perm);
 138
 139                 if (!desc_addr)
 140                         return true;
 141
 142                 desc_len -= desc_chunck_len;
 143                 desc_iova += desc_chunck_len;
 144         }
 145
 146         return false;
 147 }
 148
 149 int __rte_experimental
 150 rte_vdpa_relay_vring_avail(int vid, uint16_t qid, void *vring_m)
 151 {
 152         struct virtio_net *dev = get_device(vid);
 153         uint16_t idx, idx_m, desc_id;
 154         struct vring_desc desc;
 155         struct vhost_virtqueue *vq;
 156         struct vring_desc *desc_ring;
 157         struct vring_desc *idesc = NULL;
 158         struct vring *s_vring;
 159         uint64_t dlen;
 160         int ret;
 161         uint8_t perm;
 162
 163         if (!dev || !vring_m)
 164                 return -1;
 165
 166         if (qid >= dev->nr_vring)
 167                 return -1;
 168
 169         if (vq_is_packed(dev))
 170                 return -1;
 171
 172         s_vring = (struct vring *)vring_m;
 173         vq = dev->virtqueue[qid];
 174         idx = vq->avail->idx;
 175         idx_m = s_vring->avail->idx;
 176         ret = (uint16_t)(idx - idx_m);
 177
 178         while (idx_m != idx) {
 179                 /* avail entry copy */
 180                 desc_id = vq->avail->ring[idx_m & (vq->size - 1)];
 181                 if (unlikely(desc_id >= vq->size))
 182                         return -1;
 183
 184                 s_vring->avail->ring[idx_m & (vq->size - 1)] = desc_id;
 185                 desc_ring = vq->desc;
 186
 187                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 188                         dlen = vq->desc[desc_id].len;
 189                         desc_ring = (struct vring_desc *)(uintptr_t)
 190                                 vhost_iova_to_vva(dev, vq,
 191                                                 vq->desc[desc_id].addr, &dlen,
 192                                                 VHOST_ACCESS_RO);
 193                         if (unlikely(!desc_ring))
 194                                 return -1;
 195
 196                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 197                                 idesc = alloc_copy_ind_table(dev, vq,
 198                                                 vq->desc[desc_id].addr,
 199                                                 vq->desc[desc_id].len);
 200                                 if (unlikely(!idesc))
 201                                         return -1;
 202
 203                                 desc_ring = idesc;
 204                         }
 205
 206                         desc_id = 0;
 207                 }
 208
 209                 /* check if the buf addr is within the guest memory */
 210                 do {
 211                         if (unlikely(desc_id >= vq->size))
 212                                 goto fail;
 213                         desc = desc_ring[desc_id];
 214                         perm = desc.flags & VRING_DESC_F_WRITE ?
 215                                 VHOST_ACCESS_WO : VHOST_ACCESS_RO;
 216                         if (invalid_desc_check(dev, vq, desc.addr, desc.len,
 217                                                 perm))
 218                                 goto fail;
 219                         desc_id = desc.next;
 220                 } while (desc.flags & VRING_DESC_F_NEXT);
 221
 222                 if (unlikely(idesc)) {
 223                         free_ind_table(idesc);
 224                         idesc = NULL;
 225                 }
 226
 227                 idx_m++;
 228         }
 229
 230         rte_smp_wmb();
 231         s_vring->avail->idx = idx;
 232
 233         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 234                 vhost_avail_event(vq) = idx;
 235
 236         return ret;
 237
 238 fail:
 239         if (unlikely(idesc))
 240                 free_ind_table(idesc);
 241         return -1;
 242 }
 243
 244 int __rte_experimental
 245 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
 246 {
 247         struct virtio_net *dev = get_device(vid);
 248         uint16_t idx, idx_m, desc_id;
 249         struct vhost_virtqueue *vq;
 250         struct vring_desc desc;
 251         struct vring_desc *desc_ring;
 252         struct vring_desc *idesc = NULL;
 253         struct vring *s_vring;
 254         uint64_t dlen;
 255         int ret;
 256
 257         if (!dev || !vring_m)
 258                 return -1;
 259
 260         if (qid >= dev->nr_vring)
 261                 return -1;
 262
 263         if (vq_is_packed(dev))
 264                 return -1;
 265
 266         s_vring = (struct vring *)vring_m;
 267         vq = dev->virtqueue[qid];
 268         idx = vq->used->idx;
 269         idx_m = s_vring->used->idx;
 270         ret = (uint16_t)(idx_m - idx);
 271
 272         while (idx != idx_m) {
 273                 /* copy used entry, used ring logging is not covered here */
 274                 vq->used->ring[idx & (vq->size - 1)] =
 275                         s_vring->used->ring[idx & (vq->size - 1)];
 276
 277                 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
 278                 desc_ring = vq->desc;
 279
 280                 if (unlikely(desc_id >= vq->size))
 281                         return -1;
 282
 283                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 284                         dlen = vq->desc[desc_id].len;
 285                         desc_ring = (struct vring_desc *)(uintptr_t)
 286                                 vhost_iova_to_vva(dev, vq,
 287                                                 vq->desc[desc_id].addr, &dlen,
 288                                                 VHOST_ACCESS_RO);
 289                         if (unlikely(!desc_ring))
 290                                 return -1;
 291
 292                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 293                                 idesc = alloc_copy_ind_table(dev, vq,
 294                                                 vq->desc[desc_id].addr,
 295                                                 vq->desc[desc_id].len);
 296                                 if (unlikely(!idesc))
 297                                         return -1;
 298
 299                                 desc_ring = idesc;
 300                         }
 301
 302                         desc_id = 0;
 303                 }
 304
 305                 /* dirty page logging for DMA writeable buffer */
 306                 do {
 307                         if (unlikely(desc_id >= vq->size))
 308                                 goto fail;
 309                         desc = desc_ring[desc_id];
 310                         if (desc.flags & VRING_DESC_F_WRITE)
 311                                 vhost_log_write(dev, desc.addr, desc.len);
 312                         desc_id = desc.next;
 313                 } while (desc.flags & VRING_DESC_F_NEXT);
 314
 315                 if (unlikely(idesc)) {
 316                         free_ind_table(idesc);
 317                         idesc = NULL;
 318                 }
 319
 320                 idx++;
 321         }
 322
 323         rte_smp_wmb();
 324         vq->used->idx = idx_m;
 325
 326         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 327                 vring_used_event(s_vring) = idx_m;
 328
 329         return ret;
 330
 331 fail:
 332         if (unlikely(idesc))
 333                 free_ind_table(idesc);
 334         return -1;
 335 }