lib/librte_vhost/vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 /**
   6  * @file
   7  *
   8  * Device specific vhost lib
   9  */
  10
  11 #include <stdbool.h>
  12
  13 #include <rte_malloc.h>
  14 #include "rte_vdpa.h"
  15 #include "vhost.h"
  16
  17 static struct rte_vdpa_device *vdpa_devices[MAX_VHOST_DEVICE];
  18 static uint32_t vdpa_device_num;
  19
  20 static bool
  21 is_same_vdpa_device(struct rte_vdpa_dev_addr *a,
  22                 struct rte_vdpa_dev_addr *b)
  23 {
  24         bool ret = true;
  25
  26         if (a->type != b->type)
  27                 return false;
  28
  29         switch (a->type) {
  30         case PCI_ADDR:
  31                 if (a->pci_addr.domain != b->pci_addr.domain ||
  32                                 a->pci_addr.bus != b->pci_addr.bus ||
  33                                 a->pci_addr.devid != b->pci_addr.devid ||
  34                                 a->pci_addr.function != b->pci_addr.function)
  35                         ret = false;
  36                 break;
  37         default:
  38                 break;
  39         }
  40
  41         return ret;
  42 }
  43
  44 int
  45 rte_vdpa_register_device(struct rte_vdpa_dev_addr *addr,
  46                 struct rte_vdpa_dev_ops *ops)
  47 {
  48         struct rte_vdpa_device *dev;
  49         char device_name[MAX_VDPA_NAME_LEN];
  50         int i;
  51
  52         if (vdpa_device_num >= MAX_VHOST_DEVICE)
  53                 return -1;
  54
  55         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
  56                 dev = vdpa_devices[i];
  57                 if (dev && is_same_vdpa_device(&dev->addr, addr))
  58                         return -1;
  59         }
  60
  61         for (i = 0; i < MAX_VHOST_DEVICE; i++) {
  62                 if (vdpa_devices[i] == NULL)
  63                         break;
  64         }
  65
  66         if (i == MAX_VHOST_DEVICE)
  67                 return -1;
  68
  69         sprintf(device_name, "vdpa-dev-%d", i);
  70         dev = rte_zmalloc(device_name, sizeof(struct rte_vdpa_device),
  71                         RTE_CACHE_LINE_SIZE);
  72         if (!dev)
  73                 return -1;
  74
  75         memcpy(&dev->addr, addr, sizeof(struct rte_vdpa_dev_addr));
  76         dev->ops = ops;
  77         vdpa_devices[i] = dev;
  78         vdpa_device_num++;
  79
  80         return i;
  81 }
  82
  83 int
  84 rte_vdpa_unregister_device(int did)
  85 {
  86         if (did < 0 || did >= MAX_VHOST_DEVICE || vdpa_devices[did] == NULL)
  87                 return -1;
  88
  89         rte_free(vdpa_devices[did]);
  90         vdpa_devices[did] = NULL;
  91         vdpa_device_num--;
  92
  93         return did;
  94 }
  95
  96 int
  97 rte_vdpa_find_device_id(struct rte_vdpa_dev_addr *addr)
  98 {
  99         struct rte_vdpa_device *dev;
 100         int i;
 101
 102         for (i = 0; i < MAX_VHOST_DEVICE; ++i) {
 103                 dev = vdpa_devices[i];
 104                 if (dev && is_same_vdpa_device(&dev->addr, addr))
 105                         return i;
 106         }
 107
 108         return -1;
 109 }
 110
 111 struct rte_vdpa_device *
 112 rte_vdpa_get_device(int did)
 113 {
 114         if (did < 0 || did >= MAX_VHOST_DEVICE)
 115                 return NULL;
 116
 117         return vdpa_devices[did];
 118 }
 119
 120 int
 121 rte_vdpa_get_device_num(void)
 122 {
 123         return vdpa_device_num;
 124 }
 125
 126 static bool
 127 invalid_desc_check(struct virtio_net *dev, struct vhost_virtqueue *vq,
 128                 uint64_t desc_iova, uint64_t desc_len, uint8_t perm)
 129 {
 130         uint64_t desc_addr, desc_chunck_len;
 131
 132         while (desc_len) {
 133                 desc_chunck_len = desc_len;
 134                 desc_addr = vhost_iova_to_vva(dev, vq,
 135                                 desc_iova,
 136                                 &desc_chunck_len,
 137                                 perm);
 138
 139                 if (!desc_addr)
 140                         return true;
 141
 142                 desc_len -= desc_chunck_len;
 143                 desc_iova += desc_chunck_len;
 144         }
 145
 146         return false;
 147 }
 148
 149 int __rte_experimental
 150 rte_vdpa_relay_vring_avail(int vid, uint16_t qid, void *vring_m)
 151 {
 152         struct virtio_net *dev = get_device(vid);
 153         uint16_t idx, idx_m, desc_id;
 154         struct vring_desc desc;
 155         struct vhost_virtqueue *vq;
 156         struct vring_desc *desc_ring;
 157         struct vring_desc *idesc = NULL;
 158         struct vring *s_vring;
 159         uint64_t dlen;
 160         int ret;
 161         uint8_t perm;
 162
 163         if (!dev || !vring_m)
 164                 return -1;
 165
 166         if (qid >= dev->nr_vring)
 167                 return -1;
 168
 169         if (vq_is_packed(dev))
 170                 return -1;
 171
 172         s_vring = (struct vring *)vring_m;
 173         vq = dev->virtqueue[qid];
 174         idx = vq->avail->idx;
 175         idx_m = s_vring->avail->idx;
 176         ret = (uint16_t)(idx - idx_m);
 177
 178         while (idx_m != idx) {
 179                 /* avail entry copy */
 180                 desc_id = vq->avail->ring[idx_m & (vq->size - 1)];
 181                 s_vring->avail->ring[idx_m & (vq->size - 1)] = desc_id;
 182                 desc_ring = vq->desc;
 183
 184                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 185                         dlen = vq->desc[desc_id].len;
 186                         desc_ring = (struct vring_desc *)(uintptr_t)
 187                                 vhost_iova_to_vva(dev, vq,
 188                                                 vq->desc[desc_id].addr, &dlen,
 189                                                 VHOST_ACCESS_RO);
 190                         if (unlikely(!desc_ring))
 191                                 return -1;
 192
 193                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 194                                 idesc = alloc_copy_ind_table(dev, vq,
 195                                                 vq->desc[desc_id].addr,
 196                                                 vq->desc[desc_id].len);
 197                                 if (unlikely(!idesc))
 198                                         return -1;
 199
 200                                 desc_ring = idesc;
 201                         }
 202
 203                         desc_id = 0;
 204                 }
 205
 206                 /* check if the buf addr is within the guest memory */
 207                 do {
 208                         desc = desc_ring[desc_id];
 209                         perm = desc.flags & VRING_DESC_F_WRITE ?
 210                                 VHOST_ACCESS_WO : VHOST_ACCESS_RO;
 211                         if (invalid_desc_check(dev, vq, desc.addr, desc.len,
 212                                                 perm)) {
 213                                 if (unlikely(idesc))
 214                                         free_ind_table(idesc);
 215                                 return -1;
 216                         }
 217                         desc_id = desc.next;
 218                 } while (desc.flags & VRING_DESC_F_NEXT);
 219
 220                 if (unlikely(idesc)) {
 221                         free_ind_table(idesc);
 222                         idesc = NULL;
 223                 }
 224
 225                 idx_m++;
 226         }
 227
 228         rte_smp_wmb();
 229         s_vring->avail->idx = idx;
 230
 231         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 232                 vhost_avail_event(vq) = idx;
 233
 234         return ret;
 235 }
 236
 237 int __rte_experimental
 238 rte_vdpa_relay_vring_used(int vid, uint16_t qid, void *vring_m)
 239 {
 240         struct virtio_net *dev = get_device(vid);
 241         uint16_t idx, idx_m, desc_id;
 242         struct vhost_virtqueue *vq;
 243         struct vring_desc desc;
 244         struct vring_desc *desc_ring;
 245         struct vring_desc *idesc = NULL;
 246         struct vring *s_vring;
 247         uint64_t dlen;
 248         int ret;
 249
 250         if (!dev || !vring_m)
 251                 return -1;
 252
 253         if (qid >= dev->nr_vring)
 254                 return -1;
 255
 256         if (vq_is_packed(dev))
 257                 return -1;
 258
 259         s_vring = (struct vring *)vring_m;
 260         vq = dev->virtqueue[qid];
 261         idx = vq->used->idx;
 262         idx_m = s_vring->used->idx;
 263         ret = (uint16_t)(idx_m - idx);
 264
 265         while (idx != idx_m) {
 266                 /* copy used entry, used ring logging is not covered here */
 267                 vq->used->ring[idx & (vq->size - 1)] =
 268                         s_vring->used->ring[idx & (vq->size - 1)];
 269
 270                 desc_id = vq->used->ring[idx & (vq->size - 1)].id;
 271                 desc_ring = vq->desc;
 272
 273                 if (vq->desc[desc_id].flags & VRING_DESC_F_INDIRECT) {
 274                         dlen = vq->desc[desc_id].len;
 275                         desc_ring = (struct vring_desc *)(uintptr_t)
 276                                 vhost_iova_to_vva(dev, vq,
 277                                                 vq->desc[desc_id].addr, &dlen,
 278                                                 VHOST_ACCESS_RO);
 279                         if (unlikely(!desc_ring))
 280                                 return -1;
 281
 282                         if (unlikely(dlen < vq->desc[desc_id].len)) {
 283                                 idesc = alloc_copy_ind_table(dev, vq,
 284                                                 vq->desc[desc_id].addr,
 285                                                 vq->desc[desc_id].len);
 286                                 if (unlikely(!idesc))
 287                                         return -1;
 288
 289                                 desc_ring = idesc;
 290                         }
 291
 292                         desc_id = 0;
 293                 }
 294
 295                 /* dirty page logging for DMA writeable buffer */
 296                 do {
 297                         desc = desc_ring[desc_id];
 298                         if (desc.flags & VRING_DESC_F_WRITE)
 299                                 vhost_log_write(dev, desc.addr, desc.len);
 300                         desc_id = desc.next;
 301                 } while (desc.flags & VRING_DESC_F_NEXT);
 302
 303                 if (unlikely(idesc)) {
 304                         free_ind_table(idesc);
 305                         idesc = NULL;
 306                 }
 307
 308                 idx++;
 309         }
 310
 311         rte_smp_wmb();
 312         vq->used->idx = idx_m;
 313
 314         if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))
 315                 vring_used_event(s_vring) = idx_m;
 316
 317         return ret;
 318 }