drivers/net/virtio/virtio_user/vhost_vdpa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Red Hat Inc.
   3  */
   4
   5 #include <sys/ioctl.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10
  11 #include <rte_memory.h>
  12
  13 #include "vhost.h"
  14 #include "virtio_user_dev.h"
  15
  16 /* vhost kernel & vdpa ioctls */
  17 #define VHOST_VIRTIO 0xAF
  18 #define VHOST_GET_FEATURES _IOR(VHOST_VIRTIO, 0x00, __u64)
  19 #define VHOST_SET_FEATURES _IOW(VHOST_VIRTIO, 0x00, __u64)
  20 #define VHOST_SET_OWNER _IO(VHOST_VIRTIO, 0x01)
  21 #define VHOST_RESET_OWNER _IO(VHOST_VIRTIO, 0x02)
  22 #define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, __u64)
  23 #define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int)
  24 #define VHOST_SET_VRING_NUM _IOW(VHOST_VIRTIO, 0x10, struct vhost_vring_state)
  25 #define VHOST_SET_VRING_ADDR _IOW(VHOST_VIRTIO, 0x11, struct vhost_vring_addr)
  26 #define VHOST_SET_VRING_BASE _IOW(VHOST_VIRTIO, 0x12, struct vhost_vring_state)
  27 #define VHOST_GET_VRING_BASE _IOWR(VHOST_VIRTIO, 0x12, struct vhost_vring_state)
  28 #define VHOST_SET_VRING_KICK _IOW(VHOST_VIRTIO, 0x20, struct vhost_vring_file)
  29 #define VHOST_SET_VRING_CALL _IOW(VHOST_VIRTIO, 0x21, struct vhost_vring_file)
  30 #define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file)
  31 #define VHOST_NET_SET_BACKEND _IOW(VHOST_VIRTIO, 0x30, struct vhost_vring_file)
  32 #define VHOST_VDPA_GET_DEVICE_ID _IOR(VHOST_VIRTIO, 0x70, __u32)
  33 #define VHOST_VDPA_GET_STATUS _IOR(VHOST_VIRTIO, 0x71, __u8)
  34 #define VHOST_VDPA_SET_STATUS _IOW(VHOST_VIRTIO, 0x72, __u8)
  35 #define VHOST_VDPA_SET_VRING_ENABLE _IOW(VHOST_VIRTIO, 0x75, struct vhost_vring_state)
  36 #define VHOST_SET_BACKEND_FEATURES _IOW(VHOST_VIRTIO, 0x25, __u64)
  37 #define VHOST_GET_BACKEND_FEATURES _IOR(VHOST_VIRTIO, 0x26, __u64)
  38
  39 /* no alignment requirement */
  40 struct vhost_iotlb_msg {
  41         uint64_t iova;
  42         uint64_t size;
  43         uint64_t uaddr;
  44 #define VHOST_ACCESS_RO      0x1
  45 #define VHOST_ACCESS_WO      0x2
  46 #define VHOST_ACCESS_RW      0x3
  47         uint8_t perm;
  48 #define VHOST_IOTLB_MISS           1
  49 #define VHOST_IOTLB_UPDATE         2
  50 #define VHOST_IOTLB_INVALIDATE     3
  51 #define VHOST_IOTLB_ACCESS_FAIL    4
  52 #define VHOST_IOTLB_BATCH_BEGIN    5
  53 #define VHOST_IOTLB_BATCH_END      6
  54         uint8_t type;
  55 };
  56
  57 #define VHOST_IOTLB_MSG_V2 0x2
  58
  59 struct vhost_msg {
  60         uint32_t type;
  61         uint32_t reserved;
  62         union {
  63                 struct vhost_iotlb_msg iotlb;
  64                 uint8_t padding[64];
  65         };
  66 };
  67
  68
  69 static int
  70 vhost_vdpa_ioctl(int fd, uint64_t request, void *arg)
  71 {
  72         int ret;
  73
  74         ret = ioctl(fd, request, arg);
  75         if (ret) {
  76                 PMD_DRV_LOG(ERR, "Vhost-vDPA ioctl %"PRIu64" failed (%s)",
  77                                 request, strerror(errno));
  78                 return -1;
  79         }
  80
  81         return 0;
  82 }
  83
  84 static int
  85 vhost_vdpa_set_owner(struct virtio_user_dev *dev)
  86 {
  87         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_OWNER, NULL);
  88 }
  89
  90 static int
  91 vhost_vdpa_get_backend_features(struct virtio_user_dev *dev, uint64_t *features)
  92 {
  93         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_GET_BACKEND_FEATURES, features);
  94 }
  95
  96 static int
  97 vhost_vdpa_set_backend_features(struct virtio_user_dev *dev, uint64_t features)
  98 {
  99         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_BACKEND_FEATURES, &features);
 100 }
 101
 102 static int
 103 vhost_vdpa_get_features(struct virtio_user_dev *dev, uint64_t *features)
 104 {
 105         int ret;
 106
 107         ret = vhost_vdpa_ioctl(dev->vhostfd, VHOST_GET_FEATURES, features);
 108         if (ret) {
 109                 PMD_DRV_LOG(ERR, "Failed to get features");
 110                 return -1;
 111         }
 112
 113         /* Multiqueue not supported for now */
 114         *features &= ~(1ULL << VIRTIO_NET_F_MQ);
 115
 116         return 0;
 117 }
 118
 119 static int
 120 vhost_vdpa_set_features(struct virtio_user_dev *dev, uint64_t features)
 121 {
 122         /* WORKAROUND */
 123         features |= 1ULL << VIRTIO_F_IOMMU_PLATFORM;
 124
 125         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_FEATURES, &features);
 126 }
 127
 128 static int
 129 vhost_vdpa_iotlb_batch_begin(struct virtio_user_dev *dev)
 130 {
 131         struct vhost_msg msg = {};
 132
 133         if (!(dev->protocol_features & (1ULL << VHOST_BACKEND_F_IOTLB_BATCH)))
 134                 return 0;
 135
 136         if (!(dev->protocol_features & (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2))) {
 137                 PMD_DRV_LOG(ERR, "IOTLB_MSG_V2 not supported by the backend.");
 138                 return -1;
 139         }
 140
 141         msg.type = VHOST_IOTLB_MSG_V2;
 142         msg.iotlb.type = VHOST_IOTLB_BATCH_BEGIN;
 143
 144         if (write(dev->vhostfd, &msg, sizeof(msg)) != sizeof(msg)) {
 145                 PMD_DRV_LOG(ERR, "Failed to send IOTLB batch begin (%s)",
 146                                 strerror(errno));
 147                 return -1;
 148         }
 149
 150         return 0;
 151 }
 152
 153 static int
 154 vhost_vdpa_iotlb_batch_end(struct virtio_user_dev *dev)
 155 {
 156         struct vhost_msg msg = {};
 157
 158         if (!(dev->protocol_features & (1ULL << VHOST_BACKEND_F_IOTLB_BATCH)))
 159                 return 0;
 160
 161         if (!(dev->protocol_features & (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2))) {
 162                 PMD_DRV_LOG(ERR, "IOTLB_MSG_V2 not supported by the backend.");
 163                 return -1;
 164         }
 165
 166         msg.type = VHOST_IOTLB_MSG_V2;
 167         msg.iotlb.type = VHOST_IOTLB_BATCH_END;
 168
 169         if (write(dev->vhostfd, &msg, sizeof(msg)) != sizeof(msg)) {
 170                 PMD_DRV_LOG(ERR, "Failed to send IOTLB batch end (%s)",
 171                                 strerror(errno));
 172                 return -1;
 173         }
 174
 175         return 0;
 176 }
 177
 178 static int
 179 vhost_vdpa_dma_map(struct virtio_user_dev *dev, void *addr,
 180                                   uint64_t iova, size_t len)
 181 {
 182         struct vhost_msg msg = {};
 183
 184         if (!(dev->protocol_features & (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2))) {
 185                 PMD_DRV_LOG(ERR, "IOTLB_MSG_V2 not supported by the backend.");
 186                 return -1;
 187         }
 188
 189         msg.type = VHOST_IOTLB_MSG_V2;
 190         msg.iotlb.type = VHOST_IOTLB_UPDATE;
 191         msg.iotlb.iova = iova;
 192         msg.iotlb.uaddr = (uint64_t)(uintptr_t)addr;
 193         msg.iotlb.size = len;
 194         msg.iotlb.perm = VHOST_ACCESS_RW;
 195
 196         PMD_DRV_LOG(DEBUG, "%s: iova: 0x%" PRIx64 ", addr: %p, len: 0x%zx",
 197                         __func__, iova, addr, len);
 198
 199         if (write(dev->vhostfd, &msg, sizeof(msg)) != sizeof(msg)) {
 200                 PMD_DRV_LOG(ERR, "Failed to send IOTLB update (%s)",
 201                                 strerror(errno));
 202                 return -1;
 203         }
 204
 205         return 0;
 206 }
 207
 208 static int
 209 vhost_vdpa_dma_unmap(struct virtio_user_dev *dev, __rte_unused void *addr,
 210                                   uint64_t iova, size_t len)
 211 {
 212         struct vhost_msg msg = {};
 213
 214         if (!(dev->protocol_features & (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2))) {
 215                 PMD_DRV_LOG(ERR, "IOTLB_MSG_V2 not supported by the backend.");
 216                 return -1;
 217         }
 218
 219         msg.type = VHOST_IOTLB_MSG_V2;
 220         msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
 221         msg.iotlb.iova = iova;
 222         msg.iotlb.size = len;
 223
 224         PMD_DRV_LOG(DEBUG, "%s: iova: 0x%" PRIx64 ", len: 0x%zx",
 225                         __func__, iova, len);
 226
 227         if (write(dev->vhostfd, &msg, sizeof(msg)) != sizeof(msg)) {
 228                 PMD_DRV_LOG(ERR, "Failed to send IOTLB invalidate (%s)",
 229                                 strerror(errno));
 230                 return -1;
 231         }
 232
 233         return 0;
 234 }
 235
 236 static int
 237 vhost_vdpa_dma_map_batch(struct virtio_user_dev *dev, void *addr,
 238                                   uint64_t iova, size_t len)
 239 {
 240         int ret;
 241
 242         if (vhost_vdpa_iotlb_batch_begin(dev) < 0)
 243                 return -1;
 244
 245         ret = vhost_vdpa_dma_map(dev, addr, iova, len);
 246
 247         if (vhost_vdpa_iotlb_batch_end(dev) < 0)
 248                 return -1;
 249
 250         return ret;
 251 }
 252
 253 static int
 254 vhost_vdpa_dma_unmap_batch(struct virtio_user_dev *dev, void *addr,
 255                                   uint64_t iova, size_t len)
 256 {
 257         int ret;
 258
 259         if (vhost_vdpa_iotlb_batch_begin(dev) < 0)
 260                 return -1;
 261
 262         ret = vhost_vdpa_dma_unmap(dev, addr, iova, len);
 263
 264         if (vhost_vdpa_iotlb_batch_end(dev) < 0)
 265                 return -1;
 266
 267         return ret;
 268 }
 269
 270 static int
 271 vhost_vdpa_map_contig(const struct rte_memseg_list *msl,
 272                 const struct rte_memseg *ms, size_t len, void *arg)
 273 {
 274         struct virtio_user_dev *dev = arg;
 275
 276         if (msl->external)
 277                 return 0;
 278
 279         return vhost_vdpa_dma_map(dev, ms->addr, ms->iova, len);
 280 }
 281
 282 static int
 283 vhost_vdpa_map(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 284                 void *arg)
 285 {
 286         struct virtio_user_dev *dev = arg;
 287
 288         /* skip external memory that isn't a heap */
 289         if (msl->external && !msl->heap)
 290                 return 0;
 291
 292         /* skip any segments with invalid IOVA addresses */
 293         if (ms->iova == RTE_BAD_IOVA)
 294                 return 0;
 295
 296         /* if IOVA mode is VA, we've already mapped the internal segments */
 297         if (!msl->external && rte_eal_iova_mode() == RTE_IOVA_VA)
 298                 return 0;
 299
 300         return vhost_vdpa_dma_map(dev, ms->addr, ms->iova, ms->len);
 301 }
 302
 303 static int
 304 vhost_vdpa_set_memory_table(struct virtio_user_dev *dev)
 305 {
 306         int ret;
 307
 308         if (vhost_vdpa_iotlb_batch_begin(dev) < 0)
 309                 return -1;
 310
 311         vhost_vdpa_dma_unmap(dev, NULL, 0, SIZE_MAX);
 312
 313         if (rte_eal_iova_mode() == RTE_IOVA_VA) {
 314                 /* with IOVA as VA mode, we can get away with mapping contiguous
 315                  * chunks rather than going page-by-page.
 316                  */
 317                 ret = rte_memseg_contig_walk_thread_unsafe(
 318                                 vhost_vdpa_map_contig, dev);
 319                 if (ret)
 320                         goto batch_end;
 321                 /* we have to continue the walk because we've skipped the
 322                  * external segments during the config walk.
 323                  */
 324         }
 325         ret = rte_memseg_walk_thread_unsafe(vhost_vdpa_map, dev);
 326
 327 batch_end:
 328         if (vhost_vdpa_iotlb_batch_end(dev) < 0)
 329                 return -1;
 330
 331         return ret;
 332 }
 333
 334 static int
 335 vhost_vdpa_set_vring_enable(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 336 {
 337         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_VDPA_SET_VRING_ENABLE, state);
 338 }
 339
 340 static int
 341 vhost_vdpa_set_vring_num(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 342 {
 343         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_VRING_NUM, state);
 344 }
 345
 346 static int
 347 vhost_vdpa_set_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 348 {
 349         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_VRING_BASE, state);
 350 }
 351
 352 static int
 353 vhost_vdpa_get_vring_base(struct virtio_user_dev *dev, struct vhost_vring_state *state)
 354 {
 355         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_GET_VRING_BASE, state);
 356 }
 357
 358 static int
 359 vhost_vdpa_set_vring_call(struct virtio_user_dev *dev, struct vhost_vring_file *file)
 360 {
 361         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_VRING_CALL, file);
 362 }
 363
 364 static int
 365 vhost_vdpa_set_vring_kick(struct virtio_user_dev *dev, struct vhost_vring_file *file)
 366 {
 367         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_VRING_KICK, file);
 368 }
 369
 370 static int
 371 vhost_vdpa_set_vring_addr(struct virtio_user_dev *dev, struct vhost_vring_addr *addr)
 372 {
 373         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_VRING_ADDR, addr);
 374 }
 375
 376 static int
 377 vhost_vdpa_get_status(struct virtio_user_dev *dev, uint8_t *status)
 378 {
 379         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_VDPA_GET_STATUS, status);
 380 }
 381
 382 static int
 383 vhost_vdpa_set_status(struct virtio_user_dev *dev, uint8_t status)
 384 {
 385         return vhost_vdpa_ioctl(dev->vhostfd, VHOST_VDPA_SET_STATUS, &status);
 386 }
 387
 388 /**
 389  * Set up environment to talk with a vhost vdpa backend.
 390  *
 391  * @return
 392  *   - (-1) if fail to set up;
 393  *   - (>=0) if successful.
 394  */
 395 static int
 396 vhost_vdpa_setup(struct virtio_user_dev *dev)
 397 {
 398         uint32_t did = (uint32_t)-1;
 399
 400         dev->vhostfd = open(dev->path, O_RDWR);
 401         if (dev->vhostfd < 0) {
 402                 PMD_DRV_LOG(ERR, "Failed to open %s: %s\n",
 403                                 dev->path, strerror(errno));
 404                 return -1;
 405         }
 406
 407         if (ioctl(dev->vhostfd, VHOST_VDPA_GET_DEVICE_ID, &did) < 0 ||
 408                         did != VIRTIO_ID_NETWORK) {
 409                 PMD_DRV_LOG(ERR, "Invalid vdpa device ID: %u\n", did);
 410                 return -1;
 411         }
 412
 413         return 0;
 414 }
 415
 416 static int
 417 vhost_vdpa_enable_queue_pair(struct virtio_user_dev *dev,
 418                                uint16_t pair_idx,
 419                                int enable)
 420 {
 421         int i;
 422
 423         if (dev->qp_enabled[pair_idx] == enable)
 424                 return 0;
 425
 426         for (i = 0; i < 2; ++i) {
 427                 struct vhost_vring_state state = {
 428                         .index = pair_idx * 2 + i,
 429                         .num   = enable,
 430                 };
 431
 432                 if (vhost_vdpa_set_vring_enable(dev, &state))
 433                         return -1;
 434         }
 435
 436         dev->qp_enabled[pair_idx] = enable;
 437
 438         return 0;
 439 }
 440
 441 struct virtio_user_backend_ops virtio_ops_vdpa = {
 442         .setup = vhost_vdpa_setup,
 443         .set_owner = vhost_vdpa_set_owner,
 444         .get_features = vhost_vdpa_get_features,
 445         .set_features = vhost_vdpa_set_features,
 446         .get_protocol_features = vhost_vdpa_get_backend_features,
 447         .set_protocol_features = vhost_vdpa_set_backend_features,
 448         .set_memory_table = vhost_vdpa_set_memory_table,
 449         .set_vring_num = vhost_vdpa_set_vring_num,
 450         .set_vring_base = vhost_vdpa_set_vring_base,
 451         .get_vring_base = vhost_vdpa_get_vring_base,
 452         .set_vring_call = vhost_vdpa_set_vring_call,
 453         .set_vring_kick = vhost_vdpa_set_vring_kick,
 454         .set_vring_addr = vhost_vdpa_set_vring_addr,
 455         .get_status = vhost_vdpa_get_status,
 456         .set_status = vhost_vdpa_set_status,
 457         .enable_qp = vhost_vdpa_enable_queue_pair,
 458         .dma_map = vhost_vdpa_dma_map_batch,
 459         .dma_unmap = vhost_vdpa_dma_unmap_batch,
 460 };