X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvhost_user.c;h=b2159e01cc625adc354835e73a8b1edf64ba1363;hb=3ea7052f4b1b;hp=92db73a1af9cfd26944f78bd6ce15673ced23119;hpb=29c7c2fdaa4e88eebee6e3abaa15858d9616b1aa;p=dpdk.git diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c index 92db73a1af..b2159e01cc 100644 --- a/lib/librte_vhost/vhost_user.c +++ b/lib/librte_vhost/vhost_user.c @@ -48,6 +48,7 @@ #include #include +#include "iotlb.h" #include "vhost.h" #include "vhost_user.h" @@ -76,6 +77,8 @@ static const char *vhost_message_str[VHOST_USER_MAX] = { [VHOST_USER_SET_VRING_ENABLE] = "VHOST_USER_SET_VRING_ENABLE", [VHOST_USER_SEND_RARP] = "VHOST_USER_SEND_RARP", [VHOST_USER_NET_SET_MTU] = "VHOST_USER_NET_SET_MTU", + [VHOST_USER_SET_SLAVE_REQ_FD] = "VHOST_USER_SET_SLAVE_REQ_FD", + [VHOST_USER_IOTLB_MSG] = "VHOST_USER_IOTLB_MSG", }; static uint64_t @@ -122,6 +125,11 @@ vhost_backend_cleanup(struct virtio_net *dev) munmap((void *)(uintptr_t)dev->log_addr, dev->log_size); dev->log_addr = 0; } + + if (dev->slave_req_fd >= 0) { + close(dev->slave_req_fd); + dev->slave_req_fd = -1; + } } /* @@ -168,8 +176,12 @@ vhost_user_set_features(struct virtio_net *dev, uint64_t features) uint64_t vhost_features = 0; rte_vhost_driver_get_features(dev->ifname, &vhost_features); - if (features & ~vhost_features) + if (features & ~vhost_features) { + RTE_LOG(ERR, VHOST_CONFIG, + "(%d) received invalid negotiated features.\n", + dev->vid); return -1; + } if ((dev->flags & VIRTIO_DEV_RUNNING) && dev->features != features) { if (dev->notify_ops->features_changed) @@ -197,11 +209,11 @@ vhost_user_set_features(struct virtio_net *dev, uint64_t features) */ static int vhost_user_set_vring_num(struct virtio_net *dev, - struct vhost_vring_state *state) + VhostUserMsg *msg) { - struct vhost_virtqueue *vq = dev->virtqueue[state->index]; + struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index]; - vq->size = state->num; + vq->size = msg->payload.state.num; if (dev->dequeue_zero_copy) { vq->nr_zmbuf = 0; @@ -226,6 +238,15 @@ vhost_user_set_vring_num(struct virtio_net *dev, return -1; } + vq->batch_copy_elems = rte_malloc(NULL, + vq->size * sizeof(struct batch_copy_elem), + RTE_CACHE_LINE_SIZE); + if (!vq->batch_copy_elems) { + RTE_LOG(ERR, VHOST_CONFIG, + "failed to allocate memory for batching copy.\n"); + return -1; + } + return 0; } @@ -242,8 +263,6 @@ numa_realloc(struct virtio_net *dev, int index) struct vhost_virtqueue *old_vq, *vq; int ret; - enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM}; - old_dev = dev; vq = old_vq = dev->virtqueue[index]; @@ -265,7 +284,7 @@ numa_realloc(struct virtio_net *dev, int index) if (!vq) return dev; - memcpy(vq, old_vq, sizeof(*vq) * VIRTIO_QNUM); + memcpy(vq, old_vq, sizeof(*vq)); rte_free(old_vq); } @@ -334,15 +353,31 @@ qva_to_vva(struct virtio_net *dev, uint64_t qva) * This function then converts these to our address space. */ static int -vhost_user_set_vring_addr(struct virtio_net *dev, struct vhost_vring_addr *addr) +vhost_user_set_vring_addr(struct virtio_net *dev, VhostUserMsg *msg) { struct vhost_virtqueue *vq; + struct vhost_vring_addr *addr = &msg->payload.addr; if (dev->mem == NULL) return -1; /* addr->index refers to the queue index. The txq 1, rxq is 0. */ - vq = dev->virtqueue[addr->index]; + vq = dev->virtqueue[msg->payload.addr.index]; + + /* + * Rings addresses should not be interpreted as long as the ring is not + * started and enabled + */ + memcpy(&vq->ring_addrs, addr, sizeof(*addr)); + + return 0; +} + +static struct virtio_net * +translate_ring_addresses(struct virtio_net *dev, int vq_index) +{ + struct vhost_virtqueue *vq = dev->virtqueue[vq_index]; + struct vhost_vring_addr *addr = &vq->ring_addrs; /* The addresses are converted from QEMU virtual to Vhost virtual. */ vq->desc = (struct vring_desc *)(uintptr_t)qva_to_vva(dev, @@ -351,11 +386,11 @@ vhost_user_set_vring_addr(struct virtio_net *dev, struct vhost_vring_addr *addr) RTE_LOG(ERR, VHOST_CONFIG, "(%d) failed to find desc ring address.\n", dev->vid); - return -1; + return NULL; } - dev = numa_realloc(dev, addr->index); - vq = dev->virtqueue[addr->index]; + dev = numa_realloc(dev, vq_index); + vq = dev->virtqueue[vq_index]; vq->avail = (struct vring_avail *)(uintptr_t)qva_to_vva(dev, addr->avail_user_addr); @@ -363,7 +398,7 @@ vhost_user_set_vring_addr(struct virtio_net *dev, struct vhost_vring_addr *addr) RTE_LOG(ERR, VHOST_CONFIG, "(%d) failed to find avail ring address.\n", dev->vid); - return -1; + return NULL; } vq->used = (struct vring_used *)(uintptr_t)qva_to_vva(dev, @@ -372,7 +407,7 @@ vhost_user_set_vring_addr(struct virtio_net *dev, struct vhost_vring_addr *addr) RTE_LOG(ERR, VHOST_CONFIG, "(%d) failed to find used ring address.\n", dev->vid); - return -1; + return NULL; } if (vq->last_used_idx != vq->used->idx) { @@ -395,7 +430,7 @@ vhost_user_set_vring_addr(struct virtio_net *dev, struct vhost_vring_addr *addr) LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n", dev->vid, vq->log_guest_addr); - return 0; + return dev; } /* @@ -403,10 +438,12 @@ vhost_user_set_vring_addr(struct virtio_net *dev, struct vhost_vring_addr *addr) */ static int vhost_user_set_vring_base(struct virtio_net *dev, - struct vhost_vring_state *state) + VhostUserMsg *msg) { - dev->virtqueue[state->index]->last_used_idx = state->num; - dev->virtqueue[state->index]->last_avail_idx = state->num; + dev->virtqueue[msg->payload.state.index]->last_used_idx = + msg->payload.state.num; + dev->virtqueue[msg->payload.state.index]->last_avail_idx = + msg->payload.state.num; return 0; } @@ -664,10 +701,11 @@ vhost_user_set_vring_call(struct virtio_net *dev, struct VhostUserMsg *pmsg) } static void -vhost_user_set_vring_kick(struct virtio_net *dev, struct VhostUserMsg *pmsg) +vhost_user_set_vring_kick(struct virtio_net **pdev, struct VhostUserMsg *pmsg) { struct vhost_vring_file file; struct vhost_virtqueue *vq; + struct virtio_net *dev = *pdev; file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK; if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK) @@ -677,7 +715,26 @@ vhost_user_set_vring_kick(struct virtio_net *dev, struct VhostUserMsg *pmsg) RTE_LOG(INFO, VHOST_CONFIG, "vring kick idx:%d file:%d\n", file.index, file.fd); + /* + * Interpret ring addresses only when ring is started and enabled. + * This is now if protocol features aren't supported. + */ + if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) { + *pdev = dev = translate_ring_addresses(dev, file.index); + if (!dev) + return; + } + vq = dev->virtqueue[file.index]; + + /* + * When VHOST_USER_F_PROTOCOL_FEATURES is not negotiated, + * the ring starts already enabled. Otherwise, it is enabled via + * the SET_VRING_ENABLE message. + */ + if (!(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) + vq->enabled = 1; + if (vq->kickfd >= 0) close(vq->kickfd); vq->kickfd = file.fd; @@ -704,9 +761,9 @@ free_zmbufs(struct vhost_virtqueue *vq) */ static int vhost_user_get_vring_base(struct virtio_net *dev, - struct vhost_vring_state *state) + VhostUserMsg *msg) { - struct vhost_virtqueue *vq = dev->virtqueue[state->index]; + struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index]; /* We have to stop the queue (virtio) if it is running. */ if (dev->flags & VIRTIO_DEV_RUNNING) { @@ -717,10 +774,11 @@ vhost_user_get_vring_base(struct virtio_net *dev, dev->flags &= ~VIRTIO_DEV_READY; /* Here we are safe to get the last used index */ - state->num = vq->last_used_idx; + msg->payload.state.num = vq->last_used_idx; RTE_LOG(INFO, VHOST_CONFIG, - "vring base idx:%d file:%d\n", state->index, state->num); + "vring base idx:%d file:%d\n", msg->payload.state.index, + msg->payload.state.num); /* * Based on current qemu vhost-user implementation, this message is * sent and only sent in vhost_vring_stop. @@ -736,6 +794,9 @@ vhost_user_get_vring_base(struct virtio_net *dev, rte_free(vq->shadow_used_ring); vq->shadow_used_ring = NULL; + rte_free(vq->batch_copy_elems); + vq->batch_copy_elems = NULL; + return 0; } @@ -744,19 +805,34 @@ vhost_user_get_vring_base(struct virtio_net *dev, * enable the virtio queue pair. */ static int -vhost_user_set_vring_enable(struct virtio_net *dev, - struct vhost_vring_state *state) +vhost_user_set_vring_enable(struct virtio_net **pdev, + VhostUserMsg *msg) { - int enable = (int)state->num; + struct virtio_net *dev = *pdev; + int enable = (int)msg->payload.state.num; RTE_LOG(INFO, VHOST_CONFIG, "set queue enable: %d to qp idx: %d\n", - enable, state->index); + enable, msg->payload.state.index); + + /* + * Interpret ring addresses only when ring is started and enabled. + * This is now if protocol features are supported. + */ + if (enable && (dev->features & + (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) { + dev = translate_ring_addresses(dev, msg->payload.state.index); + if (!dev) + return -1; + + *pdev = dev; + } if (dev->notify_ops->vring_state_changed) - dev->notify_ops->vring_state_changed(dev->vid, state->index, enable); + dev->notify_ops->vring_state_changed(dev->vid, + msg->payload.state.index, enable); - dev->virtqueue[state->index]->enabled = enable; + dev->virtqueue[msg->payload.state.index]->enabled = enable; return 0; } @@ -868,6 +944,60 @@ vhost_user_net_set_mtu(struct virtio_net *dev, struct VhostUserMsg *msg) return 0; } +static int +vhost_user_set_req_fd(struct virtio_net *dev, struct VhostUserMsg *msg) +{ + int fd = msg->fds[0]; + + if (fd < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "Invalid file descriptor for slave channel (%d)\n", + fd); + return -1; + } + + dev->slave_req_fd = fd; + + return 0; +} + +static int +vhost_user_iotlb_msg(struct virtio_net *dev, struct VhostUserMsg *msg) +{ + struct vhost_iotlb_msg *imsg = &msg->payload.iotlb; + uint16_t i; + uint64_t vva; + + switch (imsg->type) { + case VHOST_IOTLB_UPDATE: + vva = qva_to_vva(dev, imsg->uaddr); + if (!vva) + return -1; + + for (i = 0; i < dev->nr_vring; i++) { + struct vhost_virtqueue *vq = dev->virtqueue[i]; + + vhost_user_iotlb_cache_insert(vq, imsg->iova, vva, + imsg->size, imsg->perm); + } + break; + case VHOST_IOTLB_INVALIDATE: + for (i = 0; i < dev->nr_vring; i++) { + struct vhost_virtqueue *vq = dev->virtqueue[i]; + + vhost_user_iotlb_cache_remove(vq, imsg->iova, + imsg->size); + } + break; + default: + RTE_LOG(ERR, VHOST_CONFIG, "Invalid IOTLB message type (%d)\n", + imsg->type); + return -1; + } + + return 0; +} + /* return bytes# of read on success or negative val on failure. */ static int read_vhost_message(int sockfd, struct VhostUserMsg *msg) @@ -901,8 +1031,16 @@ read_vhost_message(int sockfd, struct VhostUserMsg *msg) static int send_vhost_message(int sockfd, struct VhostUserMsg *msg) { - int ret; + if (!msg) + return 0; + + return send_fd_message(sockfd, (char *)msg, + VHOST_USER_HDR_SIZE + msg->size, NULL, 0); +} +static int +send_vhost_reply(int sockfd, struct VhostUserMsg *msg) +{ if (!msg) return 0; @@ -911,10 +1049,7 @@ send_vhost_message(int sockfd, struct VhostUserMsg *msg) msg->flags |= VHOST_USER_VERSION; msg->flags |= VHOST_USER_REPLY_MASK; - ret = send_fd_message(sockfd, (char *)msg, - VHOST_USER_HDR_SIZE + msg->size, NULL, 0); - - return ret; + return send_vhost_message(sockfd, msg); } /* @@ -1006,7 +1141,7 @@ vhost_user_msg_handler(int vid, int fd) case VHOST_USER_GET_FEATURES: msg.payload.u64 = vhost_user_get_features(dev); msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_FEATURES: vhost_user_set_features(dev, msg.payload.u64); @@ -1015,7 +1150,7 @@ vhost_user_msg_handler(int vid, int fd) case VHOST_USER_GET_PROTOCOL_FEATURES: msg.payload.u64 = VHOST_USER_PROTOCOL_FEATURES; msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_PROTOCOL_FEATURES: vhost_user_set_protocol_features(dev, msg.payload.u64); @@ -1037,7 +1172,7 @@ vhost_user_msg_handler(int vid, int fd) /* it needs a reply */ msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_LOG_FD: close(msg.fds[0]); @@ -1045,23 +1180,23 @@ vhost_user_msg_handler(int vid, int fd) break; case VHOST_USER_SET_VRING_NUM: - vhost_user_set_vring_num(dev, &msg.payload.state); + vhost_user_set_vring_num(dev, &msg); break; case VHOST_USER_SET_VRING_ADDR: - vhost_user_set_vring_addr(dev, &msg.payload.addr); + vhost_user_set_vring_addr(dev, &msg); break; case VHOST_USER_SET_VRING_BASE: - vhost_user_set_vring_base(dev, &msg.payload.state); + vhost_user_set_vring_base(dev, &msg); break; case VHOST_USER_GET_VRING_BASE: - vhost_user_get_vring_base(dev, &msg.payload.state); + vhost_user_get_vring_base(dev, &msg); msg.size = sizeof(msg.payload.state); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_VRING_KICK: - vhost_user_set_vring_kick(dev, &msg); + vhost_user_set_vring_kick(&dev, &msg); break; case VHOST_USER_SET_VRING_CALL: vhost_user_set_vring_call(dev, &msg); @@ -1076,11 +1211,11 @@ vhost_user_msg_handler(int vid, int fd) case VHOST_USER_GET_QUEUE_NUM: msg.payload.u64 = VHOST_MAX_QUEUE_PAIRS; msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); break; case VHOST_USER_SET_VRING_ENABLE: - vhost_user_set_vring_enable(dev, &msg.payload.state); + vhost_user_set_vring_enable(&dev, &msg); break; case VHOST_USER_SEND_RARP: vhost_user_send_rarp(dev, &msg); @@ -1090,16 +1225,30 @@ vhost_user_msg_handler(int vid, int fd) ret = vhost_user_net_set_mtu(dev, &msg); break; + case VHOST_USER_SET_SLAVE_REQ_FD: + ret = vhost_user_set_req_fd(dev, &msg); + break; + + case VHOST_USER_IOTLB_MSG: + ret = vhost_user_iotlb_msg(dev, &msg); + break; + default: ret = -1; break; } + /* + * The virtio_net struct might have been reallocated on a different + * NUMA node, so dev pointer might no more be valid. + */ + dev = get_device(vid); + if (msg.flags & VHOST_USER_NEED_REPLY) { msg.payload.u64 = !!ret; msg.size = sizeof(msg.payload.u64); - send_vhost_message(fd, &msg); + send_vhost_reply(fd, &msg); } if (!(dev->flags & VIRTIO_DEV_RUNNING) && virtio_is_ready(dev)) { @@ -1118,3 +1267,29 @@ vhost_user_msg_handler(int vid, int fd) return 0; } + +int +vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm) +{ + int ret; + struct VhostUserMsg msg = { + .request = (enum VhostUserRequest)VHOST_USER_SLAVE_IOTLB_MSG, + .flags = VHOST_USER_VERSION, + .size = sizeof(msg.payload.iotlb), + .payload.iotlb = { + .iova = iova, + .perm = perm, + .type = VHOST_IOTLB_MISS, + }, + }; + + ret = send_vhost_message(dev->slave_req_fd, &msg); + if (ret < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to send IOTLB miss message (%d)\n", + ret); + return ret; + } + + return 0; +}