[VHOST_USER_NET_SET_MTU] = "VHOST_USER_NET_SET_MTU",
[VHOST_USER_SET_SLAVE_REQ_FD] = "VHOST_USER_SET_SLAVE_REQ_FD",
[VHOST_USER_IOTLB_MSG] = "VHOST_USER_IOTLB_MSG",
+ [VHOST_USER_CRYPTO_CREATE_SESS] = "VHOST_USER_CRYPTO_CREATE_SESS",
+ [VHOST_USER_CRYPTO_CLOSE_SESS] = "VHOST_USER_CRYPTO_CLOSE_SESS",
};
static uint64_t
static int
vhost_user_reset_owner(struct virtio_net *dev)
{
- struct rte_vdpa_device *vdpa_dev;
- int did = -1;
-
- if (dev->flags & VIRTIO_DEV_RUNNING) {
- did = dev->vdpa_dev_id;
- vdpa_dev = rte_vdpa_get_device(did);
- if (vdpa_dev && vdpa_dev->ops->dev_close)
- vdpa_dev->ops->dev_close(dev->vid);
- dev->flags &= ~VIRTIO_DEV_RUNNING;
- dev->notify_ops->destroy_device(dev->vid);
- }
+ vhost_destroy_device_notify(dev);
cleanup_device(dev, 0);
reset_device(dev);
dev->notify_ops->features_changed(dev->vid, features);
}
- did = dev->vdpa_dev_id;
- vdpa_dev = rte_vdpa_get_device(did);
- if (vdpa_dev && vdpa_dev->ops->set_features)
- vdpa_dev->ops->set_features(dev->vid);
-
dev->features = features;
if (dev->features &
((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) {
}
}
+ did = dev->vdpa_dev_id;
+ vdpa_dev = rte_vdpa_get_device(did);
+ if (vdpa_dev && vdpa_dev->ops->set_features)
+ vdpa_dev->ops->set_features(dev->vid);
+
return 0;
}
/* Converts QEMU virtual address to Vhost virtual address. */
static uint64_t
-qva_to_vva(struct virtio_net *dev, uint64_t qva)
+qva_to_vva(struct virtio_net *dev, uint64_t qva, uint64_t *len)
{
- struct rte_vhost_mem_region *reg;
+ struct rte_vhost_mem_region *r;
uint32_t i;
/* Find the region where the address lives. */
for (i = 0; i < dev->mem->nregions; i++) {
- reg = &dev->mem->regions[i];
+ r = &dev->mem->regions[i];
- if (qva >= reg->guest_user_addr &&
- qva < reg->guest_user_addr + reg->size) {
- return qva - reg->guest_user_addr +
- reg->host_user_addr;
+ if (qva >= r->guest_user_addr &&
+ qva < r->guest_user_addr + r->size) {
+
+ if (unlikely(*len > r->guest_user_addr + r->size - qva))
+ *len = r->guest_user_addr + r->size - qva;
+
+ return qva - r->guest_user_addr +
+ r->host_user_addr;
}
}
+ *len = 0;
return 0;
}
*/
static uint64_t
ring_addr_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq,
- uint64_t ra, uint64_t size)
+ uint64_t ra, uint64_t *size)
{
if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
uint64_t vva;
vva = vhost_user_iotlb_cache_find(vq, ra,
- &size, VHOST_ACCESS_RW);
+ size, VHOST_ACCESS_RW);
if (!vva)
vhost_user_iotlb_miss(dev, ra, VHOST_ACCESS_RW);
return vva;
}
- return qva_to_vva(dev, ra);
+ return qva_to_vva(dev, ra, size);
}
static struct virtio_net *
{
struct vhost_virtqueue *vq = dev->virtqueue[vq_index];
struct vhost_vring_addr *addr = &vq->ring_addrs;
+ uint64_t len;
/* The addresses are converted from QEMU virtual to Vhost virtual. */
if (vq->desc && vq->avail && vq->used)
return dev;
+ len = sizeof(struct vring_desc) * vq->size;
vq->desc = (struct vring_desc *)(uintptr_t)ring_addr_to_vva(dev,
- vq, addr->desc_user_addr, sizeof(struct vring_desc));
- if (vq->desc == 0) {
+ vq, addr->desc_user_addr, &len);
+ if (vq->desc == 0 || len != sizeof(struct vring_desc) * vq->size) {
RTE_LOG(DEBUG, VHOST_CONFIG,
- "(%d) failed to find desc ring address.\n",
+ "(%d) failed to map desc ring.\n",
dev->vid);
return dev;
}
vq = dev->virtqueue[vq_index];
addr = &vq->ring_addrs;
+ len = sizeof(struct vring_avail) + sizeof(uint16_t) * vq->size;
vq->avail = (struct vring_avail *)(uintptr_t)ring_addr_to_vva(dev,
- vq, addr->avail_user_addr, sizeof(struct vring_avail));
- if (vq->avail == 0) {
+ vq, addr->avail_user_addr, &len);
+ if (vq->avail == 0 ||
+ len != sizeof(struct vring_avail) +
+ sizeof(uint16_t) * vq->size) {
RTE_LOG(DEBUG, VHOST_CONFIG,
- "(%d) failed to find avail ring address.\n",
+ "(%d) failed to map avail ring.\n",
dev->vid);
return dev;
}
+ len = sizeof(struct vring_used) +
+ sizeof(struct vring_used_elem) * vq->size;
vq->used = (struct vring_used *)(uintptr_t)ring_addr_to_vva(dev,
- vq, addr->used_user_addr, sizeof(struct vring_used));
- if (vq->used == 0) {
+ vq, addr->used_user_addr, &len);
+ if (vq->used == 0 || len != sizeof(struct vring_used) +
+ sizeof(struct vring_used_elem) * vq->size) {
RTE_LOG(DEBUG, VHOST_CONFIG,
- "(%d) failed to find used ring address.\n",
+ "(%d) failed to map used ring.\n",
dev->vid);
return dev;
}
}
static int
-vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg)
+vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *pmsg)
{
+ struct virtio_net *dev = *pdev;
struct VhostUserMemory memory = pmsg->payload.memory;
struct rte_vhost_mem_region *reg;
void *mmap_addr;
mmap_offset);
}
+ for (i = 0; i < dev->nr_vring; i++) {
+ struct vhost_virtqueue *vq = dev->virtqueue[i];
+
+ if (vq->desc || vq->avail || vq->used) {
+ /*
+ * If the memory table got updated, the ring addresses
+ * need to be translated again as virtual addresses have
+ * changed.
+ */
+ vring_invalidate(dev, vq);
+
+ dev = translate_ring_addresses(dev, i);
+ if (!dev)
+ return -1;
+
+ *pdev = dev;
+ }
+ }
+
dump_guest_pages(dev);
return 0;
VhostUserMsg *msg)
{
struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
- struct rte_vdpa_device *vdpa_dev;
- int did = -1;
/* We have to stop the queue (virtio) if it is running. */
- if (dev->flags & VIRTIO_DEV_RUNNING) {
- did = dev->vdpa_dev_id;
- vdpa_dev = rte_vdpa_get_device(did);
- if (vdpa_dev && vdpa_dev->ops->dev_close)
- vdpa_dev->ops->dev_close(dev->vid);
- dev->flags &= ~VIRTIO_DEV_RUNNING;
- dev->notify_ops->destroy_device(dev->vid);
- }
+ vhost_destroy_device_notify(dev);
dev->flags &= ~VIRTIO_DEV_READY;
dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;
struct virtio_net *dev = *pdev;
struct vhost_iotlb_msg *imsg = &msg->payload.iotlb;
uint16_t i;
- uint64_t vva;
+ uint64_t vva, len;
switch (imsg->type) {
case VHOST_IOTLB_UPDATE:
- vva = qva_to_vva(dev, imsg->uaddr);
+ len = imsg->size;
+ vva = qva_to_vva(dev, imsg->uaddr, &len);
if (!vva)
return -1;
struct vhost_virtqueue *vq = dev->virtqueue[i];
vhost_user_iotlb_cache_insert(vq, imsg->iova, vva,
- imsg->size, imsg->perm);
+ len, imsg->perm);
if (is_vring_iotlb_update(vq, imsg))
*pdev = dev = translate_ring_addresses(dev, i);
return send_vhost_message(sockfd, msg, NULL, 0);
}
+static int
+send_vhost_slave_message(struct virtio_net *dev, struct VhostUserMsg *msg,
+ int *fds, int fd_num)
+{
+ int ret;
+
+ if (msg->flags & VHOST_USER_NEED_REPLY)
+ rte_spinlock_lock(&dev->slave_req_lock);
+
+ ret = send_vhost_message(dev->slave_req_fd, msg, fds, fd_num);
+ if (ret < 0 && (msg->flags & VHOST_USER_NEED_REPLY))
+ rte_spinlock_unlock(&dev->slave_req_lock);
+
+ return ret;
+}
+
/*
* Allocate a queue pair if it hasn't been allocated yet
*/
break;
case VHOST_USER_SET_MEM_TABLE:
- ret = vhost_user_set_mem_table(dev, &msg);
+ ret = vhost_user_set_mem_table(&dev, &msg);
break;
case VHOST_USER_SET_LOG_BASE:
if (vdpa_dev->ops->dev_conf)
vdpa_dev->ops->dev_conf(dev->vid);
dev->flags |= VIRTIO_DEV_VDPA_CONFIGURED;
+ if (vhost_user_host_notifier_ctrl(dev->vid, true) != 0) {
+ RTE_LOG(INFO, VHOST_CONFIG,
+ "(%d) software relay is used for vDPA, performance may be low.\n",
+ dev->vid);
+ }
}
return 0;
}
+static int process_slave_message_reply(struct virtio_net *dev,
+ const VhostUserMsg *msg)
+{
+ VhostUserMsg msg_reply;
+ int ret;
+
+ if ((msg->flags & VHOST_USER_NEED_REPLY) == 0)
+ return 0;
+
+ if (read_vhost_message(dev->slave_req_fd, &msg_reply) < 0) {
+ ret = -1;
+ goto out;
+ }
+
+ if (msg_reply.request.slave != msg->request.slave) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "Received unexpected msg type (%u), expected %u\n",
+ msg_reply.request.slave, msg->request.slave);
+ ret = -1;
+ goto out;
+ }
+
+ ret = msg_reply.payload.u64 ? -1 : 0;
+
+out:
+ rte_spinlock_unlock(&dev->slave_req_lock);
+ return ret;
+}
+
int
vhost_user_iotlb_miss(struct virtio_net *dev, uint64_t iova, uint8_t perm)
{
return 0;
}
+
+static int vhost_user_slave_set_vring_host_notifier(struct virtio_net *dev,
+ int index, int fd,
+ uint64_t offset,
+ uint64_t size)
+{
+ int *fdp = NULL;
+ size_t fd_num = 0;
+ int ret;
+ struct VhostUserMsg msg = {
+ .request.slave = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
+ .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY,
+ .size = sizeof(msg.payload.area),
+ .payload.area = {
+ .u64 = index & VHOST_USER_VRING_IDX_MASK,
+ .size = size,
+ .offset = offset,
+ },
+ };
+
+ if (fd < 0)
+ msg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
+ else {
+ fdp = &fd;
+ fd_num = 1;
+ }
+
+ ret = send_vhost_slave_message(dev, &msg, fdp, fd_num);
+ if (ret < 0) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "Failed to set host notifier (%d)\n", ret);
+ return ret;
+ }
+
+ return process_slave_message_reply(dev, &msg);
+}
+
+int vhost_user_host_notifier_ctrl(int vid, bool enable)
+{
+ struct virtio_net *dev;
+ struct rte_vdpa_device *vdpa_dev;
+ int vfio_device_fd, did, ret = 0;
+ uint64_t offset, size;
+ unsigned int i;
+
+ dev = get_device(vid);
+ if (!dev)
+ return -ENODEV;
+
+ did = dev->vdpa_dev_id;
+ if (did < 0)
+ return -EINVAL;
+
+ if (!(dev->features & (1ULL << VIRTIO_F_VERSION_1)) ||
+ !(dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES)) ||
+ !(dev->protocol_features &
+ (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ)) ||
+ !(dev->protocol_features &
+ (1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD)) ||
+ !(dev->protocol_features &
+ (1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER)))
+ return -ENOTSUP;
+
+ vdpa_dev = rte_vdpa_get_device(did);
+
+ RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_vfio_device_fd, -ENOTSUP);
+ RTE_FUNC_PTR_OR_ERR_RET(vdpa_dev->ops->get_notify_area, -ENOTSUP);
+
+ vfio_device_fd = vdpa_dev->ops->get_vfio_device_fd(vid);
+ if (vfio_device_fd < 0)
+ return -ENOTSUP;
+
+ if (enable) {
+ for (i = 0; i < dev->nr_vring; i++) {
+ if (vdpa_dev->ops->get_notify_area(vid, i, &offset,
+ &size) < 0) {
+ ret = -ENOTSUP;
+ goto disable;
+ }
+
+ if (vhost_user_slave_set_vring_host_notifier(dev, i,
+ vfio_device_fd, offset, size) < 0) {
+ ret = -EFAULT;
+ goto disable;
+ }
+ }
+ } else {
+disable:
+ for (i = 0; i < dev->nr_vring; i++) {
+ vhost_user_slave_set_vring_host_notifier(dev, i, -1,
+ 0, 0);
+ }
+ }
+
+ return ret;
+}
git.droids-corp.org / dpdk.git / blobdiff