X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvirtio-net.c;h=fe1a77e80b73d1a887259f364c02139a4a764163;hb=21e10f983ebcd90798ffd394ab28ed8f11f923ee;hp=b520ec5ee2afdac8cb6e0fb6f883f20877c76c3d;hpb=af295ad4698c81913965f1354d9d3ad054e988f6;p=dpdk.git diff --git a/lib/librte_vhost/virtio-net.c b/lib/librte_vhost/virtio-net.c index b520ec5ee2..fe1a77e80b 100644 --- a/lib/librte_vhost/virtio-net.c +++ b/lib/librte_vhost/virtio-net.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #ifdef RTE_LIBRTE_VHOST_NUMA @@ -67,11 +68,24 @@ struct virtio_net_device_ops const *notify_ops; /* root address of the linked list of managed virtio devices */ static struct virtio_net_config_ll *ll_root; +#define VHOST_USER_F_PROTOCOL_FEATURES 30 + /* Features supported by this lib. */ #define VHOST_SUPPORTED_FEATURES ((1ULL << VIRTIO_NET_F_MRG_RXBUF) | \ (1ULL << VIRTIO_NET_F_CTRL_VQ) | \ (1ULL << VIRTIO_NET_F_CTRL_RX) | \ - (1ULL << VHOST_F_LOG_ALL)) + (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \ + (VHOST_SUPPORTS_MQ) | \ + (1ULL << VIRTIO_F_VERSION_1) | \ + (1ULL << VHOST_F_LOG_ALL) | \ + (1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \ + (1ULL << VIRTIO_NET_F_HOST_TSO4) | \ + (1ULL << VIRTIO_NET_F_HOST_TSO6) | \ + (1ULL << VIRTIO_NET_F_CSUM) | \ + (1ULL << VIRTIO_NET_F_GUEST_CSUM) | \ + (1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ + (1ULL << VIRTIO_NET_F_GUEST_TSO6)) + static uint64_t VHOST_FEATURES = VHOST_SUPPORTED_FEATURES; @@ -175,29 +189,30 @@ add_config_ll_entry(struct virtio_net_config_ll *new_ll_dev) } +static void +cleanup_vq(struct vhost_virtqueue *vq, int destroy) +{ + if ((vq->callfd >= 0) && (destroy != 0)) + close(vq->callfd); + if (vq->kickfd >= 0) + close(vq->kickfd); +} + /* * Unmap any memory, close any file descriptors and * free any memory owned by a device. */ static void -cleanup_device(struct virtio_net *dev) +cleanup_device(struct virtio_net *dev, int destroy) { - /* Unmap QEMU memory file if mapped. */ - if (dev->mem) { - munmap((void *)(uintptr_t)dev->mem->mapped_address, - (size_t)dev->mem->mapped_size); - free(dev->mem); - } + uint32_t i; + + vhost_backend_cleanup(dev); - /* Close any event notifiers opened by device. */ - if ((int)dev->virtqueue[VIRTIO_RXQ]->callfd >= 0) - close((int)dev->virtqueue[VIRTIO_RXQ]->callfd); - if ((int)dev->virtqueue[VIRTIO_RXQ]->kickfd >= 0) - close((int)dev->virtqueue[VIRTIO_RXQ]->kickfd); - if ((int)dev->virtqueue[VIRTIO_TXQ]->callfd >= 0) - close((int)dev->virtqueue[VIRTIO_TXQ]->callfd); - if ((int)dev->virtqueue[VIRTIO_TXQ]->kickfd >= 0) - close((int)dev->virtqueue[VIRTIO_TXQ]->kickfd); + for (i = 0; i < dev->virt_qp_nb; i++) { + cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ], destroy); + cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ], destroy); + } } /* @@ -206,9 +221,11 @@ cleanup_device(struct virtio_net *dev) static void free_device(struct virtio_net_config_ll *ll_dev) { - /* Free any malloc'd memory */ - rte_free(ll_dev->dev.virtqueue[VIRTIO_RXQ]); - rte_free(ll_dev->dev.virtqueue[VIRTIO_TXQ]); + uint32_t i; + + for (i = 0; i < ll_dev->dev.virt_qp_nb; i++) + rte_free(ll_dev->dev.virtqueue[i * VIRTIO_QNUM]); + rte_free(ll_dev); } @@ -222,17 +239,17 @@ rm_config_ll_entry(struct virtio_net_config_ll *ll_dev, /* First remove the device and then clean it up. */ if (ll_dev == ll_root) { ll_root = ll_dev->next; - cleanup_device(&ll_dev->dev); + cleanup_device(&ll_dev->dev, 1); free_device(ll_dev); return ll_root; } else { if (likely(ll_dev_last != NULL)) { ll_dev_last->next = ll_dev->next; - cleanup_device(&ll_dev->dev); + cleanup_device(&ll_dev->dev, 1); free_device(ll_dev); return ll_dev_last->next; } else { - cleanup_device(&ll_dev->dev); + cleanup_device(&ll_dev->dev, 1); free_device(ll_dev); RTE_LOG(ERR, VHOST_CONFIG, "Remove entry from config_ll failed\n"); @@ -241,34 +258,91 @@ rm_config_ll_entry(struct virtio_net_config_ll *ll_dev, } } -/* - * Initialise all variables in device structure. - */ static void -init_device(struct virtio_net *dev) +init_vring_queue(struct vhost_virtqueue *vq, int qp_idx) { - uint64_t vq_offset; + memset(vq, 0, sizeof(struct vhost_virtqueue)); - /* - * Virtqueues have already been malloced so - * we don't want to set them to NULL. - */ - vq_offset = offsetof(struct virtio_net, mem); + vq->kickfd = -1; + vq->callfd = -1; + + /* Backends are set to -1 indicating an inactive device. */ + vq->backend = -1; + + /* always set the default vq pair to enabled */ + if (qp_idx == 0) + vq->enabled = 1; +} - /* Set everything to 0. */ - memset((void *)(uintptr_t)((uint64_t)(uintptr_t)dev + vq_offset), 0, - (sizeof(struct virtio_net) - (size_t)vq_offset)); - memset(dev->virtqueue[VIRTIO_RXQ], 0, sizeof(struct vhost_virtqueue)); - memset(dev->virtqueue[VIRTIO_TXQ], 0, sizeof(struct vhost_virtqueue)); +static void +init_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) +{ + uint32_t base_idx = qp_idx * VIRTIO_QNUM; - dev->virtqueue[VIRTIO_RXQ]->kickfd = (eventfd_t)-1; - dev->virtqueue[VIRTIO_RXQ]->callfd = (eventfd_t)-1; - dev->virtqueue[VIRTIO_TXQ]->kickfd = (eventfd_t)-1; - dev->virtqueue[VIRTIO_TXQ]->callfd = (eventfd_t)-1; + init_vring_queue(dev->virtqueue[base_idx + VIRTIO_RXQ], qp_idx); + init_vring_queue(dev->virtqueue[base_idx + VIRTIO_TXQ], qp_idx); +} - /* Backends are set to -1 indicating an inactive device. */ - dev->virtqueue[VIRTIO_RXQ]->backend = VIRTIO_DEV_STOPPED; - dev->virtqueue[VIRTIO_TXQ]->backend = VIRTIO_DEV_STOPPED; +static void +reset_vring_queue(struct vhost_virtqueue *vq, int qp_idx) +{ + int callfd; + + callfd = vq->callfd; + init_vring_queue(vq, qp_idx); + vq->callfd = callfd; +} + +static void +reset_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) +{ + uint32_t base_idx = qp_idx * VIRTIO_QNUM; + + reset_vring_queue(dev->virtqueue[base_idx + VIRTIO_RXQ], qp_idx); + reset_vring_queue(dev->virtqueue[base_idx + VIRTIO_TXQ], qp_idx); +} + +static int +alloc_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx) +{ + struct vhost_virtqueue *virtqueue = NULL; + uint32_t virt_rx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_RXQ; + uint32_t virt_tx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_TXQ; + + virtqueue = rte_malloc(NULL, + sizeof(struct vhost_virtqueue) * VIRTIO_QNUM, 0); + if (virtqueue == NULL) { + RTE_LOG(ERR, VHOST_CONFIG, + "Failed to allocate memory for virt qp:%d.\n", qp_idx); + return -1; + } + + dev->virtqueue[virt_rx_q_idx] = virtqueue; + dev->virtqueue[virt_tx_q_idx] = virtqueue + VIRTIO_TXQ; + + init_vring_queue_pair(dev, qp_idx); + + dev->virt_qp_nb += 1; + + return 0; +} + +/* + * Reset some variables in device structure, while keeping few + * others untouched, such as device_fh, ifname, virt_qp_nb: they + * should be same unless the device is removed. + */ +static void +reset_device(struct virtio_net *dev) +{ + uint32_t i; + + dev->features = 0; + dev->protocol_features = 0; + dev->flags = 0; + + for (i = 0; i < dev->virt_qp_nb; i++) + reset_vring_queue_pair(dev, i); } /* @@ -276,14 +350,13 @@ init_device(struct virtio_net *dev) * initialised and a new entry is added to the device configuration linked * list. */ -static int -new_device(struct vhost_device_ctx ctx) +int +vhost_new_device(struct vhost_device_ctx ctx) { struct virtio_net_config_ll *new_ll_dev; - struct vhost_virtqueue *virtqueue_rx, *virtqueue_tx; /* Setup device and virtqueues. */ - new_ll_dev = rte_malloc(NULL, sizeof(struct virtio_net_config_ll), 0); + new_ll_dev = rte_zmalloc(NULL, sizeof(struct virtio_net_config_ll), 0); if (new_ll_dev == NULL) { RTE_LOG(ERR, VHOST_CONFIG, "(%"PRIu64") Failed to allocate memory for dev.\n", @@ -291,31 +364,6 @@ new_device(struct vhost_device_ctx ctx) return -1; } - virtqueue_rx = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); - if (virtqueue_rx == NULL) { - rte_free(new_ll_dev); - RTE_LOG(ERR, VHOST_CONFIG, - "(%"PRIu64") Failed to allocate memory for rxq.\n", - ctx.fh); - return -1; - } - - virtqueue_tx = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0); - if (virtqueue_tx == NULL) { - rte_free(virtqueue_rx); - rte_free(new_ll_dev); - RTE_LOG(ERR, VHOST_CONFIG, - "(%"PRIu64") Failed to allocate memory for txq.\n", - ctx.fh); - return -1; - } - - new_ll_dev->dev.virtqueue[VIRTIO_RXQ] = virtqueue_rx; - new_ll_dev->dev.virtqueue[VIRTIO_TXQ] = virtqueue_tx; - - /* Initialise device and virtqueues. */ - init_device(&new_ll_dev->dev); - new_ll_dev->next = NULL; /* Add entry to device configuration linked list. */ @@ -328,8 +376,8 @@ new_device(struct vhost_device_ctx ctx) * Function is called from the CUSE release function. This function will * cleanup the device and remove it from device configuration linked list. */ -static void -destroy_device(struct vhost_device_ctx ctx) +void +vhost_destroy_device(struct vhost_device_ctx ctx) { struct virtio_net_config_ll *ll_dev_cur_ctx, *ll_dev_last = NULL; struct virtio_net_config_ll *ll_dev_cur = ll_root; @@ -357,8 +405,8 @@ destroy_device(struct vhost_device_ctx ctx) } } -static void -set_ifname(struct vhost_device_ctx ctx, +void +vhost_set_ifname(struct vhost_device_ctx ctx, const char *if_name, unsigned int if_len) { struct virtio_net *dev; @@ -380,8 +428,8 @@ set_ifname(struct vhost_device_ctx ctx, * This function just returns success at the moment unless * the device hasn't been initialised. */ -static int -set_owner(struct vhost_device_ctx ctx) +int +vhost_set_owner(struct vhost_device_ctx ctx) { struct virtio_net *dev; @@ -395,16 +443,20 @@ set_owner(struct vhost_device_ctx ctx) /* * Called from CUSE IOCTL: VHOST_RESET_OWNER */ -static int -reset_owner(struct vhost_device_ctx ctx) +int +vhost_reset_owner(struct vhost_device_ctx ctx) { - struct virtio_net_config_ll *ll_dev; + struct virtio_net *dev; - ll_dev = get_config_ll_entry(ctx); + dev = get_device(ctx); + if (dev == NULL) + return -1; - cleanup_device(&ll_dev->dev); - init_device(&ll_dev->dev); + if (dev->flags & VIRTIO_DEV_RUNNING) + notify_ops->destroy_device(dev); + cleanup_device(dev, 0); + reset_device(dev); return 0; } @@ -412,8 +464,8 @@ reset_owner(struct vhost_device_ctx ctx) * Called from CUSE IOCTL: VHOST_GET_FEATURES * The features that we support are requested. */ -static int -get_features(struct vhost_device_ctx ctx, uint64_t *pu) +int +vhost_get_features(struct vhost_device_ctx ctx, uint64_t *pu) { struct virtio_net *dev; @@ -430,10 +482,12 @@ get_features(struct vhost_device_ctx ctx, uint64_t *pu) * Called from CUSE IOCTL: VHOST_SET_FEATURES * We receive the negotiated features supported by us and the virtio device. */ -static int -set_features(struct vhost_device_ctx ctx, uint64_t *pu) +int +vhost_set_features(struct vhost_device_ctx ctx, uint64_t *pu) { struct virtio_net *dev; + uint16_t vhost_hlen; + uint16_t i; dev = get_device(ctx); if (dev == NULL) @@ -441,27 +495,26 @@ set_features(struct vhost_device_ctx ctx, uint64_t *pu) if (*pu & ~VHOST_FEATURES) return -1; - /* Store the negotiated feature list for the device. */ dev->features = *pu; - - /* Set the vhost_hlen depending on if VIRTIO_NET_F_MRG_RXBUF is set. */ - if (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) { - LOG_DEBUG(VHOST_CONFIG, - "(%"PRIu64") Mergeable RX buffers enabled\n", - dev->device_fh); - dev->virtqueue[VIRTIO_RXQ]->vhost_hlen = - sizeof(struct virtio_net_hdr_mrg_rxbuf); - dev->virtqueue[VIRTIO_TXQ]->vhost_hlen = - sizeof(struct virtio_net_hdr_mrg_rxbuf); + if (dev->features & + ((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) { + vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf); } else { - LOG_DEBUG(VHOST_CONFIG, - "(%"PRIu64") Mergeable RX buffers disabled\n", - dev->device_fh); - dev->virtqueue[VIRTIO_RXQ]->vhost_hlen = - sizeof(struct virtio_net_hdr); - dev->virtqueue[VIRTIO_TXQ]->vhost_hlen = - sizeof(struct virtio_net_hdr); + vhost_hlen = sizeof(struct virtio_net_hdr); + } + LOG_DEBUG(VHOST_CONFIG, + "(%"PRIu64") Mergeable RX buffers %s, virtio 1 %s\n", + dev->device_fh, + (dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off", + (dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off"); + + for (i = 0; i < dev->virt_qp_nb; i++) { + uint16_t base_idx = i * VIRTIO_QNUM; + + dev->virtqueue[base_idx + VIRTIO_RXQ]->vhost_hlen = vhost_hlen; + dev->virtqueue[base_idx + VIRTIO_TXQ]->vhost_hlen = vhost_hlen; } + return 0; } @@ -469,8 +522,9 @@ set_features(struct vhost_device_ctx ctx, uint64_t *pu) * Called from CUSE IOCTL: VHOST_SET_VRING_NUM * The virtio device sends us the size of the descriptor ring. */ -static int -set_vring_num(struct vhost_device_ctx ctx, struct vhost_vring_state *state) +int +vhost_set_vring_num(struct vhost_device_ctx ctx, + struct vhost_vring_state *state) { struct virtio_net *dev; @@ -485,7 +539,7 @@ set_vring_num(struct vhost_device_ctx ctx, struct vhost_vring_state *state) } /* - * Reallocate virtio_det and vhost_virtqueue data structure to make them on the + * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the * same numa node as the memory of vring descriptor. */ #ifdef RTE_LIBRTE_VHOST_NUMA @@ -571,14 +625,14 @@ numa_realloc(struct virtio_net *dev, int index __rte_unused) * The virtio device sends us the desc, used and avail ring addresses. * This function then converts these to our address space. */ -static int -set_vring_addr(struct vhost_device_ctx ctx, struct vhost_vring_addr *addr) +int +vhost_set_vring_addr(struct vhost_device_ctx ctx, struct vhost_vring_addr *addr) { struct virtio_net *dev; struct vhost_virtqueue *vq; dev = get_device(ctx); - if (dev == NULL) + if ((dev == NULL) || (dev->mem == NULL)) return -1; /* addr->index refers to the queue index. The txq 1, rxq is 0. */ @@ -615,12 +669,16 @@ set_vring_addr(struct vhost_device_ctx ctx, struct vhost_vring_addr *addr) return -1; } + vq->log_guest_addr = addr->log_guest_addr; + LOG_DEBUG(VHOST_CONFIG, "(%"PRIu64") mapped address desc: %p\n", dev->device_fh, vq->desc); LOG_DEBUG(VHOST_CONFIG, "(%"PRIu64") mapped address avail: %p\n", dev->device_fh, vq->avail); LOG_DEBUG(VHOST_CONFIG, "(%"PRIu64") mapped address used: %p\n", dev->device_fh, vq->used); + LOG_DEBUG(VHOST_CONFIG, "(%"PRIu64") log_guest_addr: %"PRIx64"\n", + dev->device_fh, vq->log_guest_addr); return 0; } @@ -629,8 +687,9 @@ set_vring_addr(struct vhost_device_ctx ctx, struct vhost_vring_addr *addr) * Called from CUSE IOCTL: VHOST_SET_VRING_BASE * The virtio device sends us the available ring last used index. */ -static int -set_vring_base(struct vhost_device_ctx ctx, struct vhost_vring_state *state) +int +vhost_set_vring_base(struct vhost_device_ctx ctx, + struct vhost_vring_state *state) { struct virtio_net *dev; @@ -649,8 +708,8 @@ set_vring_base(struct vhost_device_ctx ctx, struct vhost_vring_state *state) * Called from CUSE IOCTL: VHOST_GET_VRING_BASE * We send the virtio device our available ring last used index. */ -static int -get_vring_base(struct vhost_device_ctx ctx, uint32_t index, +int +vhost_get_vring_base(struct vhost_device_ctx ctx, uint32_t index, struct vhost_vring_state *state) { struct virtio_net *dev; @@ -672,21 +731,32 @@ get_vring_base(struct vhost_device_ctx ctx, uint32_t index, * The virtio device sends an eventfd to interrupt the guest. This fd gets * copied into our process space. */ -static int -set_vring_call(struct vhost_device_ctx ctx, struct vhost_vring_file *file) +int +vhost_set_vring_call(struct vhost_device_ctx ctx, struct vhost_vring_file *file) { struct virtio_net *dev; struct vhost_virtqueue *vq; + uint32_t cur_qp_idx = file->index / VIRTIO_QNUM; dev = get_device(ctx); if (dev == NULL) return -1; + /* + * FIXME: VHOST_SET_VRING_CALL is the first per-vring message + * we get, so we do vring queue pair allocation here. + */ + if (cur_qp_idx + 1 > dev->virt_qp_nb) { + if (alloc_vring_queue_pair(dev, cur_qp_idx) < 0) + return -1; + } + /* file->index refers to the queue index. The txq is 1, rxq is 0. */ vq = dev->virtqueue[file->index]; + assert(vq != NULL); - if ((int)vq->callfd >= 0) - close((int)vq->callfd); + if (vq->callfd >= 0) + close(vq->callfd); vq->callfd = file->fd; @@ -698,8 +768,8 @@ set_vring_call(struct vhost_device_ctx ctx, struct vhost_vring_file *file) * The virtio device sends an eventfd that it can use to notify us. * This fd gets copied into our process space. */ -static int -set_vring_kick(struct vhost_device_ctx ctx, struct vhost_vring_file *file) +int +vhost_set_vring_kick(struct vhost_device_ctx ctx, struct vhost_vring_file *file) { struct virtio_net *dev; struct vhost_virtqueue *vq; @@ -711,8 +781,8 @@ set_vring_kick(struct vhost_device_ctx ctx, struct vhost_vring_file *file) /* file->index refers to the queue index. The txq is 1, rxq is 0. */ vq = dev->virtqueue[file->index]; - if ((int)vq->kickfd >= 0) - close((int)vq->kickfd); + if (vq->kickfd >= 0) + close(vq->kickfd); vq->kickfd = file->fd; @@ -728,8 +798,8 @@ set_vring_kick(struct vhost_device_ctx ctx, struct vhost_vring_file *file) * At that point we remove the device from the data core. * The device will still exist in the device configuration linked list. */ -static int -set_backend(struct vhost_device_ctx ctx, struct vhost_vring_file *file) +int +vhost_set_backend(struct vhost_device_ctx ctx, struct vhost_vring_file *file) { struct virtio_net *dev; @@ -756,42 +826,6 @@ set_backend(struct vhost_device_ctx ctx, struct vhost_vring_file *file) return 0; } -/* - * Function pointers are set for the device operations to allow CUSE to call - * functions when an IOCTL, device_add or device_release is received. - */ -static const struct vhost_net_device_ops vhost_device_ops = { - .new_device = new_device, - .destroy_device = destroy_device, - - .set_ifname = set_ifname, - - .get_features = get_features, - .set_features = set_features, - - .set_vring_num = set_vring_num, - .set_vring_addr = set_vring_addr, - .set_vring_base = set_vring_base, - .get_vring_base = get_vring_base, - - .set_vring_kick = set_vring_kick, - .set_vring_call = set_vring_call, - - .set_backend = set_backend, - - .set_owner = set_owner, - .reset_owner = reset_owner, -}; - -/* - * Called by main to setup callbacks when registering CUSE device. - */ -struct vhost_net_device_ops const * -get_virtio_net_callbacks(void) -{ - return &vhost_device_ops; -} - int rte_vhost_enable_guest_notification(struct virtio_net *dev, uint16_t queue_id, int enable) { @@ -801,8 +835,7 @@ int rte_vhost_enable_guest_notification(struct virtio_net *dev, return -1; } - dev->virtqueue[queue_id]->used->flags = - enable ? 0 : VRING_USED_F_NO_NOTIFY; + dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY; return 0; }