#include <unistd.h>
#include <pthread.h>
#include <stdbool.h>
+#include <sys/epoll.h>
#include <rte_mbuf.h>
#include <rte_ethdev_driver.h>
#include "rte_eth_vhost.h"
-static int vhost_logtype;
+RTE_LOG_REGISTER(vhost_logtype, pmd.net.vhost, NOTICE);
#define VHOST_LOG(level, ...) \
rte_log(RTE_LOG_ ## level, vhost_logtype, __VA_ARGS__)
#define ETH_VHOST_IFACE_ARG "iface"
#define ETH_VHOST_QUEUES_ARG "queues"
#define ETH_VHOST_CLIENT_ARG "client"
-#define ETH_VHOST_DEQUEUE_ZERO_COPY "dequeue-zero-copy"
#define ETH_VHOST_IOMMU_SUPPORT "iommu-support"
#define ETH_VHOST_POSTCOPY_SUPPORT "postcopy-support"
#define ETH_VHOST_VIRTIO_NET_F_HOST_TSO "tso"
+#define ETH_VHOST_LINEAR_BUF "linear-buffer"
+#define ETH_VHOST_EXT_BUF "ext-buffer"
#define VHOST_MAX_PKT_BURST 32
static const char *valid_arguments[] = {
ETH_VHOST_IFACE_ARG,
ETH_VHOST_QUEUES_ARG,
ETH_VHOST_CLIENT_ARG,
- ETH_VHOST_DEQUEUE_ZERO_COPY,
ETH_VHOST_IOMMU_SUPPORT,
ETH_VHOST_POSTCOPY_SUPPORT,
ETH_VHOST_VIRTIO_NET_F_HOST_TSO,
+ ETH_VHOST_LINEAR_BUF,
+ ETH_VHOST_EXT_BUF,
NULL
};
VHOST_BROADCAST_PKT,
VHOST_MULTICAST_PKT,
VHOST_UNICAST_PKT,
+ VHOST_PKT,
+ VHOST_BYTE,
+ VHOST_MISSED_PKT,
VHOST_ERRORS_PKT,
VHOST_ERRORS_FRAGMENTED,
VHOST_ERRORS_JABBER,
uint16_t port;
uint16_t virtqueue_id;
struct vhost_stats stats;
+ int intr_enable;
+ rte_spinlock_t intr_lock;
};
struct pmd_internal {
rte_atomic32_t dev_attached;
- char *dev_name;
char *iface_name;
uint64_t flags;
uint64_t disable_flags;
/* [rx]_is prepended to the name string here */
static const struct vhost_xstats_name_off vhost_rxport_stat_strings[] = {
{"good_packets",
- offsetof(struct vhost_queue, stats.pkts)},
+ offsetof(struct vhost_queue, stats.xstats[VHOST_PKT])},
{"total_bytes",
- offsetof(struct vhost_queue, stats.bytes)},
+ offsetof(struct vhost_queue, stats.xstats[VHOST_BYTE])},
{"missed_pkts",
- offsetof(struct vhost_queue, stats.missed_pkts)},
+ offsetof(struct vhost_queue, stats.xstats[VHOST_MISSED_PKT])},
{"broadcast_packets",
offsetof(struct vhost_queue, stats.xstats[VHOST_BROADCAST_PKT])},
{"multicast_packets",
/* [tx]_ is prepended to the name string here */
static const struct vhost_xstats_name_off vhost_txport_stat_strings[] = {
{"good_packets",
- offsetof(struct vhost_queue, stats.pkts)},
+ offsetof(struct vhost_queue, stats.xstats[VHOST_PKT])},
{"total_bytes",
- offsetof(struct vhost_queue, stats.bytes)},
+ offsetof(struct vhost_queue, stats.xstats[VHOST_BYTE])},
{"missed_pkts",
- offsetof(struct vhost_queue, stats.missed_pkts)},
+ offsetof(struct vhost_queue, stats.xstats[VHOST_MISSED_PKT])},
{"broadcast_packets",
offsetof(struct vhost_queue, stats.xstats[VHOST_BROADCAST_PKT])},
{"multicast_packets",
if (n < nxstats)
return nxstats;
- for (i = 0; i < dev->data->nb_rx_queues; i++) {
- vq = dev->data->rx_queues[i];
- if (!vq)
- continue;
- vq->stats.xstats[VHOST_UNICAST_PKT] = vq->stats.pkts
- - (vq->stats.xstats[VHOST_BROADCAST_PKT]
- + vq->stats.xstats[VHOST_MULTICAST_PKT]);
- }
- for (i = 0; i < dev->data->nb_tx_queues; i++) {
- vq = dev->data->tx_queues[i];
- if (!vq)
- continue;
- vq->stats.xstats[VHOST_UNICAST_PKT] = vq->stats.pkts
- + vq->stats.missed_pkts
- - (vq->stats.xstats[VHOST_BROADCAST_PKT]
- + vq->stats.xstats[VHOST_MULTICAST_PKT]);
- }
for (t = 0; t < VHOST_NB_XSTATS_RXPORT; t++) {
xstats[count].value = 0;
for (i = 0; i < dev->data->nb_rx_queues; i++) {
}
static inline void
-vhost_count_multicast_broadcast(struct vhost_queue *vq,
+vhost_count_xcast_packets(struct vhost_queue *vq,
struct rte_mbuf *mbuf)
{
struct rte_ether_addr *ea = NULL;
pstats->xstats[VHOST_BROADCAST_PKT]++;
else
pstats->xstats[VHOST_MULTICAST_PKT]++;
+ } else {
+ pstats->xstats[VHOST_UNICAST_PKT]++;
}
}
static void
-vhost_update_packet_xstats(struct vhost_queue *vq,
- struct rte_mbuf **bufs,
- uint16_t count)
+vhost_update_packet_xstats(struct vhost_queue *vq, struct rte_mbuf **bufs,
+ uint16_t count, uint64_t nb_bytes,
+ uint64_t nb_missed)
{
uint32_t pkt_len = 0;
uint64_t i = 0;
uint64_t index;
struct vhost_stats *pstats = &vq->stats;
+ pstats->xstats[VHOST_BYTE] += nb_bytes;
+ pstats->xstats[VHOST_MISSED_PKT] += nb_missed;
+ pstats->xstats[VHOST_UNICAST_PKT] += nb_missed;
+
for (i = 0; i < count ; i++) {
+ pstats->xstats[VHOST_PKT]++;
pkt_len = bufs[i]->pkt_len;
if (pkt_len == 64) {
pstats->xstats[VHOST_64_PKT]++;
else if (pkt_len > 1522)
pstats->xstats[VHOST_1523_TO_MAX_PKT]++;
}
- vhost_count_multicast_broadcast(vq, bufs[i]);
+ vhost_count_xcast_packets(vq, bufs[i]);
}
}
struct vhost_queue *r = q;
uint16_t i, nb_rx = 0;
uint16_t nb_receive = nb_bufs;
+ uint64_t nb_bytes = 0;
if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
return 0;
if (r->internal->vlan_strip)
rte_vlan_strip(bufs[i]);
- r->stats.bytes += bufs[i]->pkt_len;
+ nb_bytes += bufs[i]->pkt_len;
}
- vhost_update_packet_xstats(r, bufs, nb_rx);
+ r->stats.bytes += nb_bytes;
+ vhost_update_packet_xstats(r, bufs, nb_rx, nb_bytes, 0);
out:
rte_atomic32_set(&r->while_queuing, 0);
struct vhost_queue *r = q;
uint16_t i, nb_tx = 0;
uint16_t nb_send = 0;
+ uint64_t nb_bytes = 0;
+ uint64_t nb_missed = 0;
if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0))
return 0;
break;
}
+ for (i = 0; likely(i < nb_tx); i++)
+ nb_bytes += bufs[i]->pkt_len;
+
+ nb_missed = nb_bufs - nb_tx;
+
r->stats.pkts += nb_tx;
+ r->stats.bytes += nb_bytes;
r->stats.missed_pkts += nb_bufs - nb_tx;
- for (i = 0; likely(i < nb_tx); i++)
- r->stats.bytes += bufs[i]->pkt_len;
-
- vhost_update_packet_xstats(r, bufs, nb_tx);
+ vhost_update_packet_xstats(r, bufs, nb_tx, nb_bytes, nb_missed);
- /* According to RFC2863 page42 section ifHCOutMulticastPkts and
- * ifHCOutBroadcastPkts, the counters "multicast" and "broadcast"
- * are increased when packets are not transmitted successfully.
+ /* According to RFC2863, ifHCOutUcastPkts, ifHCOutMulticastPkts and
+ * ifHCOutBroadcastPkts counters are increased when packets are not
+ * transmitted successfully.
*/
for (i = nb_tx; i < nb_bufs; i++)
- vhost_count_multicast_broadcast(r, bufs[i]);
+ vhost_count_xcast_packets(r, bufs[i]);
for (i = 0; likely(i < nb_tx); i++)
rte_pktmbuf_free(bufs[i]);
return list;
}
+static int
+eth_vhost_update_intr(struct rte_eth_dev *eth_dev, uint16_t rxq_idx)
+{
+ struct rte_intr_handle *handle = eth_dev->intr_handle;
+ struct rte_epoll_event rev;
+ int epfd, ret;
+
+ if (!handle)
+ return 0;
+
+ if (handle->efds[rxq_idx] == handle->elist[rxq_idx].fd)
+ return 0;
+
+ VHOST_LOG(INFO, "kickfd for rxq-%d was changed, updating handler.\n",
+ rxq_idx);
+
+ if (handle->elist[rxq_idx].fd != -1)
+ VHOST_LOG(ERR, "Unexpected previous kickfd value (Got %d, expected -1).\n",
+ handle->elist[rxq_idx].fd);
+
+ /*
+ * First remove invalid epoll event, and then install
+ * the new one. May be solved with a proper API in the
+ * future.
+ */
+ epfd = handle->elist[rxq_idx].epfd;
+ rev = handle->elist[rxq_idx];
+ ret = rte_epoll_ctl(epfd, EPOLL_CTL_DEL, rev.fd,
+ &handle->elist[rxq_idx]);
+ if (ret) {
+ VHOST_LOG(ERR, "Delete epoll event failed.\n");
+ return ret;
+ }
+
+ rev.fd = handle->efds[rxq_idx];
+ handle->elist[rxq_idx] = rev;
+ ret = rte_epoll_ctl(epfd, EPOLL_CTL_ADD, rev.fd,
+ &handle->elist[rxq_idx]);
+ if (ret) {
+ VHOST_LOG(ERR, "Add epoll event failed.\n");
+ return ret;
+ }
+
+ return 0;
+}
+
static int
eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
{
struct rte_vhost_vring vring;
struct vhost_queue *vq;
- int ret = 0;
+ int old_intr_enable, ret = 0;
vq = dev->data->rx_queues[qid];
if (!vq) {
return -1;
}
+ rte_spinlock_lock(&vq->intr_lock);
+ old_intr_enable = vq->intr_enable;
+ vq->intr_enable = 1;
+ ret = eth_vhost_update_intr(dev, qid);
+ rte_spinlock_unlock(&vq->intr_lock);
+
+ if (ret < 0) {
+ VHOST_LOG(ERR, "Failed to update rxq%d's intr\n", qid);
+ vq->intr_enable = old_intr_enable;
+ return ret;
+ }
+
ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
if (ret < 0) {
VHOST_LOG(ERR, "Failed to get rxq%d's vring\n", qid);
rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0);
rte_wmb();
+ vq->intr_enable = 0;
+
return 0;
}
{
struct rte_vhost_vring vring;
struct vhost_queue *vq;
- int count = 0;
int nb_rxq = dev->data->nb_rx_queues;
int i;
int ret;
VHOST_LOG(INFO, "Prepare intr vec\n");
for (i = 0; i < nb_rxq; i++) {
+ dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
+ dev->intr_handle->efds[i] = -1;
vq = dev->data->rx_queues[i];
if (!vq) {
VHOST_LOG(INFO, "rxq-%d not setup yet, skip!\n", i);
"rxq-%d's kickfd is invalid, skip!\n", i);
continue;
}
- dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
dev->intr_handle->efds[i] = vring.kickfd;
- count++;
VHOST_LOG(INFO, "Installed intr vec for rxq-%d\n", i);
}
- dev->intr_handle->nb_efd = count;
- dev->intr_handle->max_intr = count + 1;
+ dev->intr_handle->nb_efd = nb_rxq;
+ dev->intr_handle->max_intr = nb_rxq + 1;
dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
return 0;
VHOST_LOG(INFO, "Vhost device %d created\n", vid);
- _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
+ rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
return 0;
}
VHOST_LOG(INFO, "Vhost device %d destroyed\n", vid);
eth_vhost_uninstall_intr(eth_dev);
- _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
+ rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
+}
+
+static int
+vring_conf_update(int vid, struct rte_eth_dev *eth_dev, uint16_t vring_id)
+{
+ struct rte_eth_conf *dev_conf = ð_dev->data->dev_conf;
+ struct pmd_internal *internal = eth_dev->data->dev_private;
+ struct vhost_queue *vq;
+ struct rte_vhost_vring vring;
+ int rx_idx = vring_id % 2 ? (vring_id - 1) >> 1 : -1;
+ int ret = 0;
+
+ /*
+ * The vring kickfd may be changed after the new device notification.
+ * Update it when the vring state is updated.
+ */
+ if (rx_idx >= 0 && rx_idx < eth_dev->data->nb_rx_queues &&
+ rte_atomic32_read(&internal->dev_attached) &&
+ rte_atomic32_read(&internal->started) &&
+ dev_conf->intr_conf.rxq) {
+ ret = rte_vhost_get_vhost_vring(vid, vring_id, &vring);
+ if (ret) {
+ VHOST_LOG(ERR, "Failed to get vring %d information.\n",
+ vring_id);
+ return ret;
+ }
+ eth_dev->intr_handle->efds[rx_idx] = vring.kickfd;
+
+ vq = eth_dev->data->rx_queues[rx_idx];
+ if (!vq) {
+ VHOST_LOG(ERR, "rxq%d is not setup yet\n", rx_idx);
+ return -1;
+ }
+
+ rte_spinlock_lock(&vq->intr_lock);
+ if (vq->intr_enable)
+ ret = eth_vhost_update_intr(eth_dev, rx_idx);
+ rte_spinlock_unlock(&vq->intr_lock);
+ }
+
+ return ret;
}
static int
eth_dev = list->eth_dev;
/* won't be NULL */
state = vring_states[eth_dev->data->port_id];
+
+ if (enable && vring_conf_update(vid, eth_dev, vring))
+ VHOST_LOG(INFO, "Failed to update vring-%d configuration.\n",
+ (int)vring);
+
rte_spinlock_lock(&state->lock);
if (state->cur[vring] == enable) {
rte_spinlock_unlock(&state->lock);
VHOST_LOG(INFO, "vring%u is %s\n",
vring, enable ? "enabled" : "disabled");
- _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_QUEUE_STATE, NULL);
+ rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_QUEUE_STATE, NULL);
return 0;
}
unsigned int numa_node = eth_dev->device->numa_node;
const char *name = eth_dev->device->name;
+ /* Don't try to setup again if it has already been done. */
+ list = find_internal_resource(internal->iface_name);
+ if (list)
+ return 0;
+
list = rte_zmalloc_socket(name, sizeof(*list), 0, numa_node);
if (list == NULL)
- goto error;
+ return -1;
vring_state = rte_zmalloc_socket(name, sizeof(*vring_state),
0, numa_node);
if (vring_state == NULL)
- goto error;
+ goto free_list;
list->eth_dev = eth_dev;
pthread_mutex_lock(&internal_list_lock);
vring_states[eth_dev->data->port_id] = vring_state;
if (rte_vhost_driver_register(internal->iface_name, internal->flags))
- goto error;
+ goto list_remove;
if (internal->disable_flags) {
if (rte_vhost_driver_disable_features(internal->iface_name,
internal->disable_flags))
- goto error;
+ goto drv_unreg;
}
if (rte_vhost_driver_callback_register(internal->iface_name,
&vhost_ops) < 0) {
VHOST_LOG(ERR, "Can't register callbacks\n");
- goto error;
+ goto drv_unreg;
}
if (rte_vhost_driver_start(internal->iface_name) < 0) {
VHOST_LOG(ERR, "Failed to start driver for %s\n",
internal->iface_name);
- goto error;
+ goto drv_unreg;
}
return 0;
-error:
+drv_unreg:
+ rte_vhost_driver_unregister(internal->iface_name);
+list_remove:
+ vring_states[eth_dev->data->port_id] = NULL;
+ pthread_mutex_lock(&internal_list_lock);
+ TAILQ_REMOVE(&internal_list, list, next);
+ pthread_mutex_unlock(&internal_list_lock);
rte_free(vring_state);
+free_list:
rte_free(list);
return -1;
return 0;
}
-static void
+static int
eth_dev_stop(struct rte_eth_dev *dev)
{
struct pmd_internal *internal = dev->data->dev_private;
+ dev->data->dev_started = 0;
rte_atomic32_set(&internal->started, 0);
update_queuing_status(dev);
+
+ return 0;
}
-static void
+static int
eth_dev_close(struct rte_eth_dev *dev)
{
struct pmd_internal *internal;
struct internal_list *list;
- unsigned int i;
+ unsigned int i, ret;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return 0;
internal = dev->data->dev_private;
if (!internal)
- return;
-
- eth_dev_stop(dev);
+ return 0;
- rte_vhost_driver_unregister(internal->iface_name);
+ ret = eth_dev_stop(dev);
list = find_internal_resource(internal->iface_name);
- if (!list)
- return;
-
- pthread_mutex_lock(&internal_list_lock);
- TAILQ_REMOVE(&internal_list, list, next);
- pthread_mutex_unlock(&internal_list_lock);
- rte_free(list);
+ if (list) {
+ rte_vhost_driver_unregister(internal->iface_name);
+ pthread_mutex_lock(&internal_list_lock);
+ TAILQ_REMOVE(&internal_list, list, next);
+ pthread_mutex_unlock(&internal_list_lock);
+ rte_free(list);
+ }
if (dev->data->rx_queues)
for (i = 0; i < dev->data->nb_rx_queues; i++)
for (i = 0; i < dev->data->nb_tx_queues; i++)
rte_free(dev->data->tx_queues[i]);
- free(internal->dev_name);
rte_free(internal->iface_name);
rte_free(internal);
rte_free(vring_states[dev->data->port_id]);
vring_states[dev->data->port_id] = NULL;
+
+ return ret;
}
static int
vq->mb_pool = mb_pool;
vq->virtqueue_id = rx_queue_id * VIRTIO_QNUM + VIRTIO_TXQ;
+ rte_spinlock_init(&vq->intr_lock);
dev->data->rx_queues[rx_queue_id] = vq;
return 0;
}
vq->virtqueue_id = tx_queue_id * VIRTIO_QNUM + VIRTIO_RXQ;
+ rte_spinlock_init(&vq->intr_lock);
dev->data->tx_queues[tx_queue_id] = vq;
return 0;
.rx_queue_release = eth_queue_release,
.tx_queue_release = eth_queue_release,
.tx_done_cleanup = eth_tx_done_cleanup,
- .rx_queue_count = eth_rx_queue_count,
.link_update = eth_link_update,
.stats_get = eth_stats_get,
.stats_reset = eth_stats_reset,
* - and point eth_dev structure to new eth_dev_data structure
*/
internal = eth_dev->data->dev_private;
- internal->dev_name = strdup(name);
- if (internal->dev_name == NULL)
- goto error;
internal->iface_name = rte_malloc_socket(name, strlen(iface_name) + 1,
0, numa_node);
if (internal->iface_name == NULL)
internal->flags = flags;
internal->disable_flags = disable_flags;
data->dev_link = pmd_link;
- data->dev_flags = RTE_ETH_DEV_INTR_LSC | RTE_ETH_DEV_CLOSE_REMOVE;
+ data->dev_flags = RTE_ETH_DEV_INTR_LSC |
+ RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
+ data->promiscuous = 1;
+ data->all_multicast = 1;
eth_dev->dev_ops = &ops;
+ eth_dev->rx_queue_count = eth_rx_queue_count;
/* finally assign rx and tx ops */
eth_dev->rx_pkt_burst = eth_vhost_rx;
return 0;
error:
- if (internal) {
+ if (internal)
rte_free(internal->iface_name);
- free(internal->dev_name);
- }
rte_eth_dev_release_port(eth_dev);
return -1;
uint64_t flags = 0;
uint64_t disable_flags = 0;
int client_mode = 0;
- int dequeue_zero_copy = 0;
int iommu_support = 0;
int postcopy_support = 0;
int tso = 0;
+ int linear_buf = 0;
+ int ext_buf = 0;
struct rte_eth_dev *eth_dev;
const char *name = rte_vdev_device_name(dev);
flags |= RTE_VHOST_USER_CLIENT;
}
- if (rte_kvargs_count(kvlist, ETH_VHOST_DEQUEUE_ZERO_COPY) == 1) {
- ret = rte_kvargs_process(kvlist, ETH_VHOST_DEQUEUE_ZERO_COPY,
- &open_int, &dequeue_zero_copy);
- if (ret < 0)
- goto out_free;
-
- if (dequeue_zero_copy)
- flags |= RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
- }
-
if (rte_kvargs_count(kvlist, ETH_VHOST_IOMMU_SUPPORT) == 1) {
ret = rte_kvargs_process(kvlist, ETH_VHOST_IOMMU_SUPPORT,
&open_int, &iommu_support);
}
}
+ if (rte_kvargs_count(kvlist, ETH_VHOST_LINEAR_BUF) == 1) {
+ ret = rte_kvargs_process(kvlist,
+ ETH_VHOST_LINEAR_BUF,
+ &open_int, &linear_buf);
+ if (ret < 0)
+ goto out_free;
+
+ if (linear_buf == 1)
+ flags |= RTE_VHOST_USER_LINEARBUF_SUPPORT;
+ }
+
+ if (rte_kvargs_count(kvlist, ETH_VHOST_EXT_BUF) == 1) {
+ ret = rte_kvargs_process(kvlist,
+ ETH_VHOST_EXT_BUF,
+ &open_int, &ext_buf);
+ if (ret < 0)
+ goto out_free;
+
+ if (ext_buf == 1)
+ flags |= RTE_VHOST_USER_EXTBUF_SUPPORT;
+ }
+
if (dev->device.numa_node == SOCKET_ID_ANY)
dev->device.numa_node = rte_socket_id();
if (eth_dev == NULL)
return 0;
- if (rte_eal_process_type() != RTE_PROC_PRIMARY)
- return rte_eth_dev_release_port(eth_dev);
-
eth_dev_close(eth_dev);
-
rte_eth_dev_release_port(eth_dev);
return 0;
"iface=<ifc> "
"queues=<int> "
"client=<0|1> "
- "dequeue-zero-copy=<0|1> "
"iommu-support=<0|1> "
"postcopy-support=<0|1> "
- "tso=<0|1>");
-
-RTE_INIT(vhost_init_log)
-{
- vhost_logtype = rte_log_register("pmd.net.vhost");
- if (vhost_logtype >= 0)
- rte_log_set_level(vhost_logtype, RTE_LOG_NOTICE);
-}
+ "tso=<0|1> "
+ "linear-buffer=<0|1> "
+ "ext-buffer=<0|1>");