#include <rte_log.h>
#include <rte_string_fns.h>
#include <rte_malloc.h>
+#include <rte_virtio_net.h>
#include "main.h"
-#include "virtio-net.h"
-#include "vhost-net-cdev.h"
#define MAX_QUEUES 128
/* Character device basename. Can be set by user. */
static char dev_basename[MAX_BASENAME_SZ] = "vhost-net";
-/* Charater device index. Can be set by user. */
-static uint32_t dev_index = 0;
/* This can be set by the user so it is made available here. */
extern uint64_t VHOST_FEATURES;
RTE_LOG(INFO, VHOST_CONFIG, "%s [EAL options] -- -p PORTMASK\n"
" --vm2vm [0|1|2]\n"
" --rx_retry [0|1] --mergeable [0|1] --stats [0-N]\n"
- " --dev-basename <name> --dev-index [0-N]\n"
+ " --dev-basename <name>\n"
" --nb-devices ND\n"
" -p PORTMASK: Set mask for ports to be used by application\n"
" --vm2vm [0|1|2]: disable/software(default)/hardware vm2vm comms\n"
" --mergeable [0|1]: disable(default)/enable RX mergeable buffers\n"
" --stats [0-N]: 0: Disable stats, N: Time in seconds to print stats\n"
" --dev-basename: The basename to be used for the character device.\n"
- " --dev-index [0-N]: Defaults to zero if not used. Index is appended to basename.\n"
" --zero-copy [0|1]: disable(default)/enable rx/tx "
"zero copy\n"
" --rx-desc-num [0-N]: the number of descriptors on rx, "
{"mergeable", required_argument, NULL, 0},
{"stats", required_argument, NULL, 0},
{"dev-basename", required_argument, NULL, 0},
- {"dev-index", required_argument, NULL, 0},
{"zero-copy", required_argument, NULL, 0},
{"rx-desc-num", required_argument, NULL, 0},
{"tx-desc-num", required_argument, NULL, 0},
}
}
- /* Set character device index. */
- if (!strncmp(long_option[option_index].name, "dev-index", MAX_LONG_OPT_SZ)) {
- ret = parse_num_opt(optarg, INT32_MAX);
- if (ret == -1) {
- RTE_LOG(INFO, VHOST_CONFIG, "Invalid argument for character device index [0..N]\n");
- us_vhost_usage(prgname);
- return -1;
- } else
- dev_index = ret;
- }
-
/* Enable/disable rx/tx zero copy. */
if (!strncmp(long_option[option_index].name,
"zero-copy", MAX_LONG_OPT_SZ)) {
* This is used to convert virtio buffer addresses.
*/
static inline uint64_t __attribute__((always_inline))
-gpa_to_hpa(struct virtio_net *dev, uint64_t guest_pa,
+gpa_to_hpa(struct vhost_dev *vdev, uint64_t guest_pa,
uint32_t buf_len, hpa_type *addr_type)
{
struct virtio_memory_regions_hpa *region;
*addr_type = PHYS_ADDR_INVALID;
- for (regionidx = 0; regionidx < dev->mem->nregions_hpa; regionidx++) {
- region = &dev->mem->regions_hpa[regionidx];
+ for (regionidx = 0; regionidx < vdev->nregions_hpa; regionidx++) {
+ region = &vdev->regions_hpa[regionidx];
if ((guest_pa >= region->guest_phys_address) &&
(guest_pa <= region->guest_phys_address_end)) {
vhost_pa = region->host_phys_addr_offset + guest_pa;
}
LOG_DEBUG(VHOST_DATA, "(%"PRIu64") GPA %p| HPA %p\n",
- dev->device_fh, (void *)(uintptr_t)guest_pa,
+ vdev->dev->device_fh, (void *)(uintptr_t)guest_pa,
(void *)(uintptr_t)vhost_pa);
return vhost_pa;
* vlan tag to a VMDQ.
*/
static int
-link_vmdq(struct virtio_net *dev, struct rte_mbuf *m)
+link_vmdq(struct vhost_dev *vdev, struct rte_mbuf *m)
{
struct ether_hdr *pkt_hdr;
struct virtio_net_data_ll *dev_ll;
+ struct virtio_net *dev = vdev->dev;
int i, ret;
/* Learn MAC address of guest device from packet */
dev_ll = ll_root_used;
while (dev_ll != NULL) {
- if (ether_addr_cmp(&(pkt_hdr->s_addr), &dev_ll->dev->mac_address)) {
+ if (ether_addr_cmp(&(pkt_hdr->s_addr), &dev_ll->vdev->mac_address)) {
RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") WARNING: This device is using an existing MAC address and has not been registered.\n", dev->device_fh);
return -1;
}
}
for (i = 0; i < ETHER_ADDR_LEN; i++)
- dev->mac_address.addr_bytes[i] = pkt_hdr->s_addr.addr_bytes[i];
+ vdev->mac_address.addr_bytes[i] = pkt_hdr->s_addr.addr_bytes[i];
/* vlan_tag currently uses the device_id. */
- dev->vlan_tag = vlan_tags[dev->device_fh];
+ vdev->vlan_tag = vlan_tags[dev->device_fh];
/* Print out VMDQ registration info. */
RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") MAC_ADDRESS %02x:%02x:%02x:%02x:%02x:%02x and VLAN_TAG %d registered\n",
dev->device_fh,
- dev->mac_address.addr_bytes[0], dev->mac_address.addr_bytes[1],
- dev->mac_address.addr_bytes[2], dev->mac_address.addr_bytes[3],
- dev->mac_address.addr_bytes[4], dev->mac_address.addr_bytes[5],
- dev->vlan_tag);
+ vdev->mac_address.addr_bytes[0], vdev->mac_address.addr_bytes[1],
+ vdev->mac_address.addr_bytes[2], vdev->mac_address.addr_bytes[3],
+ vdev->mac_address.addr_bytes[4], vdev->mac_address.addr_bytes[5],
+ vdev->vlan_tag);
/* Register the MAC address. */
- ret = rte_eth_dev_mac_addr_add(ports[0], &dev->mac_address, (uint32_t)dev->device_fh);
+ ret = rte_eth_dev_mac_addr_add(ports[0], &vdev->mac_address, (uint32_t)dev->device_fh);
if (ret)
RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") Failed to add device MAC address to VMDQ\n",
dev->device_fh);
/* Enable stripping of the vlan tag as we handle routing. */
- rte_eth_dev_set_vlan_strip_on_queue(ports[0], (uint16_t)dev->vmdq_rx_q, 1);
+ rte_eth_dev_set_vlan_strip_on_queue(ports[0], (uint16_t)vdev->vmdq_rx_q, 1);
/* Set device as ready for RX. */
- dev->ready = DEVICE_RX;
+ vdev->ready = DEVICE_RX;
return 0;
}
* queue before disabling RX on the device.
*/
static inline void
-unlink_vmdq(struct virtio_net *dev)
+unlink_vmdq(struct vhost_dev *vdev)
{
unsigned i = 0;
unsigned rx_count;
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
- if (dev->ready == DEVICE_RX) {
+ if (vdev->ready == DEVICE_RX) {
/*clear MAC and VLAN settings*/
- rte_eth_dev_mac_addr_remove(ports[0], &dev->mac_address);
+ rte_eth_dev_mac_addr_remove(ports[0], &vdev->mac_address);
for (i = 0; i < 6; i++)
- dev->mac_address.addr_bytes[i] = 0;
+ vdev->mac_address.addr_bytes[i] = 0;
- dev->vlan_tag = 0;
+ vdev->vlan_tag = 0;
/*Clear out the receive buffers*/
rx_count = rte_eth_rx_burst(ports[0],
- (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
+ (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
while (rx_count) {
for (i = 0; i < rx_count; i++)
rte_pktmbuf_free(pkts_burst[i]);
rx_count = rte_eth_rx_burst(ports[0],
- (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
+ (uint16_t)vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
}
- dev->ready = DEVICE_MAC_LEARNING;
+ vdev->ready = DEVICE_MAC_LEARNING;
}
}
* the packet on that devices RX queue. If not then return.
*/
static inline unsigned __attribute__((always_inline))
-virtio_tx_local(struct virtio_net *dev, struct rte_mbuf *m)
+virtio_tx_local(struct vhost_dev *vdev, struct rte_mbuf *m)
{
struct virtio_net_data_ll *dev_ll;
struct ether_hdr *pkt_hdr;
uint64_t ret = 0;
+ struct virtio_net *dev = vdev->dev;
+ struct virtio_net *tdev; /* destination virito device */
pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
dev_ll = ll_root_used;
while (dev_ll != NULL) {
- if ((dev_ll->dev->ready == DEVICE_RX) && ether_addr_cmp(&(pkt_hdr->d_addr),
- &dev_ll->dev->mac_address)) {
+ if ((dev_ll->vdev->ready == DEVICE_RX) && ether_addr_cmp(&(pkt_hdr->d_addr),
+ &dev_ll->vdev->mac_address)) {
/* Drop the packet if the TX packet is destined for the TX device. */
- if (dev_ll->dev->device_fh == dev->device_fh) {
+ if (dev_ll->vdev->dev->device_fh == dev->device_fh) {
LOG_DEBUG(VHOST_DATA, "(%"PRIu64") TX: Source and destination MAC addresses are the same. Dropping packet.\n",
- dev_ll->dev->device_fh);
+ dev->device_fh);
return 0;
}
+ tdev = dev_ll->vdev->dev;
- LOG_DEBUG(VHOST_DATA, "(%"PRIu64") TX: MAC address is local\n", dev_ll->dev->device_fh);
+ LOG_DEBUG(VHOST_DATA, "(%"PRIu64") TX: MAC address is local\n", tdev->device_fh);
- if (dev_ll->dev->remove) {
+ if (dev_ll->vdev->remove) {
/*drop the packet if the device is marked for removal*/
- LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Device is marked for removal\n", dev_ll->dev->device_fh);
+ LOG_DEBUG(VHOST_DATA, "(%"PRIu64") Device is marked for removal\n", tdev->device_fh);
} else {
uint32_t mergeable =
dev_ll->dev->features &
if (enable_stats) {
rte_atomic64_add(
- &dev_statistics[dev_ll->dev->device_fh].rx_total_atomic,
+ &dev_statistics[tdev->device_fh].rx_total_atomic,
1);
rte_atomic64_add(
- &dev_statistics[dev_ll->dev->device_fh].rx_atomic,
+ &dev_statistics[tdev->device_fh].rx_atomic,
ret);
- dev_statistics[dev->device_fh].tx_total++;
- dev_statistics[dev->device_fh].tx += ret;
+ dev_statistics[tdev->device_fh].tx_total++;
+ dev_statistics[tdev->device_fh].tx += ret;
}
}
* or the physical port.
*/
static inline void __attribute__((always_inline))
-virtio_tx_route(struct virtio_net* dev, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, uint16_t vlan_tag)
+virtio_tx_route(struct vhost_dev *vdev, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, uint16_t vlan_tag)
{
struct mbuf_table *tx_q;
struct vlan_ethhdr *vlan_hdr;
const uint16_t lcore_id = rte_lcore_id();
struct virtio_net_data_ll *dev_ll = ll_root_used;
struct ether_hdr *pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
+ struct virtio_net *dev = vdev->dev;
/*check if destination is local VM*/
- if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(dev, m) == 0))
+ if ((vm2vm_mode == VM2VM_SOFTWARE) && (virtio_tx_local(vdev, m) == 0))
return;
if (vm2vm_mode == VM2VM_HARDWARE) {
while (dev_ll != NULL) {
- if ((dev_ll->dev->ready == DEVICE_RX)
+ if ((dev_ll->vdev->ready == DEVICE_RX)
&& ether_addr_cmp(&(pkt_hdr->d_addr),
- &dev_ll->dev->mac_address)) {
+ &dev_ll->vdev->mac_address)) {
/*
* Drop the packet if the TX packet is
* destined for the TX device.
*/
- if (dev_ll->dev->device_fh == dev->device_fh) {
+ if (dev_ll->vdev->dev->device_fh == dev->device_fh) {
LOG_DEBUG(VHOST_DATA,
"(%"PRIu64") TX: Source and destination"
" MAC addresses are the same. Dropping "
"packet.\n",
- dev_ll->dev->device_fh);
+ dev_ll->vdev->device_fh);
return;
}
offset = 4;
vlan_tag =
(uint16_t)
- vlan_tags[(uint16_t)dev_ll->dev->device_fh];
+ vlan_tags[(uint16_t)dev_ll->vdev->dev->device_fh];
LOG_DEBUG(VHOST_DATA,
"(%"PRIu64") TX: pkt to local VM device id:"
"(%"PRIu64") vlan tag: %d.\n",
- dev->device_fh, dev_ll->dev->device_fh,
+ dev->device_fh, dev_ll->vdev->dev->device_fh,
vlan_tag);
break;
{
struct rte_mempool *mbuf_pool = arg;
struct virtio_net *dev = NULL;
+ struct vhost_dev *vdev = NULL;
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
struct virtio_net_data_ll *dev_ll;
struct mbuf_table *tx_q;
while (dev_ll != NULL) {
/*get virtio device ID*/
- dev = dev_ll->dev;
+ vdev = dev_ll->vdev;
+ dev = vdev->dev;
mergeable =
dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF);
- if (dev->remove) {
+ if (vdev->remove) {
dev_ll = dev_ll->next;
- unlink_vmdq(dev);
- dev->ready = DEVICE_SAFE_REMOVE;
+ unlink_vmdq(vdev);
+ vdev->ready = DEVICE_SAFE_REMOVE;
continue;
}
- if (likely(dev->ready == DEVICE_RX)) {
+ if (likely(vdev->ready == DEVICE_RX)) {
/*Handle guest RX*/
rx_count = rte_eth_rx_burst(ports[0],
- (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
+ vdev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
if (rx_count) {
if (likely(mergeable == 0))
if (enable_stats) {
rte_atomic64_add(
- &dev_statistics[dev_ll->dev->device_fh].rx_total_atomic,
+ &dev_statistics[dev_ll->vdev->dev->device_fh].rx_total_atomic,
rx_count);
rte_atomic64_add(
- &dev_statistics[dev_ll->dev->device_fh].rx_atomic, ret_count);
+ &dev_statistics[dev_ll->vdev->dev->device_fh].rx_atomic, ret_count);
}
while (likely(rx_count)) {
rx_count--;
}
}
- if (!dev->remove) {
+ if (!vdev->remove) {
/*Handle guest TX*/
if (likely(mergeable == 0))
virtio_dev_tx(dev, mbuf_pool);
struct rte_mbuf *mbuf = NULL;
struct vpool *vpool;
hpa_type addr_type;
+ struct vhost_dev *vdev = (struct vhost_dev *)dev->priv;
- vpool = &vpool_array[dev->vmdq_rx_q];
+ vpool = &vpool_array[vdev->vmdq_rx_q];
vq = dev->virtqueue[VIRTIO_RXQ];
do {
- if (unlikely(get_available_ring_index_zcp(dev, &res_base_idx,
+ if (unlikely(get_available_ring_index_zcp(vdev->dev, &res_base_idx,
1) != 1))
return;
desc_idx = vq->avail->ring[(res_base_idx) & (vq->size - 1)];
if (desc->flags & VRING_DESC_F_NEXT) {
desc = &vq->desc[desc->next];
buff_addr = gpa_to_vva(dev, desc->addr);
- phys_addr = gpa_to_hpa(dev, desc->addr, desc->len,
+ phys_addr = gpa_to_hpa(vdev, desc->addr, desc->len,
&addr_type);
} else {
buff_addr = gpa_to_vva(dev,
desc->addr + vq->vhost_hlen);
- phys_addr = gpa_to_hpa(dev,
+ phys_addr = gpa_to_hpa(vdev,
desc->addr + vq->vhost_hlen,
desc->len, &addr_type);
}
struct virtio_net_data_ll *dev_ll = ll_root_used;
struct ether_hdr *pkt_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
uint16_t vlan_tag = (uint16_t)vlan_tags[(uint16_t)dev->device_fh];
+ uint16_t vmdq_rx_q = ((struct vhost_dev *)dev->priv)->vmdq_rx_q;
/*Add packet to the port tx queue*/
- tx_q = &tx_queue_zcp[(uint16_t)dev->vmdq_rx_q];
+ tx_q = &tx_queue_zcp[vmdq_rx_q];
len = tx_q->len;
/* Allocate an mbuf and populate the structure. */
- vpool = &vpool_array[MAX_QUEUES + (uint16_t)dev->vmdq_rx_q];
+ vpool = &vpool_array[MAX_QUEUES + vmdq_rx_q];
rte_ring_sc_dequeue(vpool->ring, (void **)&mbuf);
if (unlikely(mbuf == NULL)) {
struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_TXQ];
*/
vlan_tag = external_pkt_default_vlan_tag;
while (dev_ll != NULL) {
- if (likely(dev_ll->dev->ready == DEVICE_RX) &&
+ if (likely(dev_ll->vdev->ready == DEVICE_RX) &&
ether_addr_cmp(&(pkt_hdr->d_addr),
- &dev_ll->dev->mac_address)) {
+ &dev_ll->vdev->mac_address)) {
/*
* Drop the packet if the TX packet is destined
* for the TX device.
*/
- if (unlikely(dev_ll->dev->device_fh
+ if (unlikely(dev_ll->vdev->dev->device_fh
== dev->device_fh)) {
LOG_DEBUG(VHOST_DATA,
"(%"PRIu64") TX: Source and destination"
"MAC addresses are the same. Dropping "
"packet.\n",
- dev_ll->dev->device_fh);
+ dev_ll->vdev->dev->device_fh);
MBUF_HEADROOM_UINT32(mbuf)
= (uint32_t)desc_idx;
__rte_mbuf_raw_free(mbuf);
offset = 4;
vlan_tag =
(uint16_t)
- vlan_tags[(uint16_t)dev_ll->dev->device_fh];
+ vlan_tags[(uint16_t)dev_ll->vdev->dev->device_fh];
LOG_DEBUG(VHOST_DATA,
"(%"PRIu64") TX: pkt to local VM device id:"
"(%"PRIu64") vlan tag: %d.\n",
- dev->device_fh, dev_ll->dev->device_fh,
+ dev->device_fh, dev_ll->vdev->dev->device_fh,
vlan_tag);
break;
uint16_t avail_idx;
uint8_t need_copy = 0;
hpa_type addr_type;
+ struct vhost_dev *vdev = (struct vhost_dev *)dev->priv;
vq = dev->virtqueue[VIRTIO_TXQ];
avail_idx = *((volatile uint16_t *)&vq->avail->idx);
/* Buffer address translation. */
buff_addr = gpa_to_vva(dev, desc->addr);
- phys_addr = gpa_to_hpa(dev, desc->addr, desc->len, &addr_type);
+ phys_addr = gpa_to_hpa(vdev, desc->addr, desc->len, &addr_type);
if (likely(packet_success < (free_entries - 1)))
/* Prefetch descriptor index. */
* If this is the first received packet we need to learn
* the MAC and setup VMDQ
*/
- if (unlikely(dev->ready == DEVICE_MAC_LEARNING)) {
- if (dev->remove || (link_vmdq(dev, &m) == -1)) {
+ if (unlikely(vdev->ready == DEVICE_MAC_LEARNING)) {
+ if (vdev->remove || (link_vmdq(vdev, &m) == -1)) {
/*
* Discard frame if device is scheduled for
* removal or a duplicate MAC address is found.
switch_worker_zcp(__attribute__((unused)) void *arg)
{
struct virtio_net *dev = NULL;
+ struct vhost_dev *vdev = NULL;
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
struct virtio_net_data_ll *dev_ll;
struct mbuf_table *tx_q;
* put back into vpool.ring.
*/
dev_ll = lcore_ll->ll_root_used;
- while ((dev_ll != NULL) && (dev_ll->dev != NULL)) {
+ while ((dev_ll != NULL) && (dev_ll->vdev != NULL)) {
/* Get virtio device ID */
- dev = dev_ll->dev;
+ vdev = dev_ll->vdev;
+ dev = vdev->dev;
- if (likely(!dev->remove)) {
- tx_q = &tx_queue_zcp[(uint16_t)dev->vmdq_rx_q];
+ if (likely(!vdev->remove)) {
+ tx_q = &tx_queue_zcp[(uint16_t)vdev->vmdq_rx_q];
if (tx_q->len) {
LOG_DEBUG(VHOST_DATA,
"TX queue drained after timeout"
tx_q->len = 0;
txmbuf_clean_zcp(dev,
- &vpool_array[MAX_QUEUES+dev->vmdq_rx_q]);
+ &vpool_array[MAX_QUEUES+vdev->vmdq_rx_q]);
}
}
dev_ll = dev_ll->next;
/* Process devices */
dev_ll = lcore_ll->ll_root_used;
- while ((dev_ll != NULL) && (dev_ll->dev != NULL)) {
- dev = dev_ll->dev;
- if (unlikely(dev->remove)) {
+ while ((dev_ll != NULL) && (dev_ll->vdev != NULL)) {
+ vdev = dev_ll->vdev;
+ dev = vdev->dev;
+ if (unlikely(vdev->remove)) {
dev_ll = dev_ll->next;
- unlink_vmdq(dev);
- dev->ready = DEVICE_SAFE_REMOVE;
+ unlink_vmdq(vdev);
+ vdev->ready = DEVICE_SAFE_REMOVE;
continue;
}
- if (likely(dev->ready == DEVICE_RX)) {
- uint32_t index = dev->vmdq_rx_q;
+ if (likely(vdev->ready == DEVICE_RX)) {
+ uint32_t index = vdev->vmdq_rx_q;
uint16_t i;
count_in_ring
= rte_ring_count(vpool_array[index].ring);
/* Handle guest RX */
rx_count = rte_eth_rx_burst(ports[0],
- (uint16_t)dev->vmdq_rx_q, pkts_burst,
+ vdev->vmdq_rx_q, pkts_burst,
MAX_PKT_BURST);
if (rx_count) {
}
}
- if (likely(!dev->remove))
+ if (likely(!vdev->remove))
/* Handle guest TX */
virtio_dev_tx_zcp(dev);
}
for (i = 0; i < size - 1; i++) {
- ll_new[i].dev = NULL;
+ ll_new[i].vdev = NULL;
ll_new[i].next = &ll_new[i+1];
}
ll_new[i].next = NULL;
struct virtio_net_data_ll *ll_main_dev_cur;
struct virtio_net_data_ll *ll_lcore_dev_last = NULL;
struct virtio_net_data_ll *ll_main_dev_last = NULL;
+ struct vhost_dev *vdev;
int lcore;
dev->flags &= ~VIRTIO_DEV_RUNNING;
+ vdev = (struct vhost_dev *)dev->priv;
/*set the remove flag. */
- dev->remove = 1;
-
- while(dev->ready != DEVICE_SAFE_REMOVE) {
+ vdev->remove = 1;
+ while(vdev->ready != DEVICE_SAFE_REMOVE) {
rte_pause();
}
/* Search for entry to be removed from lcore ll */
- ll_lcore_dev_cur = lcore_info[dev->coreid].lcore_ll->ll_root_used;
+ ll_lcore_dev_cur = lcore_info[vdev->coreid].lcore_ll->ll_root_used;
while (ll_lcore_dev_cur != NULL) {
- if (ll_lcore_dev_cur->dev == dev) {
+ if (ll_lcore_dev_cur->vdev == vdev) {
break;
} else {
ll_lcore_dev_last = ll_lcore_dev_cur;
ll_main_dev_cur = ll_root_used;
ll_main_dev_last = NULL;
while (ll_main_dev_cur != NULL) {
- if (ll_main_dev_cur->dev == dev) {
+ if (ll_main_dev_cur->vdev == vdev) {
break;
} else {
ll_main_dev_last = ll_main_dev_cur;
}
/* Remove entries from the lcore and main ll. */
- rm_data_ll_entry(&lcore_info[ll_lcore_dev_cur->dev->coreid].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);
+ rm_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);
rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);
/* Set the dev_removal_flag on each lcore. */
}
/* Add the entries back to the lcore and main free ll.*/
- put_data_ll_free_entry(&lcore_info[ll_lcore_dev_cur->dev->coreid].lcore_ll->ll_root_free, ll_lcore_dev_cur);
+ put_data_ll_free_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_free, ll_lcore_dev_cur);
put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);
/* Decrement number of device on the lcore. */
- lcore_info[ll_lcore_dev_cur->dev->coreid].lcore_ll->device_num--;
+ lcore_info[vdev->coreid].lcore_ll->device_num--;
RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been removed from data core\n", dev->device_fh);
if (zero_copy) {
- struct vpool *vpool = &vpool_array[dev->vmdq_rx_q];
+ struct vpool *vpool = &vpool_array[vdev->vmdq_rx_q];
/* Stop the RX queue. */
- if (rte_eth_dev_rx_queue_stop(ports[0], dev->vmdq_rx_q) != 0) {
+ if (rte_eth_dev_rx_queue_stop(ports[0], vdev->vmdq_rx_q) != 0) {
LOG_DEBUG(VHOST_CONFIG,
"(%"PRIu64") In destroy_device: Failed to stop "
"rx queue:%d\n",
dev->device_fh,
- dev->vmdq_rx_q);
+ vdev->vmdq_rx_q);
}
LOG_DEBUG(VHOST_CONFIG,
"(%"PRIu64") in destroy_device: Start put mbuf in "
"mempool back to ring for RX queue: %d\n",
- dev->device_fh, dev->vmdq_rx_q);
+ dev->device_fh, vdev->vmdq_rx_q);
mbuf_destroy_zcp(vpool);
/* Stop the TX queue. */
- if (rte_eth_dev_tx_queue_stop(ports[0], dev->vmdq_rx_q) != 0) {
+ if (rte_eth_dev_tx_queue_stop(ports[0], vdev->vmdq_rx_q) != 0) {
LOG_DEBUG(VHOST_CONFIG,
"(%"PRIu64") In destroy_device: Failed to "
"stop tx queue:%d\n",
- dev->device_fh, dev->vmdq_rx_q);
+ dev->device_fh, vdev->vmdq_rx_q);
}
- vpool = &vpool_array[dev->vmdq_rx_q + MAX_QUEUES];
+ vpool = &vpool_array[vdev->vmdq_rx_q + MAX_QUEUES];
LOG_DEBUG(VHOST_CONFIG,
"(%"PRIu64") destroy_device: Start put mbuf in mempool "
"back to ring for TX queue: %d, dev:(%"PRIu64")\n",
- dev->device_fh, (dev->vmdq_rx_q + MAX_QUEUES),
+ dev->device_fh, (vdev->vmdq_rx_q + MAX_QUEUES),
dev->device_fh);
mbuf_destroy_zcp(vpool);
+ rte_free(vdev->regions_hpa);
}
+ rte_free(vdev);
}
+/*
+ * Calculate the region count of physical continous regions for one particular
+ * region of whose vhost virtual address is continous. The particular region
+ * start from vva_start, with size of 'size' in argument.
+ */
+static uint32_t
+check_hpa_regions(uint64_t vva_start, uint64_t size)
+{
+ uint32_t i, nregions = 0, page_size = getpagesize();
+ uint64_t cur_phys_addr = 0, next_phys_addr = 0;
+ if (vva_start % page_size) {
+ LOG_DEBUG(VHOST_CONFIG,
+ "in check_countinous: vva start(%p) mod page_size(%d) "
+ "has remainder\n",
+ (void *)(uintptr_t)vva_start, page_size);
+ return 0;
+ }
+ if (size % page_size) {
+ LOG_DEBUG(VHOST_CONFIG,
+ "in check_countinous: "
+ "size((%"PRIu64")) mod page_size(%d) has remainder\n",
+ size, page_size);
+ return 0;
+ }
+ for (i = 0; i < size - page_size; i = i + page_size) {
+ cur_phys_addr
+ = rte_mem_virt2phy((void *)(uintptr_t)(vva_start + i));
+ next_phys_addr = rte_mem_virt2phy(
+ (void *)(uintptr_t)(vva_start + i + page_size));
+ if ((cur_phys_addr + page_size) != next_phys_addr) {
+ ++nregions;
+ LOG_DEBUG(VHOST_CONFIG,
+ "in check_continuous: hva addr:(%p) is not "
+ "continuous with hva addr:(%p), diff:%d\n",
+ (void *)(uintptr_t)(vva_start + (uint64_t)i),
+ (void *)(uintptr_t)(vva_start + (uint64_t)i
+ + page_size), page_size);
+ LOG_DEBUG(VHOST_CONFIG,
+ "in check_continuous: hpa addr:(%p) is not "
+ "continuous with hpa addr:(%p), "
+ "diff:(%"PRIu64")\n",
+ (void *)(uintptr_t)cur_phys_addr,
+ (void *)(uintptr_t)next_phys_addr,
+ (next_phys_addr-cur_phys_addr));
+ }
+ }
+ return nregions;
+}
+
+/*
+ * Divide each region whose vhost virtual address is continous into a few
+ * sub-regions, make sure the physical address within each sub-region are
+ * continous. And fill offset(to GPA) and size etc. information of each
+ * sub-region into regions_hpa.
+ */
+static uint32_t
+fill_hpa_memory_regions(struct virtio_memory_regions_hpa *mem_region_hpa, struct virtio_memory *virtio_memory)
+{
+ uint32_t regionidx, regionidx_hpa = 0, i, k, page_size = getpagesize();
+ uint64_t cur_phys_addr = 0, next_phys_addr = 0, vva_start;
+
+ if (mem_region_hpa == NULL)
+ return 0;
+
+ for (regionidx = 0; regionidx < virtio_memory->nregions; regionidx++) {
+ vva_start = virtio_memory->regions[regionidx].guest_phys_address +
+ virtio_memory->regions[regionidx].address_offset;
+ mem_region_hpa[regionidx_hpa].guest_phys_address
+ = virtio_memory->regions[regionidx].guest_phys_address;
+ mem_region_hpa[regionidx_hpa].host_phys_addr_offset =
+ rte_mem_virt2phy((void *)(uintptr_t)(vva_start)) -
+ mem_region_hpa[regionidx_hpa].guest_phys_address;
+ LOG_DEBUG(VHOST_CONFIG,
+ "in fill_hpa_regions: guest phys addr start[%d]:(%p)\n",
+ regionidx_hpa,
+ (void *)(uintptr_t)
+ (mem_region_hpa[regionidx_hpa].guest_phys_address));
+ LOG_DEBUG(VHOST_CONFIG,
+ "in fill_hpa_regions: host phys addr start[%d]:(%p)\n",
+ regionidx_hpa,
+ (void *)(uintptr_t)
+ (mem_region_hpa[regionidx_hpa].host_phys_addr_offset));
+ for (i = 0, k = 0;
+ i < virtio_memory->regions[regionidx].memory_size -
+ page_size;
+ i += page_size) {
+ cur_phys_addr = rte_mem_virt2phy(
+ (void *)(uintptr_t)(vva_start + i));
+ next_phys_addr = rte_mem_virt2phy(
+ (void *)(uintptr_t)(vva_start +
+ i + page_size));
+ if ((cur_phys_addr + page_size) != next_phys_addr) {
+ mem_region_hpa[regionidx_hpa].guest_phys_address_end =
+ mem_region_hpa[regionidx_hpa].guest_phys_address +
+ k + page_size;
+ mem_region_hpa[regionidx_hpa].memory_size
+ = k + page_size;
+ LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest "
+ "phys addr end [%d]:(%p)\n",
+ regionidx_hpa,
+ (void *)(uintptr_t)
+ (mem_region_hpa[regionidx_hpa].guest_phys_address_end));
+ LOG_DEBUG(VHOST_CONFIG,
+ "in fill_hpa_regions: guest phys addr "
+ "size [%d]:(%p)\n",
+ regionidx_hpa,
+ (void *)(uintptr_t)
+ (mem_region_hpa[regionidx_hpa].memory_size));
+ mem_region_hpa[regionidx_hpa + 1].guest_phys_address
+ = mem_region_hpa[regionidx_hpa].guest_phys_address_end;
+ ++regionidx_hpa;
+ mem_region_hpa[regionidx_hpa].host_phys_addr_offset =
+ next_phys_addr -
+ mem_region_hpa[regionidx_hpa].guest_phys_address;
+ LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest"
+ " phys addr start[%d]:(%p)\n",
+ regionidx_hpa,
+ (void *)(uintptr_t)
+ (mem_region_hpa[regionidx_hpa].guest_phys_address));
+ LOG_DEBUG(VHOST_CONFIG,
+ "in fill_hpa_regions: host phys addr "
+ "start[%d]:(%p)\n",
+ regionidx_hpa,
+ (void *)(uintptr_t)
+ (mem_region_hpa[regionidx_hpa].host_phys_addr_offset));
+ k = 0;
+ } else {
+ k += page_size;
+ }
+ }
+ mem_region_hpa[regionidx_hpa].guest_phys_address_end
+ = mem_region_hpa[regionidx_hpa].guest_phys_address
+ + k + page_size;
+ mem_region_hpa[regionidx_hpa].memory_size = k + page_size;
+ LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest phys addr end "
+ "[%d]:(%p)\n", regionidx_hpa,
+ (void *)(uintptr_t)
+ (mem_region_hpa[regionidx_hpa].guest_phys_address_end));
+ LOG_DEBUG(VHOST_CONFIG, "in fill_hpa_regions: guest phys addr size "
+ "[%d]:(%p)\n", regionidx_hpa,
+ (void *)(uintptr_t)
+ (mem_region_hpa[regionidx_hpa].memory_size));
+ ++regionidx_hpa;
+ }
+ return regionidx_hpa;
+}
+
/*
* A new device is added to a data core. First the device is added to the main linked list
* and the allocated to a specific data core.
struct virtio_net_data_ll *ll_dev;
int lcore, core_add = 0;
uint32_t device_num_min = num_devices;
+ struct vhost_dev *vdev;
+ uint32_t regionidx;
+
+ vdev = rte_zmalloc("vhost device", sizeof(*vdev), CACHE_LINE_SIZE);
+ if (vdev == NULL) {
+ RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Couldn't allocate memory for vhost dev\n",
+ dev->device_fh);
+ return -1;
+ }
+ vdev->dev = dev;
+ dev->priv = vdev;
+
+ if (zero_copy) {
+ vdev->nregions_hpa = dev->mem->nregions;
+ for (regionidx = 0; regionidx < dev->mem->nregions; regionidx++) {
+ vdev->nregions_hpa
+ += check_hpa_regions(
+ dev->mem->regions[regionidx].guest_phys_address
+ + dev->mem->regions[regionidx].address_offset,
+ dev->mem->regions[regionidx].memory_size);
+
+ }
+
+ vdev->regions_hpa = (struct virtio_memory_regions_hpa *) rte_zmalloc("vhost hpa region",
+ sizeof(struct virtio_memory_regions_hpa) * vdev->nregions_hpa,
+ CACHE_LINE_SIZE);
+ if (vdev->regions_hpa == NULL) {
+ RTE_LOG(ERR, VHOST_CONFIG, "Cannot allocate memory for hpa region\n");
+ rte_free(vdev);
+ return -1;
+ }
+
+
+ if (fill_hpa_memory_regions(
+ vdev->regions_hpa, dev->mem
+ ) != vdev->nregions_hpa) {
+
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "hpa memory regions number mismatch: "
+ "[%d]\n", vdev->nregions_hpa);
+ rte_free(vdev->regions_hpa);
+ rte_free(vdev);
+ return -1;
+ }
+ }
+
/* Add device to main ll */
ll_dev = get_data_ll_free_entry(&ll_root_free);
RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") No free entry found in linked list. Device limit "
"of %d devices per core has been reached\n",
dev->device_fh, num_devices);
+ if (vdev->regions_hpa)
+ rte_free(vdev->regions_hpa);
+ rte_free(vdev);
return -1;
}
- ll_dev->dev = dev;
+ ll_dev->vdev = vdev;
add_data_ll_entry(&ll_root_used, ll_dev);
- ll_dev->dev->vmdq_rx_q
- = ll_dev->dev->device_fh * (num_queues / num_devices);
+ vdev->vmdq_rx_q
+ = dev->device_fh * (num_queues / num_devices);
if (zero_copy) {
- uint32_t index = ll_dev->dev->vmdq_rx_q;
+ uint32_t index = vdev->vmdq_rx_q;
uint32_t count_in_ring, i;
struct mbuf_table *tx_q;
dev->device_fh,
rte_ring_count(vpool_array[index].ring));
- tx_q = &tx_queue_zcp[(uint16_t)dev->vmdq_rx_q];
- tx_q->txq_id = dev->vmdq_rx_q;
+ tx_q = &tx_queue_zcp[(uint16_t)vdev->vmdq_rx_q];
+ tx_q->txq_id = vdev->vmdq_rx_q;
- if (rte_eth_dev_tx_queue_start(ports[0], dev->vmdq_rx_q) != 0) {
- struct vpool *vpool = &vpool_array[dev->vmdq_rx_q];
+ if (rte_eth_dev_tx_queue_start(ports[0], vdev->vmdq_rx_q) != 0) {
+ struct vpool *vpool = &vpool_array[vdev->vmdq_rx_q];
LOG_DEBUG(VHOST_CONFIG,
"(%"PRIu64") In new_device: Failed to start "
"tx queue:%d\n",
- dev->device_fh, dev->vmdq_rx_q);
+ dev->device_fh, vdev->vmdq_rx_q);
mbuf_destroy_zcp(vpool);
+ rte_free(vdev->regions_hpa);
+ rte_free(vdev);
return -1;
}
- if (rte_eth_dev_rx_queue_start(ports[0], dev->vmdq_rx_q) != 0) {
- struct vpool *vpool = &vpool_array[dev->vmdq_rx_q];
+ if (rte_eth_dev_rx_queue_start(ports[0], vdev->vmdq_rx_q) != 0) {
+ struct vpool *vpool = &vpool_array[vdev->vmdq_rx_q];
LOG_DEBUG(VHOST_CONFIG,
"(%"PRIu64") In new_device: Failed to start "
"rx queue:%d\n",
- dev->device_fh, dev->vmdq_rx_q);
+ dev->device_fh, vdev->vmdq_rx_q);
/* Stop the TX queue. */
if (rte_eth_dev_tx_queue_stop(ports[0],
- dev->vmdq_rx_q) != 0) {
+ vdev->vmdq_rx_q) != 0) {
LOG_DEBUG(VHOST_CONFIG,
"(%"PRIu64") In new_device: Failed to "
"stop tx queue:%d\n",
- dev->device_fh, dev->vmdq_rx_q);
+ dev->device_fh, vdev->vmdq_rx_q);
}
mbuf_destroy_zcp(vpool);
+ rte_free(vdev->regions_hpa);
+ rte_free(vdev);
return -1;
}
}
/*reset ready flag*/
- dev->ready = DEVICE_MAC_LEARNING;
- dev->remove = 0;
+ vdev->ready = DEVICE_MAC_LEARNING;
+ vdev->remove = 0;
/* Find a suitable lcore to add the device. */
RTE_LCORE_FOREACH_SLAVE(lcore) {
ll_dev = get_data_ll_free_entry(&lcore_info[ll_dev->dev->coreid].lcore_ll->ll_root_free);
if (ll_dev == NULL) {
RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Failed to add device to data core\n", dev->device_fh);
- dev->ready = DEVICE_SAFE_REMOVE;
+ vdev->ready = DEVICE_SAFE_REMOVE;
destroy_device(dev);
+ if (vdev->regions_hpa)
+ rte_free(vdev->regions_hpa);
+ rte_free(vdev);
return -1;
}
- ll_dev->dev = dev;
+ ll_dev->vdev = vdev;
+ vdev->coreid = core_add;
+
add_data_ll_entry(&lcore_info[ll_dev->dev->coreid].lcore_ll->ll_root_used, ll_dev);
/* Initialize device stats */
/* Disable notifications. */
set_irq_status(dev);
- lcore_info[ll_dev->dev->coreid].lcore_ll->device_num++;
+ lcore_info[vdev->coreid].lcore_ll->device_num++;
dev->flags |= VIRTIO_DEV_RUNNING;
- RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d\n", dev->device_fh, dev->coreid);
+ RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d\n", dev->device_fh, vdev->coreid);
return 0;
}
dev_ll = ll_root_used;
while (dev_ll != NULL) {
- device_fh = (uint32_t)dev_ll->dev->device_fh;
+ device_fh = (uint32_t)dev_ll->vdev->dev->device_fh;
tx_total = dev_statistics[device_fh].tx_total;
tx = dev_statistics[device_fh].tx;
tx_dropped = tx_total - tx;
}
/* Register CUSE device to handle IOCTLs. */
- ret = register_cuse_device((char*)&dev_basename, dev_index, get_virtio_net_callbacks());
+ ret = rte_vhost_driver_register((char *)&dev_basename);
if (ret != 0)
rte_exit(EXIT_FAILURE,"CUSE device setup failure.\n");
- init_virtio_net(&virtio_net_device_ops);
+ rte_vhost_driver_callback_register(&virtio_net_device_ops);
/* Start CUSE session. */
- start_cuse_session_loop();
+ rte_vhost_driver_session_start();
return 0;
}
git.droids-corp.org / dpdk.git / blobdiff