- desc = &vq->desc[head[entry_success]];
-
- /* Discard first buffer as it is the virtio header */
- desc = &vq->desc[desc->next];
-
- /* Buffer address translation. */
- vb_addr = gpa_to_vva(dev, desc->addr);
- /* Prefetch buffer address. */
- rte_prefetch0((void *)(uintptr_t)vb_addr);
-
- used_idx = vq->last_used_idx & (vq->size - 1);
-
- if (entry_success < (free_entries - 1)) {
- /* Prefetch descriptor index. */
- rte_prefetch0(&vq->desc[head[entry_success+1]]);
- rte_prefetch0(&vq->used->ring[(used_idx + 1) & (vq->size - 1)]);
- }
-
- /* Update used index buffer information. */
- vq->used->ring[used_idx].id = head[entry_success];
- vq->used->ring[used_idx].len = 0;
-
- vb_offset = 0;
- vb_avail = desc->len;
- seg_offset = 0;
- seg_avail = buf_size;
- cpy_len = RTE_MIN(vb_avail, seg_avail);
-
- PRINT_PACKET(dev, (uintptr_t)vb_addr, desc->len, 0);
-
- /* Allocate an mbuf and populate the structure. */
- m = rte_pktmbuf_alloc(mbuf_pool);
- if (unlikely(m == NULL)) {
- RTE_LOG(ERR, VHOST_DATA,
- "Failed to allocate memory for mbuf.\n");
- return;
- }
-
- seg_num++;
- cur = m;
- prev = m;
- while (cpy_len != 0) {
- rte_memcpy((void *)(rte_pktmbuf_mtod(cur, char *) + seg_offset),
- (void *)((uintptr_t)(vb_addr + vb_offset)),
- cpy_len);
-
- seg_offset += cpy_len;
- vb_offset += cpy_len;
- vb_avail -= cpy_len;
- seg_avail -= cpy_len;
-
- if (vb_avail != 0) {
- /*
- * The segment reachs to its end,
- * while the virtio buffer in TX vring has
- * more data to be copied.
- */
- cur->pkt.data_len = seg_offset;
- m->pkt.pkt_len += seg_offset;
- /* Allocate mbuf and populate the structure. */
- cur = rte_pktmbuf_alloc(mbuf_pool);
- if (unlikely(cur == NULL)) {
- RTE_LOG(ERR, VHOST_DATA, "Failed to "
- "allocate memory for mbuf.\n");
- rte_pktmbuf_free(m);
- alloc_err = 1;
- break;
- }
-
- seg_num++;
- prev->pkt.next = cur;
- prev = cur;
- seg_offset = 0;
- seg_avail = buf_size;
- } else {
- if (desc->flags & VRING_DESC_F_NEXT) {
- /*
- * There are more virtio buffers in
- * same vring entry need to be copied.
- */
- if (seg_avail == 0) {
- /*
- * The current segment hasn't
- * room to accomodate more
- * data.
- */
- cur->pkt.data_len = seg_offset;
- m->pkt.pkt_len += seg_offset;
- /*
- * Allocate an mbuf and
- * populate the structure.
- */
- cur = rte_pktmbuf_alloc(mbuf_pool);
- if (unlikely(cur == NULL)) {
- RTE_LOG(ERR,
- VHOST_DATA,
- "Failed to "
- "allocate memory "
- "for mbuf\n");
- rte_pktmbuf_free(m);
- alloc_err = 1;
- break;
- }
- seg_num++;
- prev->pkt.next = cur;
- prev = cur;
- seg_offset = 0;
- seg_avail = buf_size;
- }
-
- desc = &vq->desc[desc->next];
-
- /* Buffer address translation. */
- vb_addr = gpa_to_vva(dev, desc->addr);
- /* Prefetch buffer address. */
- rte_prefetch0((void *)(uintptr_t)vb_addr);
- vb_offset = 0;
- vb_avail = desc->len;
-
- PRINT_PACKET(dev, (uintptr_t)vb_addr,
- desc->len, 0);
- } else {
- /* The whole packet completes. */
- cur->pkt.data_len = seg_offset;
- m->pkt.pkt_len += seg_offset;
- vb_avail = 0;
- }
- }
-
- cpy_len = RTE_MIN(vb_avail, seg_avail);
- }
-
- if (unlikely(alloc_err == 1))
- break;
-
- m->pkt.nb_segs = seg_num;
-
- /*
- * If this is the first received packet we need to learn
- * the MAC and setup VMDQ
- */
- if (dev->ready == DEVICE_MAC_LEARNING) {
- if (dev->remove || (link_vmdq(dev, m) == -1)) {
- /*
- * Discard frame if device is scheduled for
- * removal or a duplicate MAC address is found.
- */
- entry_success = free_entries;
- vq->last_used_idx += entry_success;
- rte_pktmbuf_free(m);
- break;
- }
- }
-
- virtio_tx_route(dev, m, mbuf_pool, (uint16_t)dev->device_fh);
- vq->last_used_idx++;
- entry_success++;
- rte_pktmbuf_free(m);
- }
-
- rte_compiler_barrier();
- vq->used->idx += entry_success;
- /* Kick guest if required. */
- if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
- eventfd_write((int)vq->kickfd, 1);
-
-}
-
-/*
- * This function is called by each data core. It handles all RX/TX registered with the
- * core. For TX the specific lcore linked list is used. For RX, MAC addresses are compared
- * with all devices in the main linked list.
- */
-static int
-switch_worker(__attribute__((unused)) void *arg)
-{
- struct rte_mempool *mbuf_pool = arg;
- struct virtio_net *dev = NULL;
- struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
- struct virtio_net_data_ll *dev_ll;
- struct mbuf_table *tx_q;
- volatile struct lcore_ll_info *lcore_ll;
- const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US;
- uint64_t prev_tsc, diff_tsc, cur_tsc, ret_count = 0;
- unsigned ret, i;
- const uint16_t lcore_id = rte_lcore_id();
- const uint16_t num_cores = (uint16_t)rte_lcore_count();
- uint16_t rx_count = 0;
- uint32_t mergeable = 0;
-
- RTE_LOG(INFO, VHOST_DATA, "Procesing on Core %u started\n", lcore_id);
- lcore_ll = lcore_info[lcore_id].lcore_ll;
- prev_tsc = 0;
-
- tx_q = &lcore_tx_queue[lcore_id];
- for (i = 0; i < num_cores; i ++) {
- if (lcore_ids[i] == lcore_id) {
- tx_q->txq_id = i;
- break;
- }
- }
-
- while(1) {
- cur_tsc = rte_rdtsc();
- /*
- * TX burst queue drain
- */
- diff_tsc = cur_tsc - prev_tsc;
- if (unlikely(diff_tsc > drain_tsc)) {
-
- if (tx_q->len) {
- LOG_DEBUG(VHOST_DATA, "TX queue drained after timeout with burst size %u \n", tx_q->len);
-
- /*Tx any packets in the queue*/
- ret = rte_eth_tx_burst(ports[0], (uint16_t)tx_q->txq_id,
- (struct rte_mbuf **)tx_q->m_table,
- (uint16_t)tx_q->len);
- if (unlikely(ret < tx_q->len)) {
- do {
- rte_pktmbuf_free(tx_q->m_table[ret]);
- } while (++ret < tx_q->len);
- }
-
- tx_q->len = 0;
- }
-
- prev_tsc = cur_tsc;
-
- }
-
- rte_prefetch0(lcore_ll->ll_root_used);
- /*
- * Inform the configuration core that we have exited the linked list and that no devices are
- * in use if requested.
- */
- if (lcore_ll->dev_removal_flag == REQUEST_DEV_REMOVAL)
- lcore_ll->dev_removal_flag = ACK_DEV_REMOVAL;
-
- /*
- * Process devices
- */
- dev_ll = lcore_ll->ll_root_used;
-
- while (dev_ll != NULL) {
- /*get virtio device ID*/
- dev = dev_ll->dev;
- mergeable =
- dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF);
-
- if (dev->remove) {
- dev_ll = dev_ll->next;
- unlink_vmdq(dev);
- dev->ready = DEVICE_SAFE_REMOVE;
- continue;
- }
- if (likely(dev->ready == DEVICE_RX)) {
- /*Handle guest RX*/
- rx_count = rte_eth_rx_burst(ports[0],
- (uint16_t)dev->vmdq_rx_q, pkts_burst, MAX_PKT_BURST);
-
- if (rx_count) {
- if (likely(mergeable == 0))
- ret_count =
- virtio_dev_rx(dev,
- pkts_burst, rx_count);
- else
- ret_count =
- virtio_dev_merge_rx(dev,
- pkts_burst, rx_count);
-
- if (enable_stats) {
- rte_atomic64_add(
- &dev_statistics[dev_ll->dev->device_fh].rx_total_atomic,
- rx_count);
- rte_atomic64_add(
- &dev_statistics[dev_ll->dev->device_fh].rx_atomic, ret_count);
- }
- while (likely(rx_count)) {
- rx_count--;
- rte_pktmbuf_free(pkts_burst[rx_count]);
- }
-
- }
- }
-
- if (!dev->remove) {
- /*Handle guest TX*/
- if (likely(mergeable == 0))
- virtio_dev_tx(dev, mbuf_pool);
- else
- virtio_dev_merge_tx(dev, mbuf_pool);
- }
-
- /*move to the next device in the list*/
- dev_ll = dev_ll->next;
- }
- }
-
- return 0;
-}
-
-/*
- * This function gets available ring number for zero copy rx.
- * Only one thread will call this funciton for a paticular virtio device,
- * so, it is designed as non-thread-safe function.
- */
-static inline uint32_t __attribute__((always_inline))
-get_available_ring_num_zcp(struct virtio_net *dev)
-{
- struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_RXQ];
- uint16_t avail_idx;
-
- avail_idx = *((volatile uint16_t *)&vq->avail->idx);
- return (uint32_t)(avail_idx - vq->last_used_idx_res);
-}
-
-/*
- * This function gets available ring index for zero copy rx,
- * it will retry 'burst_rx_retry_num' times till it get enough ring index.
- * Only one thread will call this funciton for a paticular virtio device,
- * so, it is designed as non-thread-safe function.
- */
-static inline uint32_t __attribute__((always_inline))
-get_available_ring_index_zcp(struct virtio_net *dev,
- uint16_t *res_base_idx, uint32_t count)
-{
- struct vhost_virtqueue *vq = dev->virtqueue[VIRTIO_RXQ];
- uint16_t avail_idx;
- uint32_t retry = 0;
- uint16_t free_entries;
-
- *res_base_idx = vq->last_used_idx_res;
- avail_idx = *((volatile uint16_t *)&vq->avail->idx);
- free_entries = (avail_idx - *res_base_idx);
-
- LOG_DEBUG(VHOST_DATA, "(%"PRIu64") in get_available_ring_index_zcp: "
- "avail idx: %d, "
- "res base idx:%d, free entries:%d\n",
- dev->device_fh, avail_idx, *res_base_idx,
- free_entries);
-
- /*
- * If retry is enabled and the queue is full then we wait
- * and retry to avoid packet loss.
- */
- if (enable_retry && unlikely(count > free_entries)) {
- for (retry = 0; retry < burst_rx_retry_num; retry++) {
- rte_delay_us(burst_rx_delay_time);
- avail_idx = *((volatile uint16_t *)&vq->avail->idx);
- free_entries = (avail_idx - *res_base_idx);
- if (count <= free_entries)
- break;
- }
- }
-
- /*check that we have enough buffers*/
- if (unlikely(count > free_entries))
- count = free_entries;
-
- if (unlikely(count == 0)) {
- LOG_DEBUG(VHOST_DATA,
- "(%"PRIu64") Fail in get_available_ring_index_zcp: "
- "avail idx: %d, res base idx:%d, free entries:%d\n",
- dev->device_fh, avail_idx,
- *res_base_idx, free_entries);
- return 0;
- }
-
- vq->last_used_idx_res = *res_base_idx + count;
-
- return count;
-}
-
-/*
- * This function put descriptor back to used list.
- */
-static inline void __attribute__((always_inline))
-put_desc_to_used_list_zcp(struct vhost_virtqueue *vq, uint16_t desc_idx)
-{
- uint16_t res_cur_idx = vq->last_used_idx;
- vq->used->ring[res_cur_idx & (vq->size - 1)].id = (uint32_t)desc_idx;
- vq->used->ring[res_cur_idx & (vq->size - 1)].len = 0;
- rte_compiler_barrier();
- *(volatile uint16_t *)&vq->used->idx += 1;
- vq->last_used_idx += 1;
-
- /* Kick the guest if necessary. */
- if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT))
- eventfd_write((int)vq->kickfd, 1);
-}
-
-/*
- * This function get available descriptor from vitio vring and un-attached mbuf
- * from vpool->ring, and then attach them together. It needs adjust the offset
- * for buff_addr and phys_addr accroding to PMD implementation, otherwise the
- * frame data may be put to wrong location in mbuf.
- */
-static inline void __attribute__((always_inline))
-attach_rxmbuf_zcp(struct virtio_net *dev)
-{
- uint16_t res_base_idx, desc_idx;
- uint64_t buff_addr, phys_addr;
- struct vhost_virtqueue *vq;
- struct vring_desc *desc;
- struct rte_mbuf *mbuf = NULL;
- struct vpool *vpool;
- hpa_type addr_type;
-
- vpool = &vpool_array[dev->vmdq_rx_q];
- vq = dev->virtqueue[VIRTIO_RXQ];
-
- do {
- if (unlikely(get_available_ring_index_zcp(dev, &res_base_idx,
- 1) != 1))
- return;
- desc_idx = vq->avail->ring[(res_base_idx) & (vq->size - 1)];
-
- desc = &vq->desc[desc_idx];
- if (desc->flags & VRING_DESC_F_NEXT) {
- desc = &vq->desc[desc->next];
- buff_addr = gpa_to_vva(dev, desc->addr);
- phys_addr = gpa_to_hpa(dev, desc->addr, desc->len,
- &addr_type);
- } else {
- buff_addr = gpa_to_vva(dev,
- desc->addr + vq->vhost_hlen);
- phys_addr = gpa_to_hpa(dev,
- desc->addr + vq->vhost_hlen,
- desc->len, &addr_type);
- }
-
- if (unlikely(addr_type == PHYS_ADDR_INVALID)) {
- RTE_LOG(ERR, VHOST_DATA, "(%"PRIu64") Invalid frame buffer"
- " address found when attaching RX frame buffer"
- " address!\n", dev->device_fh);
- put_desc_to_used_list_zcp(vq, desc_idx);
- continue;
- }
-
- /*
- * Check if the frame buffer address from guest crosses
- * sub-region or not.
- */
- if (unlikely(addr_type == PHYS_ADDR_CROSS_SUBREG)) {
- RTE_LOG(ERR, VHOST_DATA,
- "(%"PRIu64") Frame buffer address cross "
- "sub-regioin found when attaching RX frame "
- "buffer address!\n",
- dev->device_fh);
- put_desc_to_used_list_zcp(vq, desc_idx);
- continue;
- }
- } while (unlikely(phys_addr == 0));
-
- rte_ring_sc_dequeue(vpool->ring, (void **)&mbuf);
- if (unlikely(mbuf == NULL)) {
- LOG_DEBUG(VHOST_DATA,
- "(%"PRIu64") in attach_rxmbuf_zcp: "
- "ring_sc_dequeue fail.\n",
- dev->device_fh);
- put_desc_to_used_list_zcp(vq, desc_idx);
- return;
- }
-
- if (unlikely(vpool->buf_size > desc->len)) {
- LOG_DEBUG(VHOST_DATA,
- "(%"PRIu64") in attach_rxmbuf_zcp: frame buffer "
- "length(%d) of descriptor idx: %d less than room "
- "size required: %d\n",
- dev->device_fh, desc->len, desc_idx, vpool->buf_size);
- put_desc_to_used_list_zcp(vq, desc_idx);
- rte_ring_sp_enqueue(vpool->ring, (void *)mbuf);