+ vdev->stats.tx_total++;
+ vdev->stats.tx++;
+ }
+
+ if (unlikely(tx_q->len == MAX_PKT_BURST))
+ do_drain_mbuf_table(tx_q);
+}
+
+
+static __rte_always_inline void
+drain_mbuf_table(struct mbuf_table *tx_q)
+{
+ static uint64_t prev_tsc;
+ uint64_t cur_tsc;
+
+ if (tx_q->len == 0)
+ return;
+
+ cur_tsc = rte_rdtsc();
+ if (unlikely(cur_tsc - prev_tsc > MBUF_TABLE_DRAIN_TSC)) {
+ prev_tsc = cur_tsc;
+
+ RTE_LOG_DP(DEBUG, VHOST_DATA,
+ "TX queue drained after timeout with burst size %u\n",
+ tx_q->len);
+ do_drain_mbuf_table(tx_q);
+ }
+}
+
+static __rte_always_inline void
+complete_async_pkts(struct vhost_dev *vdev, uint16_t qid)
+{
+ struct rte_mbuf *p_cpl[MAX_PKT_BURST];
+ uint16_t complete_count;
+
+ complete_count = rte_vhost_poll_enqueue_completed(vdev->vid,
+ qid, p_cpl, MAX_PKT_BURST);
+ vdev->nr_async_pkts -= complete_count;
+ if (complete_count)
+ free_pkts(p_cpl, complete_count);
+}
+
+static __rte_always_inline void
+drain_eth_rx(struct vhost_dev *vdev)
+{
+ uint16_t rx_count, enqueue_count;
+ struct rte_mbuf *pkts[MAX_PKT_BURST], *comp_pkts[MAX_PKT_BURST];
+ uint32_t nr_comp = 0;
+
+ rx_count = rte_eth_rx_burst(ports[0], vdev->vmdq_rx_q,
+ pkts, MAX_PKT_BURST);
+
+ while (likely(vdev->nr_async_pkts))
+ complete_async_pkts(vdev, VIRTIO_RXQ);
+
+ if (!rx_count)
+ return;
+
+ /*
+ * When "enable_retry" is set, here we wait and retry when there
+ * is no enough free slots in the queue to hold @rx_count packets,
+ * to diminish packet loss.
+ */
+ if (enable_retry &&
+ unlikely(rx_count > rte_vhost_avail_entries(vdev->vid,
+ VIRTIO_RXQ))) {
+ uint32_t retry;
+
+ for (retry = 0; retry < burst_rx_retry_num; retry++) {
+ rte_delay_us(burst_rx_delay_time);
+ if (rx_count <= rte_vhost_avail_entries(vdev->vid,
+ VIRTIO_RXQ))
+ break;
+ }