+static __rte_always_inline void
+dma_error_handler_packed(struct vhost_virtqueue *vq, uint16_t slot_idx,
+ uint32_t nr_err, uint32_t *pkt_idx)
+{
+ uint16_t descs_err = 0;
+ uint16_t buffers_err = 0;
+ struct async_inflight_info *pkts_info = vq->async_pkts_info;
+
+ *pkt_idx -= nr_err;
+ /* calculate the sum of buffers and descs of DMA-error packets. */
+ while (nr_err-- > 0) {
+ descs_err += pkts_info[slot_idx % vq->size].descs;
+ buffers_err += pkts_info[slot_idx % vq->size].nr_buffers;
+ slot_idx--;
+ }
+
+ if (vq->last_avail_idx >= descs_err) {
+ vq->last_avail_idx -= descs_err;
+ } else {
+ vq->last_avail_idx = vq->last_avail_idx + vq->size - descs_err;
+ vq->avail_wrap_counter ^= 1;
+ }
+
+ vq->shadow_used_idx -= buffers_err;
+}
+
+static __rte_noinline uint32_t
+virtio_dev_rx_async_submit_packed(struct virtio_net *dev,
+ struct vhost_virtqueue *vq, uint16_t queue_id,
+ struct rte_mbuf **pkts, uint32_t count)
+{
+ uint32_t pkt_idx = 0, pkt_burst_idx = 0;
+ uint32_t remained = count;
+ int32_t n_xfer;
+ uint16_t num_buffers;
+ uint16_t num_descs;
+
+ struct rte_vhost_iov_iter *it_pool = vq->it_pool;
+ struct iovec *vec_pool = vq->vec_pool;
+ struct rte_vhost_async_desc tdes[MAX_PKT_BURST];
+ struct iovec *src_iovec = vec_pool;
+ struct iovec *dst_iovec = vec_pool + (VHOST_MAX_ASYNC_VEC >> 1);
+ struct async_inflight_info *pkts_info = vq->async_pkts_info;
+ uint32_t n_pkts = 0, pkt_err = 0;
+ uint16_t slot_idx = 0;
+ uint16_t segs_await = 0;
+ uint16_t iovec_idx = 0, it_idx = 0;
+
+ do {
+ rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
+
+ num_buffers = 0;
+ num_descs = 0;
+ if (unlikely(virtio_dev_rx_async_packed(dev, vq, pkts[pkt_idx],
+ &num_descs, &num_buffers,
+ &src_iovec[iovec_idx], &dst_iovec[iovec_idx],
+ &it_pool[it_idx], &it_pool[it_idx + 1]) < 0))
+ break;
+
+ slot_idx = (vq->async_pkts_idx + pkt_idx) % vq->size;
+
+ async_fill_desc(&tdes[pkt_burst_idx++], &it_pool[it_idx],
+ &it_pool[it_idx + 1]);
+ pkts_info[slot_idx].descs = num_descs;
+ pkts_info[slot_idx].nr_buffers = num_buffers;
+ pkts_info[slot_idx].mbuf = pkts[pkt_idx];
+ iovec_idx += it_pool[it_idx].nr_segs;
+ segs_await += it_pool[it_idx].nr_segs;
+ it_idx += 2;
+
+ pkt_idx++;
+ remained--;
+ vq_inc_last_avail_packed(vq, num_descs);
+
+ /*
+ * conditions to trigger async device transfer:
+ * - buffered packet number reaches transfer threshold
+ * - unused async iov number is less than max vhost vector
+ */
+ if (unlikely(pkt_burst_idx >= VHOST_ASYNC_BATCH_THRESHOLD ||
+ ((VHOST_MAX_ASYNC_VEC >> 1) - segs_await < BUF_VECTOR_MAX))) {
+ n_xfer = vq->async_ops.transfer_data(dev->vid,
+ queue_id, tdes, 0, pkt_burst_idx);
+ if (likely(n_xfer >= 0)) {
+ n_pkts = n_xfer;
+ } else {
+ VHOST_LOG_DATA(ERR,
+ "(%d) %s: failed to transfer data for queue id %d.\n",
+ dev->vid, __func__, queue_id);
+ n_pkts = 0;
+ }
+
+ iovec_idx = 0;
+ it_idx = 0;
+ segs_await = 0;
+
+ if (unlikely(n_pkts < pkt_burst_idx)) {
+ /*
+ * log error packets number here and do actual
+ * error processing when applications poll
+ * completion
+ */
+ pkt_err = pkt_burst_idx - n_pkts;
+ pkt_burst_idx = 0;
+ break;
+ }
+
+ pkt_burst_idx = 0;
+ }
+ } while (pkt_idx < count);
+
+ if (pkt_burst_idx) {
+ n_xfer = vq->async_ops.transfer_data(dev->vid, queue_id, tdes, 0, pkt_burst_idx);
+ if (likely(n_xfer >= 0)) {
+ n_pkts = n_xfer;
+ } else {
+ VHOST_LOG_DATA(ERR, "(%d) %s: failed to transfer data for queue id %d.\n",
+ dev->vid, __func__, queue_id);
+ n_pkts = 0;
+ }
+
+ if (unlikely(n_pkts < pkt_burst_idx))
+ pkt_err = pkt_burst_idx - n_pkts;
+ }
+
+ if (unlikely(pkt_err))
+ dma_error_handler_packed(vq, slot_idx, pkt_err, &pkt_idx);
+
+ if (likely(vq->shadow_used_idx)) {
+ /* keep used descriptors. */
+ store_dma_desc_info_packed(vq->shadow_used_packed, vq->async_buffers_packed,
+ vq->size, 0, vq->async_buffer_idx_packed,
+ vq->shadow_used_idx);
+
+ vq->async_buffer_idx_packed += vq->shadow_used_idx;
+ if (vq->async_buffer_idx_packed >= vq->size)
+ vq->async_buffer_idx_packed -= vq->size;
+
+ vq->async_pkts_idx += pkt_idx;
+ if (vq->async_pkts_idx >= vq->size)
+ vq->async_pkts_idx -= vq->size;
+
+ vq->shadow_used_idx = 0;
+ vq->async_pkts_inflight_n += pkt_idx;
+ }
+
+ return pkt_idx;
+}
+
+static __rte_always_inline void
+write_back_completed_descs_split(struct vhost_virtqueue *vq, uint16_t n_descs)
+{
+ uint16_t nr_left = n_descs;
+ uint16_t nr_copy;
+ uint16_t to, from;
+
+ do {
+ from = vq->last_async_desc_idx_split & (vq->size - 1);
+ nr_copy = nr_left + from <= vq->size ? nr_left : vq->size - from;
+ to = vq->last_used_idx & (vq->size - 1);
+
+ if (to + nr_copy <= vq->size) {
+ rte_memcpy(&vq->used->ring[to], &vq->async_descs_split[from],
+ nr_copy * sizeof(struct vring_used_elem));
+ } else {
+ uint16_t size = vq->size - to;
+
+ rte_memcpy(&vq->used->ring[to], &vq->async_descs_split[from],
+ size * sizeof(struct vring_used_elem));
+ rte_memcpy(&vq->used->ring[0], &vq->async_descs_split[from + size],
+ (nr_copy - size) * sizeof(struct vring_used_elem));
+ }
+
+ vq->last_async_desc_idx_split += nr_copy;
+ vq->last_used_idx += nr_copy;
+ nr_left -= nr_copy;
+ } while (nr_left > 0);
+}
+
+static __rte_always_inline void
+write_back_completed_descs_packed(struct vhost_virtqueue *vq,
+ uint16_t n_buffers)
+{
+ uint16_t nr_left = n_buffers;
+ uint16_t from, to;
+
+ do {
+ from = vq->last_async_buffer_idx_packed;
+ to = (from + nr_left) % vq->size;
+ if (to > from) {
+ vhost_update_used_packed(vq, vq->async_buffers_packed + from, to - from);
+ vq->last_async_buffer_idx_packed += nr_left;
+ nr_left = 0;
+ } else {
+ vhost_update_used_packed(vq, vq->async_buffers_packed + from,
+ vq->size - from);
+ vq->last_async_buffer_idx_packed = 0;
+ nr_left -= vq->size - from;
+ }
+ } while (nr_left > 0);
+}
+
+static __rte_always_inline uint16_t
+vhost_poll_enqueue_completed(struct virtio_net *dev, uint16_t queue_id,
+ struct rte_mbuf **pkts, uint16_t count)
+{
+ struct vhost_virtqueue *vq;
+ struct async_inflight_info *pkts_info;
+ int32_t n_cpl;
+ uint16_t n_pkts_cpl = 0, n_pkts_put = 0, n_descs = 0, n_buffers = 0;
+ uint16_t start_idx, pkts_idx, vq_size;
+ uint16_t from, i;
+
+ vq = dev->virtqueue[queue_id];
+ pkts_idx = vq->async_pkts_idx % vq->size;
+ pkts_info = vq->async_pkts_info;
+ vq_size = vq->size;
+ start_idx = virtio_dev_rx_async_get_info_idx(pkts_idx,
+ vq_size, vq->async_pkts_inflight_n);
+
+ if (count > vq->async_last_pkts_n) {
+ n_cpl = vq->async_ops.check_completed_copies(dev->vid,
+ queue_id, 0, count - vq->async_last_pkts_n);
+ if (likely(n_cpl >= 0)) {
+ n_pkts_cpl = n_cpl;
+ } else {
+ VHOST_LOG_DATA(ERR,
+ "(%d) %s: failed to check completed copies for queue id %d.\n",
+ dev->vid, __func__, queue_id);
+ n_pkts_cpl = 0;
+ }
+ }
+
+ n_pkts_cpl += vq->async_last_pkts_n;
+ n_pkts_put = RTE_MIN(n_pkts_cpl, count);
+ if (unlikely(n_pkts_put == 0)) {
+ vq->async_last_pkts_n = n_pkts_cpl;
+ return 0;
+ }
+
+ if (vq_is_packed(dev)) {
+ for (i = 0; i < n_pkts_put; i++) {
+ from = (start_idx + i) % vq_size;
+ n_buffers += pkts_info[from].nr_buffers;
+ pkts[i] = pkts_info[from].mbuf;
+ }
+ } else {
+ for (i = 0; i < n_pkts_put; i++) {
+ from = (start_idx + i) & (vq_size - 1);
+ n_descs += pkts_info[from].descs;
+ pkts[i] = pkts_info[from].mbuf;
+ }
+ }
+ vq->async_last_pkts_n = n_pkts_cpl - n_pkts_put;
+ vq->async_pkts_inflight_n -= n_pkts_put;
+
+ if (likely(vq->enabled && vq->access_ok)) {
+ if (vq_is_packed(dev)) {
+ write_back_completed_descs_packed(vq, n_buffers);
+
+ vhost_vring_call_packed(dev, vq);
+ } else {
+ write_back_completed_descs_split(vq, n_descs);
+
+ __atomic_add_fetch(&vq->used->idx, n_descs,
+ __ATOMIC_RELEASE);
+ vhost_vring_call_split(dev, vq);
+ }
+ } else {
+ if (vq_is_packed(dev)) {
+ vq->last_async_buffer_idx_packed += n_buffers;
+ if (vq->last_async_buffer_idx_packed >= vq->size)
+ vq->last_async_buffer_idx_packed -= vq->size;
+ } else {
+ vq->last_async_desc_idx_split += n_descs;
+ }
+ }
+
+ return n_pkts_put;
+}
+
+uint16_t
+rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id,
+ struct rte_mbuf **pkts, uint16_t count)
+{
+ struct virtio_net *dev = get_device(vid);
+ struct vhost_virtqueue *vq;
+ uint16_t n_pkts_cpl = 0;
+
+ if (unlikely(!dev))
+ return 0;
+
+ VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
+ if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
+ VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n",
+ dev->vid, __func__, queue_id);
+ return 0;
+ }
+
+ vq = dev->virtqueue[queue_id];
+
+ if (unlikely(!vq->async_registered)) {
+ VHOST_LOG_DATA(ERR, "(%d) %s: async not registered for queue id %d.\n",
+ dev->vid, __func__, queue_id);
+ return 0;
+ }
+
+ rte_spinlock_lock(&vq->access_lock);
+
+ n_pkts_cpl = vhost_poll_enqueue_completed(dev, queue_id, pkts, count);
+
+ rte_spinlock_unlock(&vq->access_lock);
+
+ return n_pkts_cpl;
+}
+
+uint16_t
+rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,
+ struct rte_mbuf **pkts, uint16_t count)
+{
+ struct virtio_net *dev = get_device(vid);
+ struct vhost_virtqueue *vq;
+ uint16_t n_pkts_cpl = 0;
+
+ if (!dev)
+ return 0;
+
+ VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
+ if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
+ VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n",
+ dev->vid, __func__, queue_id);
+ return 0;
+ }
+
+ vq = dev->virtqueue[queue_id];
+
+ if (unlikely(!vq->async_registered)) {
+ VHOST_LOG_DATA(ERR, "(%d) %s: async not registered for queue id %d.\n",
+ dev->vid, __func__, queue_id);
+ return 0;
+ }
+
+ n_pkts_cpl = vhost_poll_enqueue_completed(dev, queue_id, pkts, count);
+
+ return n_pkts_cpl;
+}
+
+static __rte_always_inline uint32_t
+virtio_dev_rx_async_submit(struct virtio_net *dev, uint16_t queue_id,
+ struct rte_mbuf **pkts, uint32_t count)
+{
+ struct vhost_virtqueue *vq;
+ uint32_t nb_tx = 0;
+
+ VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
+ if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
+ VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n",
+ dev->vid, __func__, queue_id);
+ return 0;
+ }
+
+ vq = dev->virtqueue[queue_id];
+
+ rte_spinlock_lock(&vq->access_lock);
+
+ if (unlikely(!vq->enabled || !vq->async_registered))
+ goto out_access_unlock;