- n_segs_cpl = vq->async_ops.check_completed_copies(vid, queue_id,
- 0, ASYNC_MAX_POLL_SEG - vq->async_last_seg_n) +
- vq->async_last_seg_n;
-
- rte_smp_wmb();
-
- while (likely((n_pkts_put < count) && n_inflight)) {
- uint64_t info = async_pending_info[
- (start_idx + n_pkts_put) & (vq_size - 1)];
- uint64_t n_segs;
- n_pkts_put++;
- n_inflight--;
- n_descs += info & ASYNC_PENDING_INFO_N_MSK;
- n_segs = info >> ASYNC_PENDING_INFO_N_SFT;
-
- if (n_segs) {
- if (unlikely(n_segs_cpl < n_segs)) {
- n_pkts_put--;
- n_inflight++;
- n_descs -= info & ASYNC_PENDING_INFO_N_MSK;
- if (n_segs_cpl) {
- async_pending_info[
- (start_idx + n_pkts_put) &
- (vq_size - 1)] =
- ((n_segs - n_segs_cpl) <<
- ASYNC_PENDING_INFO_N_SFT) |
- (info & ASYNC_PENDING_INFO_N_MSK);
- n_segs_cpl = 0;
- }
- break;
+ if (count > vq->async_last_pkts_n)
+ n_pkts_cpl = vq->async_ops.check_completed_copies(vid,
+ queue_id, 0, count - vq->async_last_pkts_n);
+ n_pkts_cpl += vq->async_last_pkts_n;
+
+ n_pkts_put = RTE_MIN(count, n_pkts_cpl);
+ if (unlikely(n_pkts_put == 0)) {
+ vq->async_last_pkts_n = n_pkts_cpl;
+ goto done;
+ }
+
+ for (i = 0; i < n_pkts_put; i++) {
+ from = (start_idx + i) & (vq_size - 1);
+ n_descs += pkts_info[from].descs;
+ pkts[i] = pkts_info[from].mbuf;
+ }
+ vq->async_last_pkts_n = n_pkts_cpl - n_pkts_put;
+ vq->async_pkts_inflight_n -= n_pkts_put;
+
+ if (likely(vq->enabled && vq->access_ok)) {
+ uint16_t nr_left = n_descs;
+ uint16_t nr_copy;
+ uint16_t to;
+
+ /* write back completed descriptors to used ring */
+ do {
+ from = vq->last_async_desc_idx & (vq->size - 1);
+ nr_copy = nr_left + from <= vq->size ? nr_left :
+ vq->size - from;
+ to = vq->last_used_idx & (vq->size - 1);
+
+ if (to + nr_copy <= vq->size) {
+ rte_memcpy(&vq->used->ring[to],
+ &vq->async_descs_split[from],
+ nr_copy *
+ sizeof(struct vring_used_elem));
+ } else {
+ uint16_t size = vq->size - to;
+
+ rte_memcpy(&vq->used->ring[to],
+ &vq->async_descs_split[from],
+ size *
+ sizeof(struct vring_used_elem));
+ rte_memcpy(vq->used->ring,
+ &vq->async_descs_split[from +
+ size], (nr_copy - size) *
+ sizeof(struct vring_used_elem));