vhost: check message header size read

[dpdk.git] / lib / librte_vhost / virtio_net.c

diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index cb54ebe..37c47c7 100644 (file)
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -825,7 +825,7 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
        else
                hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
 
-       VHOST_LOG_DEBUG(VHOST_DATA, "(%d) RX: num merge buffers %d\n",
+       VHOST_LOG_DATA(DEBUG, "(%d) RX: num merge buffers %d\n",
                dev->vid, num_buffers);
 
        if (unlikely(buf_len < dev->vhost_hlen)) {
@@ -1009,14 +1009,14 @@ virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
                if (unlikely(reserve_avail_buf_split(dev, vq,
                                                pkt_len, buf_vec, &num_buffers,
                                                avail_head, &nr_vec) < 0)) {
-                       VHOST_LOG_DEBUG(VHOST_DATA,
+                       VHOST_LOG_DATA(DEBUG,
                                "(%d) failed to get enough desc from vring\n",
                                dev->vid);
                        vq->shadow_used_idx -= num_buffers;
                        break;
                }
 
-               VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
+               VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n",
                        dev->vid, vq->last_avail_idx,
                        vq->last_avail_idx + num_buffers);
 
@@ -1131,13 +1131,13 @@ virtio_dev_rx_single_packed(struct virtio_net *dev,
        rte_smp_rmb();
        if (unlikely(vhost_enqueue_single_packed(dev, vq, pkt, buf_vec,
                                                 &nr_descs) < 0)) {
-               VHOST_LOG_DEBUG(VHOST_DATA,
+               VHOST_LOG_DATA(DEBUG,
                                "(%d) failed to get enough desc from vring\n",
                                dev->vid);
                return -1;
        }
 
-       VHOST_LOG_DEBUG(VHOST_DATA, "(%d) current index %d | end index %d\n",
+       VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n",
                        dev->vid, vq->last_avail_idx,
                        vq->last_avail_idx + nr_descs);
 
@@ -1159,7 +1159,8 @@ virtio_dev_rx_packed(struct virtio_net *dev,
                rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
 
                if (remained >= PACKED_BATCH_SIZE) {
-                       if (!virtio_dev_rx_batch_packed(dev, vq, pkts)) {
+                       if (!virtio_dev_rx_batch_packed(dev, vq,
+                                                       &pkts[pkt_idx])) {
                                pkt_idx += PACKED_BATCH_SIZE;
                                remained -= PACKED_BATCH_SIZE;
                                continue;
@@ -1191,9 +1192,9 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id,
        struct vhost_virtqueue *vq;
        uint32_t nb_tx = 0;
 
-       VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
+       VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
        if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) {
-               RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
+               VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n",
                        dev->vid, __func__, queue_id);
                return 0;
        }
@@ -1241,7 +1242,7 @@ rte_vhost_enqueue_burst(int vid, uint16_t queue_id,
                return 0;
 
        if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
-               RTE_LOG(ERR, VHOST_DATA,
+               VHOST_LOG_DATA(ERR,
                        "(%d) %s: built-in vhost net backend is disabled.\n",
                        dev->vid, __func__);
                return 0;
@@ -1358,7 +1359,7 @@ vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
                        m->l4_len = sizeof(struct rte_udp_hdr);
                        break;
                default:
-                       RTE_LOG(WARNING, VHOST_DATA,
+                       VHOST_LOG_DATA(WARNING,
                                "unsupported gso type %u.\n", hdr->gso_type);
                        break;
                }
@@ -1530,7 +1531,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
                if (mbuf_avail == 0) {
                        cur = rte_pktmbuf_alloc(mbuf_pool);
                        if (unlikely(cur == NULL)) {
-                               RTE_LOG(ERR, VHOST_DATA, "Failed to "
+                               VHOST_LOG_DATA(ERR, "Failed to "
                                        "allocate memory for mbuf.\n");
                                error = -1;
                                goto out;
@@ -1635,7 +1636,7 @@ virtio_dev_extbuf_alloc(struct rte_mbuf *pkt, uint32_t size)
                                              virtio_dev_extbuf_free, buf);
                if (unlikely(shinfo == NULL)) {
                        rte_free(buf);
-                       RTE_LOG(ERR, VHOST_DATA, "Failed to init shinfo\n");
+                       VHOST_LOG_DATA(ERR, "Failed to init shinfo\n");
                        return -1;
                }
        }
@@ -1657,7 +1658,7 @@ virtio_dev_pktmbuf_alloc(struct virtio_net *dev, struct rte_mempool *mp,
        struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp);
 
        if (unlikely(pkt == NULL)) {
-               RTE_LOG(ERR, VHOST_DATA,
+               VHOST_LOG_DATA(ERR,
                        "Failed to allocate memory for mbuf.\n");
                return NULL;
        }
@@ -1725,11 +1726,11 @@ virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
 
        rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
 
-       VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
+       VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__);
 
        count = RTE_MIN(count, MAX_PKT_BURST);
        count = RTE_MIN(count, free_entries);
-       VHOST_LOG_DEBUG(VHOST_DATA, "(%d) about to dequeue %u buffers\n",
+       VHOST_LOG_DATA(DEBUG, "(%d) about to dequeue %u buffers\n",
                        dev->vid, count);
 
        for (i = 0; i < count; i++) {
@@ -1937,7 +1938,7 @@ vhost_dequeue_single_packed(struct virtio_net *dev,
 
        *pkts = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, buf_len);
        if (unlikely(*pkts == NULL)) {
-               RTE_LOG(ERR, VHOST_DATA,
+               VHOST_LOG_DATA(ERR,
                        "Failed to allocate memory for mbuf.\n");
                return -1;
        }
@@ -2132,6 +2133,20 @@ virtio_dev_tx_packed_zmbuf(struct virtio_net *dev,
        return pkt_idx;
 }
 
+static __rte_always_inline bool
+next_desc_is_avail(const struct vhost_virtqueue *vq)
+{
+       bool wrap_counter = vq->avail_wrap_counter;
+       uint16_t next_used_idx = vq->last_used_idx + 1;
+
+       if (next_used_idx >= vq->size) {
+               next_used_idx -= vq->size;
+               wrap_counter ^= 1;
+       }
+
+       return desc_is_avail(&vq->desc_packed[next_used_idx], wrap_counter);
+}
+
 static __rte_noinline uint16_t
 virtio_dev_tx_packed(struct virtio_net *dev,
                     struct vhost_virtqueue *vq,
@@ -2164,9 +2179,20 @@ virtio_dev_tx_packed(struct virtio_net *dev,
 
        } while (remained);
 
-       if (vq->shadow_used_idx)
+       if (vq->shadow_used_idx) {
                do_data_copy_dequeue(vq);
 
+               if (remained && !next_desc_is_avail(vq)) {
+                       /*
+                        * The guest may be waiting to TX some buffers to
+                        * enqueue more to avoid bufferfloat, so we try to
+                        * reduce latency here.
+                        */
+                       vhost_flush_dequeue_shadow_packed(dev, vq);
+                       vhost_vring_call_packed(dev, vq);
+               }
+       }
+
        return pkt_idx;
 }
 
@@ -2183,14 +2209,15 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
                return 0;
 
        if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) {
-               RTE_LOG(ERR, VHOST_DATA,
+               VHOST_LOG_DATA(ERR,
                        "(%d) %s: built-in vhost net backend is disabled.\n",
                        dev->vid, __func__);
                return 0;
        }
 
        if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) {
-               RTE_LOG(ERR, VHOST_DATA, "(%d) %s: invalid virtqueue idx %d.\n",
+               VHOST_LOG_DATA(ERR,
+                       "(%d) %s: invalid virtqueue idx %d.\n",
                        dev->vid, __func__, queue_id);
                return 0;
        }
@@ -2235,8 +2262,7 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id,
 
                rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac);
                if (rarp_mbuf == NULL) {
-                       RTE_LOG(ERR, VHOST_DATA,
-                               "Failed to make RARP packet.\n");
+                       VHOST_LOG_DATA(ERR, "Failed to make RARP packet.\n");
                        count = 0;
                        goto out;
                }