{
int i;
uint16_t used_idx = vq->last_used_idx;
+ uint16_t head_idx = vq->last_used_idx;
+ uint16_t head_flags = 0;
/* Split loop in two to save memory barriers */
for (i = 0; i < vq->shadow_used_idx; i++) {
flags &= ~VRING_DESC_F_AVAIL;
}
- vq->desc_packed[vq->last_used_idx].flags = flags;
+ if (i > 0) {
+ vq->desc_packed[vq->last_used_idx].flags = flags;
- vhost_log_cache_used_vring(dev, vq,
+ vhost_log_cache_used_vring(dev, vq,
vq->last_used_idx *
sizeof(struct vring_packed_desc),
sizeof(struct vring_packed_desc));
+ } else {
+ head_idx = vq->last_used_idx;
+ head_flags = flags;
+ }
vq->last_used_idx += vq->shadow_used_packed[i].count;
if (vq->last_used_idx >= vq->size) {
}
}
- rte_smp_wmb();
+ vq->desc_packed[head_idx].flags = head_flags;
+
+ vhost_log_cache_used_vring(dev, vq,
+ head_idx *
+ sizeof(struct vring_packed_desc),
+ sizeof(struct vring_packed_desc));
+
vq->shadow_used_idx = 0;
vhost_log_cache_sync(dev, vq);
}
/* IP cksum verification cannot be bypassed, then calculate here */
if (m_buf->ol_flags & PKT_TX_IP_CKSUM) {
- struct ipv4_hdr *ipv4_hdr;
+ struct rte_ipv4_hdr *ipv4_hdr;
- ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct ipv4_hdr *,
+ ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct rte_ipv4_hdr *,
m_buf->l2_len);
ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr);
}
uint16_t vec_id = *vec_idx;
uint32_t len = 0;
uint64_t dlen;
+ uint32_t nr_descs = vq->size;
+ uint32_t cnt = 0;
struct vring_desc *descs = vq->desc;
struct vring_desc *idesc = NULL;
+ if (unlikely(idx >= vq->size))
+ return -1;
+
*desc_chain_head = idx;
if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) {
dlen = vq->desc[idx].len;
+ nr_descs = dlen / sizeof(struct vring_desc);
+ if (unlikely(nr_descs > vq->size))
+ return -1;
+
descs = (struct vring_desc *)(uintptr_t)
vhost_iova_to_vva(dev, vq, vq->desc[idx].addr,
&dlen,
}
while (1) {
- if (unlikely(idx >= vq->size)) {
+ if (unlikely(idx >= nr_descs || cnt++ >= nr_descs)) {
free_ind_table(idesc);
return -1;
}
if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
return -1;
+ /*
+ * The ordering between desc flags and desc
+ * content reads need to be enforced.
+ */
+ rte_smp_rmb();
+
*desc_count = 0;
*len = 0;
if (unlikely(vec_id >= BUF_VECTOR_MAX))
return -1;
+ if (unlikely(*desc_count >= vq->size))
+ return -1;
+
*desc_count += 1;
*buf_id = descs[avail_idx].id;
struct buf_vector buf_vec[BUF_VECTOR_MAX];
uint16_t avail_head;
- rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
avail_head = *((volatile uint16_t *)&vq->avail->idx);
+ /*
+ * The ordering between avail index and
+ * desc reads needs to be enforced.
+ */
+ rte_smp_rmb();
+
+ rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
+
for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen;
uint16_t nr_vec = 0;
static void
parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
{
- struct ipv4_hdr *ipv4_hdr;
- struct ipv6_hdr *ipv6_hdr;
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
void *l3_hdr = NULL;
- struct ether_hdr *eth_hdr;
+ struct rte_ether_hdr *eth_hdr;
uint16_t ethertype;
- eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
+ eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
- m->l2_len = sizeof(struct ether_hdr);
+ m->l2_len = sizeof(struct rte_ether_hdr);
ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
- if (ethertype == ETHER_TYPE_VLAN) {
- struct vlan_hdr *vlan_hdr = (struct vlan_hdr *)(eth_hdr + 1);
+ if (ethertype == RTE_ETHER_TYPE_VLAN) {
+ struct rte_vlan_hdr *vlan_hdr =
+ (struct rte_vlan_hdr *)(eth_hdr + 1);
- m->l2_len += sizeof(struct vlan_hdr);
+ m->l2_len += sizeof(struct rte_vlan_hdr);
ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
}
l3_hdr = (char *)eth_hdr + m->l2_len;
switch (ethertype) {
- case ETHER_TYPE_IPv4:
+ case RTE_ETHER_TYPE_IPv4:
ipv4_hdr = l3_hdr;
*l4_proto = ipv4_hdr->next_proto_id;
m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
*l4_hdr = (char *)l3_hdr + m->l3_len;
m->ol_flags |= PKT_TX_IPV4;
break;
- case ETHER_TYPE_IPv6:
+ case RTE_ETHER_TYPE_IPv6:
ipv6_hdr = l3_hdr;
*l4_proto = ipv6_hdr->proto;
- m->l3_len = sizeof(struct ipv6_hdr);
+ m->l3_len = sizeof(struct rte_ipv6_hdr);
*l4_hdr = (char *)l3_hdr + m->l3_len;
m->ol_flags |= PKT_TX_IPV6;
break;
}
}
-static __rte_always_inline void
-put_zmbuf(struct zcopy_mbuf *zmbuf)
-{
- zmbuf->in_use = 0;
-}
-
static __rte_always_inline int
copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
struct buf_vector *buf_vec, uint16_t nr_vec,
return NULL;
}
-static __rte_always_inline bool
-mbuf_is_consumed(struct rte_mbuf *m)
-{
- while (m) {
- if (rte_mbuf_refcnt_read(m) > 1)
- return false;
- m = m->next;
- }
-
- return true;
-}
-
-static __rte_always_inline void
-restore_mbuf(struct rte_mbuf *m)
-{
- uint32_t mbuf_size, priv_size;
-
- while (m) {
- priv_size = rte_pktmbuf_priv_size(m->pool);
- mbuf_size = sizeof(struct rte_mbuf) + priv_size;
- /* start of buffer is after mbuf structure and priv data */
-
- m->buf_addr = (char *)m + mbuf_size;
- m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size;
- m = m->next;
- }
-}
-
static __rte_always_inline uint16_t
virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
}
}
- rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
-
free_entries = *((volatile uint16_t *)&vq->avail->idx) -
vq->last_avail_idx;
if (free_entries == 0)
return 0;
+ /*
+ * The ordering between avail index and
+ * desc reads needs to be enforced.
+ */
+ rte_smp_rmb();
+
+ rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
+
VHOST_LOG_DEBUG(VHOST_DATA, "(%d) %s\n", dev->vid, __func__);
count = RTE_MIN(count, MAX_PKT_BURST);
{
uint16_t i;
- rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
-
if (unlikely(dev->dequeue_zero_copy)) {
struct zcopy_mbuf *zmbuf, *next;
git.droids-corp.org / dpdk.git / blobdiff