Simply replace the smp barriers with atomic thread fence for vhost control
path, if there are no synchronization points.
Signed-off-by: Joyce Kong <joyce.kong@arm.com>
Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
return;
/* To make sure guest memory updates are committed before logging */
return;
/* To make sure guest memory updates are committed before logging */
+ rte_atomic_thread_fence(__ATOMIC_RELEASE);
page = addr / VHOST_LOG_PAGE;
while (page * VHOST_LOG_PAGE < addr + len) {
page = addr / VHOST_LOG_PAGE;
while (page * VHOST_LOG_PAGE < addr + len) {
if (unlikely(!dev->log_base))
return;
if (unlikely(!dev->log_base))
return;
+ rte_atomic_thread_fence(__ATOMIC_RELEASE);
log_base = (unsigned long *)(uintptr_t)dev->log_base;
log_base = (unsigned long *)(uintptr_t)dev->log_base;
+ rte_atomic_thread_fence(__ATOMIC_RELEASE);
vq->log_cache_nb_elem = 0;
}
vq->log_cache_nb_elem = 0;
}
* No more room for a new log cache entry,
* so write the dirty log map directly.
*/
* No more room for a new log cache entry,
* so write the dirty log map directly.
*/
+ rte_atomic_thread_fence(__ATOMIC_RELEASE);
vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
return;
vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page);
return;
if (unlikely(idx >= vq->size))
return -1;
if (unlikely(idx >= vq->size))
return -1;
+ rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
vq->inflight_split->desc[idx].inflight = 0;
vq->inflight_split->desc[idx].inflight = 0;
+ rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
vq->inflight_split->used_idx = last_used_idx;
return 0;
vq->inflight_split->used_idx = last_used_idx;
return 0;
if (unlikely(head >= vq->size))
return -1;
if (unlikely(head >= vq->size))
return -1;
+ rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
inflight_info->desc[head].inflight = 0;
inflight_info->desc[head].inflight = 0;
+ rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
inflight_info->old_free_head = inflight_info->free_head;
inflight_info->old_used_idx = inflight_info->used_idx;
inflight_info->old_free_head = inflight_info->free_head;
inflight_info->old_used_idx = inflight_info->used_idx;
vq->avail_wrap_counter << 15;
}
vq->avail_wrap_counter << 15;
}
+ rte_atomic_thread_fence(__ATOMIC_RELEASE);
vq->device_event->flags = flags;
return 0;
vq->device_event->flags = flags;
return 0;
vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
{
/* Flush used->idx update before we read avail->flags. */
vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq)
{
/* Flush used->idx update before we read avail->flags. */
+ rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
/* Don't kick guest if we don't reach index specified by guest. */
if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
/* Don't kick guest if we don't reach index specified by guest. */
if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) {
bool signalled_used_valid, kick = false;
/* Flush used desc update. */
bool signalled_used_valid, kick = false;
/* Flush used desc update. */
+ rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
if (vq->driver_event->flags !=
if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) {
if (vq->driver_event->flags !=
+ rte_atomic_thread_fence(__ATOMIC_ACQUIRE);
off_wrap = vq->driver_event->off_wrap;
off = off_wrap & ~(1 << 15);
off_wrap = vq->driver_event->off_wrap;
off = off_wrap & ~(1 << 15);
if (inflight_split->used_idx != used->idx) {
inflight_split->desc[last_io].inflight = 0;
if (inflight_split->used_idx != used->idx) {
inflight_split->desc[last_io].inflight = 0;
+ rte_atomic_thread_fence(__ATOMIC_SEQ_CST);
inflight_split->used_idx = used->idx;
}
inflight_split->used_idx = used->idx;
}
queue_id, 0, count - vq->async_last_pkts_n);
n_pkts_cpl += vq->async_last_pkts_n;
queue_id, 0, count - vq->async_last_pkts_n);
n_pkts_cpl += vq->async_last_pkts_n;
+ rte_atomic_thread_fence(__ATOMIC_RELEASE);
while (likely((n_pkts_put < count) && n_inflight)) {
uint16_t info_idx = (start_idx + n_pkts_put) & (vq_size - 1);
while (likely((n_pkts_put < count) && n_inflight)) {
uint16_t info_idx = (start_idx + n_pkts_put) & (vq_size - 1);