1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2009-2012,2016 Microsoft Corp.
3 * Copyright (c) 2012 NetApp Inc.
4 * Copyright (c) 2012 Citrix Inc.
15 #include <rte_tailq.h>
17 #include <rte_malloc.h>
19 #include <rte_atomic.h>
20 #include <rte_memory.h>
21 #include <rte_pause.h>
22 #include <rte_bus_vmbus.h>
26 /* Increase bufring index by inc with wraparound */
27 static inline uint32_t vmbus_br_idxinc(uint32_t idx, uint32_t inc, uint32_t sz)
36 void vmbus_br_setup(struct vmbus_br *br, void *buf, unsigned int blen)
39 br->windex = br->vbr->windex;
40 br->dsize = blen - sizeof(struct vmbus_bufring);
44 * When we write to the ring buffer, check if the host needs to be
48 * - The host guarantees that while it is draining the TX bufring,
49 * it will set the br_imask to indicate it does not need to be
50 * interrupted when new data are added.
51 * - The host guarantees that it will completely drain the TX bufring
52 * before exiting the read loop. Further, once the TX bufring is
53 * empty, it will clear the br_imask and re-check to see if new
57 vmbus_txbr_need_signal(const struct vmbus_bufring *vbr, uint32_t old_windex)
66 * This is the only case we need to signal when the
67 * ring transitions from being empty to non-empty.
69 return old_windex == vbr->rindex;
72 static inline uint32_t
73 vmbus_txbr_copyto(const struct vmbus_br *tbr, uint32_t windex,
74 const void *src0, uint32_t cplen)
76 uint8_t *br_data = tbr->vbr->data;
77 uint32_t br_dsize = tbr->dsize;
78 const uint8_t *src = src0;
80 /* XXX use double mapping like Linux kernel? */
81 if (cplen > br_dsize - windex) {
82 uint32_t fraglen = br_dsize - windex;
84 /* Wrap-around detected */
85 memcpy(br_data + windex, src, fraglen);
86 memcpy(br_data, src + fraglen, cplen - fraglen);
88 memcpy(br_data + windex, src, cplen);
91 return vmbus_br_idxinc(windex, cplen, br_dsize);
95 * Write scattered channel packet to TX bufring.
97 * The offset of this channel packet is written as a 64bits value
98 * immediately after this channel packet.
100 * The write goes through three stages:
101 * 1. Reserve space in ring buffer for the new data.
102 * Writer atomically moves priv_write_index.
103 * 2. Copy the new data into the ring.
104 * 3. Update the tail of the ring (visible to host) that indicates
105 * next read location. Writer updates write_index
108 vmbus_txbr_write(struct vmbus_br *tbr, const struct iovec iov[], int iovlen,
111 struct vmbus_bufring *vbr = tbr->vbr;
112 uint32_t ring_size = tbr->dsize;
113 uint32_t old_windex, next_windex, windex, total;
114 uint64_t save_windex;
118 for (i = 0; i < iovlen; i++)
119 total += iov[i].iov_len;
120 total += sizeof(save_windex);
122 /* Reserve space in ring */
126 /* Get current free location */
127 old_windex = tbr->windex;
129 /* Prevent compiler reordering this with calculation */
130 rte_compiler_barrier();
132 avail = vmbus_br_availwrite(tbr, old_windex);
134 /* If not enough space in ring, then tell caller. */
138 next_windex = vmbus_br_idxinc(old_windex, total, ring_size);
140 /* Atomic update of next write_index for other threads */
141 } while (!rte_atomic32_cmpset(&tbr->windex, old_windex, next_windex));
143 /* Space from old..new is now reserved */
145 for (i = 0; i < iovlen; i++) {
146 windex = vmbus_txbr_copyto(tbr, windex,
147 iov[i].iov_base, iov[i].iov_len);
150 /* Set the offset of the current channel packet. */
151 save_windex = ((uint64_t)old_windex) << 32;
152 windex = vmbus_txbr_copyto(tbr, windex, &save_windex,
153 sizeof(save_windex));
155 /* The region reserved should match region used */
156 RTE_ASSERT(windex == next_windex);
158 /* Ensure that data is available before updating host index */
161 /* Checkin for our reservation. wait for our turn to update host */
162 while (!rte_atomic32_cmpset(&vbr->windex, old_windex, next_windex))
165 /* If host had read all data before this, then need to signal */
166 *need_sig |= vmbus_txbr_need_signal(vbr, old_windex);
170 static inline uint32_t
171 vmbus_rxbr_copyfrom(const struct vmbus_br *rbr, uint32_t rindex,
172 void *dst0, size_t cplen)
174 const uint8_t *br_data = rbr->vbr->data;
175 uint32_t br_dsize = rbr->dsize;
178 if (cplen > br_dsize - rindex) {
179 uint32_t fraglen = br_dsize - rindex;
181 /* Wrap-around detected. */
182 memcpy(dst, br_data + rindex, fraglen);
183 memcpy(dst + fraglen, br_data, cplen - fraglen);
185 memcpy(dst, br_data + rindex, cplen);
188 return vmbus_br_idxinc(rindex, cplen, br_dsize);
191 /* Copy data from receive ring but don't change index */
193 vmbus_rxbr_peek(const struct vmbus_br *rbr, void *data, size_t dlen)
198 * The requested data and the 64bits channel packet
199 * offset should be there at least.
201 avail = vmbus_br_availread(rbr);
202 if (avail < dlen + sizeof(uint64_t))
205 vmbus_rxbr_copyfrom(rbr, rbr->vbr->rindex, data, dlen);
210 * Copy data from receive ring and change index
212 * We assume (dlen + skip) == sizeof(channel packet).
215 vmbus_rxbr_read(struct vmbus_br *rbr, void *data, size_t dlen, size_t skip)
217 struct vmbus_bufring *vbr = rbr->vbr;
218 uint32_t br_dsize = rbr->dsize;
221 if (vmbus_br_availread(rbr) < dlen + skip + sizeof(uint64_t))
224 /* Record where host was when we started read (for debug) */
225 rbr->windex = rbr->vbr->windex;
228 * Copy channel packet from RX bufring.
230 rindex = vmbus_br_idxinc(rbr->vbr->rindex, skip, br_dsize);
231 rindex = vmbus_rxbr_copyfrom(rbr, rindex, data, dlen);
234 * Discard this channel packet's 64bits offset, which is useless to us.
236 rindex = vmbus_br_idxinc(rindex, sizeof(uint64_t), br_dsize);
238 /* Update the read index _after_ the channel packet is fetched. */
239 rte_compiler_barrier();
241 vbr->rindex = rindex;