1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2018, Microsoft Corporation.
12 #include <rte_tailq.h>
14 #include <rte_malloc.h>
16 #include <rte_atomic.h>
17 #include <rte_memory.h>
18 #include <rte_bus_vmbus.h>
23 vmbus_sync_set_bit(volatile uint32_t *addr, uint32_t mask)
25 /* Use GCC builtin which atomic does atomic OR operation */
26 __sync_or_and_fetch(addr, mask);
30 vmbus_send_interrupt(const struct rte_vmbus_device *dev, uint32_t relid)
35 int_addr = dev->int_page + relid / 32;
36 int_mask = 1u << (relid % 32);
38 vmbus_sync_set_bit(int_addr, int_mask);
42 vmbus_set_monitor(const struct rte_vmbus_device *dev, uint32_t monitor_id)
44 uint32_t *monitor_addr, monitor_mask;
45 unsigned int trigger_index;
47 trigger_index = monitor_id / HV_MON_TRIG_LEN;
48 monitor_mask = 1u << (monitor_id % HV_MON_TRIG_LEN);
50 monitor_addr = &dev->monitor_page->trigs[trigger_index].pending;
51 vmbus_sync_set_bit(monitor_addr, monitor_mask);
55 vmbus_set_event(const struct rte_vmbus_device *dev,
56 const struct vmbus_channel *chan)
58 vmbus_send_interrupt(dev, chan->relid);
59 vmbus_set_monitor(dev, chan->monitor_id);
63 * Notify host that there are data pending on our TX bufring.
65 * Since this in userspace, rely on the monitor page.
66 * Can't do a hypercall from userspace.
69 rte_vmbus_chan_signal_tx(const struct vmbus_channel *chan)
71 const struct rte_vmbus_device *dev = chan->device;
72 const struct vmbus_br *tbr = &chan->txbr;
74 /* Make sure all updates are done before signaling host */
77 /* If host is ignoring interrupts? */
81 vmbus_set_event(dev, chan);
85 /* Do a simple send directly using transmit ring. */
86 int rte_vmbus_chan_send(struct vmbus_channel *chan, uint16_t type,
87 void *data, uint32_t dlen,
88 uint64_t xactid, uint32_t flags, bool *need_sig)
90 struct vmbus_chanpkt pkt;
91 unsigned int pktlen, pad_pktlen;
92 const uint32_t hlen = sizeof(pkt);
93 bool send_evt = false;
99 pad_pktlen = RTE_ALIGN(pktlen, sizeof(uint64_t));
102 pkt.hdr.flags = flags;
103 pkt.hdr.hlen = hlen >> VMBUS_CHANPKT_SIZE_SHIFT;
104 pkt.hdr.tlen = pad_pktlen >> VMBUS_CHANPKT_SIZE_SHIFT;
105 pkt.hdr.xactid = xactid;
107 iov[0].iov_base = &pkt;
108 iov[0].iov_len = hlen;
109 iov[1].iov_base = data;
110 iov[1].iov_len = dlen;
111 iov[2].iov_base = &pad;
112 iov[2].iov_len = pad_pktlen - pktlen;
114 error = vmbus_txbr_write(&chan->txbr, iov, 3, &send_evt);
117 * caller sets need_sig to non-NULL if it will handle
118 * signaling if required later.
119 * if need_sig is NULL, signal now if needed.
122 *need_sig |= send_evt;
123 else if (error == 0 && send_evt)
124 rte_vmbus_chan_signal_tx(chan);
128 /* Do a scatter/gather send where the descriptor points to data. */
129 int rte_vmbus_chan_send_sglist(struct vmbus_channel *chan,
130 struct vmbus_gpa sg[], uint32_t sglen,
131 void *data, uint32_t dlen,
132 uint64_t xactid, bool *need_sig)
134 struct vmbus_chanpkt_sglist pkt;
135 unsigned int pktlen, pad_pktlen, hlen;
136 bool send_evt = false;
141 hlen = offsetof(struct vmbus_chanpkt_sglist, gpa[sglen]);
142 pktlen = hlen + dlen;
143 pad_pktlen = RTE_ALIGN(pktlen, sizeof(uint64_t));
145 pkt.hdr.type = VMBUS_CHANPKT_TYPE_GPA;
146 pkt.hdr.flags = VMBUS_CHANPKT_FLAG_RC;
147 pkt.hdr.hlen = hlen >> VMBUS_CHANPKT_SIZE_SHIFT;
148 pkt.hdr.tlen = pad_pktlen >> VMBUS_CHANPKT_SIZE_SHIFT;
149 pkt.hdr.xactid = xactid;
153 iov[0].iov_base = &pkt;
154 iov[0].iov_len = sizeof(pkt);
155 iov[1].iov_base = sg;
156 iov[1].iov_len = sizeof(struct vmbus_gpa) * sglen;
157 iov[2].iov_base = data;
158 iov[2].iov_len = dlen;
159 iov[3].iov_base = &pad;
160 iov[3].iov_len = pad_pktlen - pktlen;
162 error = vmbus_txbr_write(&chan->txbr, iov, 4, &send_evt);
164 /* if caller is batching, just propagate the status */
166 *need_sig |= send_evt;
167 else if (error == 0 && send_evt)
168 rte_vmbus_chan_signal_tx(chan);
172 bool rte_vmbus_chan_rx_empty(const struct vmbus_channel *channel)
174 const struct vmbus_br *br = &channel->rxbr;
176 return br->vbr->rindex == br->vbr->windex;
179 static int vmbus_read_and_signal(struct vmbus_channel *chan,
180 void *data, size_t dlen, size_t skip)
182 struct vmbus_br *rbr = &chan->rxbr;
183 uint32_t write_sz, pending_sz, bytes_read;
186 /* Record where host was when we started read (for debug) */
187 rbr->windex = rbr->vbr->windex;
189 /* Read data and skip packet header */
190 error = vmbus_rxbr_read(rbr, data, dlen, skip);
194 /* No need for signaling on older versions */
195 if (!rbr->vbr->feature_bits.feat_pending_send_sz)
198 /* Make sure reading of pending happens after new read index */
201 pending_sz = rbr->vbr->pending_send;
206 write_sz = vmbus_br_availwrite(rbr, rbr->vbr->windex);
207 bytes_read = dlen + skip + sizeof(uint64_t);
209 /* If there was space before then host was not blocked */
210 if (write_sz - bytes_read > pending_sz)
213 /* If pending write will not fit */
214 if (write_sz <= pending_sz)
217 vmbus_set_event(chan->device, chan);
221 /* TODO: replace this with inplace ring buffer (no copy) */
222 int rte_vmbus_chan_recv(struct vmbus_channel *chan, void *data, uint32_t *len,
223 uint64_t *request_id)
225 struct vmbus_chanpkt_hdr pkt;
226 uint32_t dlen, hlen, bufferlen = *len;
231 error = vmbus_rxbr_peek(&chan->rxbr, &pkt, sizeof(pkt));
235 if (unlikely(pkt.hlen < VMBUS_CHANPKT_HLEN_MIN)) {
236 VMBUS_LOG(ERR, "VMBUS recv, invalid hlen %u", pkt.hlen);
237 /* XXX this channel is dead actually. */
241 if (unlikely(pkt.hlen > pkt.tlen)) {
242 VMBUS_LOG(ERR, "VMBUS recv,invalid hlen %u and tlen %u",
247 /* Length are in quad words */
248 hlen = pkt.hlen << VMBUS_CHANPKT_SIZE_SHIFT;
249 dlen = (pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT) - hlen;
252 /* If caller buffer is not large enough */
253 if (unlikely(dlen > bufferlen))
257 *request_id = pkt.xactid;
259 /* Read data and skip the header */
260 return vmbus_read_and_signal(chan, data, dlen, hlen);
263 int rte_vmbus_chan_recv_raw(struct vmbus_channel *chan,
264 void *data, uint32_t *len)
266 struct vmbus_chanpkt_hdr pkt;
267 uint32_t dlen, bufferlen = *len;
270 error = vmbus_rxbr_peek(&chan->rxbr, &pkt, sizeof(pkt));
274 if (unlikely(pkt.hlen < VMBUS_CHANPKT_HLEN_MIN)) {
275 VMBUS_LOG(ERR, "VMBUS recv, invalid hlen %u", pkt.hlen);
276 /* XXX this channel is dead actually. */
280 if (unlikely(pkt.hlen > pkt.tlen)) {
281 VMBUS_LOG(ERR, "VMBUS recv,invalid hlen %u and tlen %u",
286 /* Length are in quad words */
287 dlen = pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT;
290 /* If caller buffer is not large enough */
291 if (unlikely(dlen > bufferlen))
294 /* Put packet header in data buffer */
295 return vmbus_read_and_signal(chan, data, dlen, 0);
298 int vmbus_chan_create(const struct rte_vmbus_device *device,
299 uint16_t relid, uint16_t subid, uint8_t monitor_id,
300 struct vmbus_channel **new_chan)
302 struct vmbus_channel *chan;
305 chan = rte_zmalloc_socket("VMBUS", sizeof(*chan), RTE_CACHE_LINE_SIZE,
306 device->device.numa_node);
310 STAILQ_INIT(&chan->subchannel_list);
311 chan->device = device;
312 chan->subchannel_id = subid;
314 chan->monitor_id = monitor_id;
317 err = vmbus_uio_map_rings(chan);
326 /* Setup the primary channel */
327 int rte_vmbus_chan_open(struct rte_vmbus_device *device,
328 struct vmbus_channel **new_chan)
332 err = vmbus_chan_create(device, device->relid, 0,
333 device->monitor_id, new_chan);
335 device->primary = *new_chan;
340 int rte_vmbus_max_channels(const struct rte_vmbus_device *device)
342 if (vmbus_uio_subchannels_supported(device, device->primary))
343 return VMBUS_MAX_CHANNELS;
348 /* Setup secondary channel */
349 int rte_vmbus_subchan_open(struct vmbus_channel *primary,
350 struct vmbus_channel **new_chan)
352 struct vmbus_channel *chan;
355 err = vmbus_uio_get_subchan(primary, &chan);
359 STAILQ_INSERT_TAIL(&primary->subchannel_list, chan, next);
364 uint16_t rte_vmbus_sub_channel_index(const struct vmbus_channel *chan)
366 return chan->subchannel_id;
369 void rte_vmbus_chan_close(struct vmbus_channel *chan)
371 const struct rte_vmbus_device *device = chan->device;
372 struct vmbus_channel *primary = device->primary;
375 STAILQ_REMOVE(&primary->subchannel_list, chan,
376 vmbus_channel, next);
381 static void vmbus_dump_ring(FILE *f, const char *id, const struct vmbus_br *br)
383 const struct vmbus_bufring *vbr = br->vbr;
384 struct vmbus_chanpkt_hdr pkt;
386 fprintf(f, "%s windex=%u rindex=%u mask=%u pending=%u feature=%#x\n",
387 id, vbr->windex, vbr->rindex, vbr->imask,
388 vbr->pending_send, vbr->feature_bits.value);
389 fprintf(f, " size=%u avail write=%u read=%u\n",
390 br->dsize, vmbus_br_availwrite(br, vbr->windex),
391 vmbus_br_availread(br));
393 if (vmbus_rxbr_peek(br, &pkt, sizeof(pkt)) == 0)
394 fprintf(f, " pkt type %#x len %u flags %#x xactid %#"PRIx64"\n",
396 pkt.tlen << VMBUS_CHANPKT_SIZE_SHIFT,
397 pkt.flags, pkt.xactid);
400 void rte_vmbus_chan_dump(FILE *f, const struct vmbus_channel *chan)
402 fprintf(f, "channel[%u] relid=%u monitor=%u\n",
403 chan->subchannel_id, chan->relid, chan->monitor_id);
404 vmbus_dump_ring(f, "rxbr", &chan->rxbr);
405 vmbus_dump_ring(f, "txbr", &chan->txbr);