X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fvirtio%2Fvirtqueue.h;h=ca9d8e6e3ca705e9bdda31047bf6d58ce8eb739c;hb=4b058fdd27ceac277aa715df6f6bb0f7b37b150b;hp=8c46a83d560ccdc4080b03e13b787541eb7f8ccf;hpb=e928fd0bb0118387f04811970b11504b199b5ed1;p=dpdk.git diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h index 8c46a83d56..ca9d8e6e3c 100644 --- a/drivers/net/virtio/virtqueue.h +++ b/drivers/net/virtio/virtqueue.h @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation */ #ifndef _VIRTQUEUE_H_ @@ -38,25 +9,50 @@ #include #include -#include #include #include "virtio_pci.h" #include "virtio_ring.h" #include "virtio_logs.h" +#include "virtio_rxtx.h" struct rte_mbuf; /* - * Per virtio_config.h in Linux. + * Per virtio_ring.h in Linux. * For virtio_pci on SMP, we don't need to order with respect to MMIO * accesses through relaxed memory I/O windows, so smp_mb() et al are * sufficient. * + * For using virtio to talk to real devices (eg. vDPA) we do need real + * barriers. */ -#define virtio_mb() rte_smp_mb() -#define virtio_rmb() rte_smp_rmb() -#define virtio_wmb() rte_smp_wmb() +static inline void +virtio_mb(uint8_t weak_barriers) +{ + if (weak_barriers) + rte_smp_mb(); + else + rte_mb(); +} + +static inline void +virtio_rmb(uint8_t weak_barriers) +{ + if (weak_barriers) + rte_smp_rmb(); + else + rte_cio_rmb(); +} + +static inline void +virtio_wmb(uint8_t weak_barriers) +{ + if (weak_barriers) + rte_smp_wmb(); + else + rte_cio_wmb(); +} #ifdef RTE_PMD_PACKET_PREFETCH #define rte_packet_prefetch(p) rte_prefetch1(p) @@ -66,6 +62,29 @@ struct rte_mbuf; #define VIRTQUEUE_MAX_NAME_SZ 32 +#ifdef RTE_VIRTIO_USER +/** + * Return the physical address (or virtual address in case of + * virtio-user) of mbuf data buffer. + * + * The address is firstly casted to the word size (sizeof(uintptr_t)) + * before casting it to uint64_t. This is to make it work with different + * combination of word size (64 bit and 32 bit) and virtio device + * (virtio-pci and virtio-user). + */ +#define VIRTIO_MBUF_ADDR(mb, vq) \ + ((uint64_t)(*(uintptr_t *)((uintptr_t)(mb) + (vq)->offset))) +#else +#define VIRTIO_MBUF_ADDR(mb, vq) ((mb)->buf_iova) +#endif + +/** + * Return the physical address (or virtual address in case of + * virtio-user) of mbuf data buffer, taking care of mbuf data offset + */ +#define VIRTIO_MBUF_DATA_DMA_ADDR(mb, vq) \ + (VIRTIO_MBUF_ADDR(mb, vq) + (mb)->data_off) + #define VTNET_SQ_RQ_QUEUE_IDX 0 #define VTNET_SQ_TQ_QUEUE_IDX 1 #define VTNET_SQ_CQ_QUEUE_IDX 2 @@ -119,8 +138,8 @@ struct virtio_net_ctrl_mac { } __attribute__((__packed__)); #define VIRTIO_NET_CTRL_MAC 1 - #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 - #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 +#define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 +#define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 /** * Control VLAN filtering @@ -135,6 +154,17 @@ struct virtio_net_ctrl_mac { #define VIRTIO_NET_CTRL_VLAN_ADD 0 #define VIRTIO_NET_CTRL_VLAN_DEL 1 +/* + * Control link announce acknowledgement + * + * The command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that + * driver has recevied the notification; device would clear the + * VIRTIO_NET_S_ANNOUNCE bit in the status field after it receives + * this command. + */ +#define VIRTIO_NET_CTRL_ANNOUNCE 3 +#define VIRTIO_NET_CTRL_ANNOUNCE_ACK 0 + struct virtio_net_ctrl_hdr { uint8_t class; uint8_t cmd; @@ -153,23 +183,42 @@ struct virtio_pmd_ctrl { uint8_t data[VIRTIO_MAX_CTRL_DATA]; }; +struct vq_desc_extra { + void *cookie; + uint16_t ndescs; + uint16_t next; +}; + struct virtqueue { - struct virtio_hw *hw; /**< virtio_hw structure pointer. */ - const struct rte_memzone *mz; /**< mem zone to populate RX ring. */ - const struct rte_memzone *virtio_net_hdr_mz; /**< memzone to populate hdr. */ - struct rte_mempool *mpool; /**< mempool for mbuf allocation */ - uint16_t queue_id; /**< DPDK queue index. */ - uint8_t port_id; /**< Device port identifier. */ - uint16_t vq_queue_index; /**< PCI queue index */ - - void *vq_ring_virt_mem; /**< linear address of vring*/ + struct virtio_hw *hw; /**< virtio_hw structure pointer. */ + struct vring vq_ring; /**< vring keeping desc, used and avail */ + struct vring_packed ring_packed; /**< vring keeping descs */ + bool avail_wrap_counter; + bool used_wrap_counter; + uint16_t event_flags_shadow; + uint16_t avail_used_flags; + /** + * Last consumed descriptor in the used table, + * trails vq_ring.used->idx. + */ + uint16_t vq_used_cons_idx; + uint16_t vq_nentries; /**< vring desc numbers */ + uint16_t vq_free_cnt; /**< num of desc available */ + uint16_t vq_avail_idx; /**< sync until needed */ + uint16_t vq_free_thresh; /**< free threshold */ + + void *vq_ring_virt_mem; /**< linear address of vring*/ unsigned int vq_ring_size; - phys_addr_t vq_ring_mem; /**< physical address of vring */ - struct vring vq_ring; /**< vring keeping desc, used and avail */ - uint16_t vq_free_cnt; /**< num of desc available */ - uint16_t vq_nentries; /**< vring desc numbers */ - uint16_t vq_free_thresh; /**< free threshold */ + union { + struct virtnet_rx rxq; + struct virtnet_tx txq; + struct virtnet_ctl cq; + }; + + rte_iova_t vq_ring_mem; /**< physical address of vring, + * or virtual address for virtio_user. */ + /** * Head of the free chain in the descriptor table. If * there are no free descriptors, this will be set to @@ -177,34 +226,11 @@ struct virtqueue { */ uint16_t vq_desc_head_idx; uint16_t vq_desc_tail_idx; - /** - * Last consumed descriptor in the used table, - * trails vq_ring.used->idx. - */ - uint16_t vq_used_cons_idx; - uint16_t vq_avail_idx; - uint64_t mbuf_initializer; /**< value to init mbufs. */ - phys_addr_t virtio_net_hdr_mem; /**< hdr for each xmit packet */ - - struct rte_mbuf **sw_ring; /**< RX software ring. */ - /* dummy mbuf, for wraparound when processing RX ring. */ - struct rte_mbuf fake_mbuf; - - /* Statistics */ - uint64_t packets; - uint64_t bytes; - uint64_t errors; - uint64_t multicast; - uint64_t broadcast; - /* Size bins in array as RFC 2819, undersized [0], 64 [1], etc */ - uint64_t size_bins[8]; - - uint16_t *notify_addr; - - struct vq_desc_extra { - void *cookie; - uint16_t ndescs; - } vq_descx[0]; + uint16_t vq_queue_index; /**< PCI queue index */ + uint16_t offset; /**< relative offset to obtain addr in mbuf */ + uint16_t *notify_addr; + struct rte_mbuf **sw_ring; /**< RX software ring. */ + struct vq_desc_extra vq_descx[0]; }; /* If multiqueue is provided by host, then we suppport it. */ @@ -213,14 +239,13 @@ struct virtqueue { #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 -#define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 - /** * This is the first element of the scatter-gather list. If you don't * specify GSO or CSUM features, you can simply ignore the header. */ struct virtio_net_hdr { #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /**< Use csum_start,csum_offset*/ +#define VIRTIO_NET_HDR_F_DATA_VALID 2 /**< Checksum is valid */ uint8_t flags; #define VIRTIO_NET_HDR_GSO_NONE 0 /**< Not a GSO frame */ #define VIRTIO_NET_HDR_GSO_TCPV4 1 /**< GSO frame, IPv4 TCP (TSO) */ @@ -247,13 +272,41 @@ struct virtio_net_hdr_mrg_rxbuf { #define VIRTIO_MAX_TX_INDIRECT 8 struct virtio_tx_region { struct virtio_net_hdr_mrg_rxbuf tx_hdr; - struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT] - __attribute__((__aligned__(16))); + union { + struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT] + __attribute__((__aligned__(16))); + struct vring_packed_desc tx_indir_pq[VIRTIO_MAX_TX_INDIRECT] + __attribute__((__aligned__(16))); + }; }; +static inline int +desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq) +{ + uint16_t used, avail, flags; + + flags = desc->flags; + used = !!(flags & VRING_DESC_F_USED(1)); + avail = !!(flags & VRING_DESC_F_AVAIL(1)); + + return avail == used && used == vq->used_wrap_counter; +} + +static inline void +vring_desc_init_packed(struct virtqueue *vq, int n) +{ + int i; + for (i = 0; i < n - 1; i++) { + vq->ring_packed.desc_packed[i].id = i; + vq->vq_descx[i].next = i + 1; + } + vq->ring_packed.desc_packed[i].id = i; + vq->vq_descx[i].next = VQ_RING_DESC_CHAIN_END; +} + /* Chain all the descriptors in the ring with an END */ static inline void -vring_desc_init(struct vring_desc *dp, uint16_t n) +vring_desc_init_split(struct vring_desc *dp, uint16_t n) { uint16_t i; @@ -265,7 +318,63 @@ vring_desc_init(struct vring_desc *dp, uint16_t n) /** * Tell the backend not to interrupt us. */ -void virtqueue_disable_intr(struct virtqueue *vq); +static inline void +virtqueue_disable_intr_packed(struct virtqueue *vq) +{ + uint16_t *event_flags = &vq->ring_packed.driver_event->desc_event_flags; + + *event_flags = RING_EVENT_FLAGS_DISABLE; +} + + +/** + * Tell the backend not to interrupt us. + */ +static inline void +virtqueue_disable_intr(struct virtqueue *vq) +{ + if (vtpci_packed_queue(vq->hw)) + virtqueue_disable_intr_packed(vq); + else + vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; +} + +/** + * Tell the backend to interrupt. Implementation for packed virtqueues. + */ +static inline void +virtqueue_enable_intr_packed(struct virtqueue *vq) +{ + uint16_t *event_flags = &vq->ring_packed.driver_event->desc_event_flags; + + if (vq->event_flags_shadow == RING_EVENT_FLAGS_DISABLE) { + virtio_wmb(vq->hw->weak_barriers); + vq->event_flags_shadow = RING_EVENT_FLAGS_ENABLE; + *event_flags = vq->event_flags_shadow; + } +} + +/** + * Tell the backend to interrupt. Implementation for split virtqueues. + */ +static inline void +virtqueue_enable_intr_split(struct virtqueue *vq) +{ + vq->vq_ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT); +} + +/** + * Tell the backend to interrupt us. + */ +static inline void +virtqueue_enable_intr(struct virtqueue *vq) +{ + if (vtpci_packed_queue(vq->hw)) + virtqueue_enable_intr_packed(vq); + else + virtqueue_enable_intr_split(vq); +} + /** * Dump virtqueue internal structures, for debug purpose only. */ @@ -273,7 +382,10 @@ void virtqueue_dump(struct virtqueue *vq); /** * Get all mbufs to be freed. */ -struct rte_mbuf *virtqueue_detatch_unused(struct virtqueue *vq); +struct rte_mbuf *virtqueue_detach_unused(struct virtqueue *vq); + +/* Flush the elements in the used ring. */ +void virtqueue_rxvq_flush(struct virtqueue *vq); static inline int virtqueue_full(const struct virtqueue *vq) @@ -281,12 +393,28 @@ virtqueue_full(const struct virtqueue *vq) return vq->vq_free_cnt == 0; } +static inline int +virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx) +{ + if (vtpci_queue_idx == hw->max_queue_pairs * 2) + return VTNET_CQ; + else if (vtpci_queue_idx % 2 == 0) + return VTNET_RQ; + else + return VTNET_TQ; +} + #define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) +void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx); +void vq_ring_free_chain_packed(struct virtqueue *vq, uint16_t used_idx); +void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, + uint16_t num); + static inline void vq_update_avail_idx(struct virtqueue *vq) { - virtio_wmb(); + virtio_wmb(vq->hw->weak_barriers); vq->vq_ring.avail->idx = vq->vq_avail_idx; } @@ -310,18 +438,36 @@ vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx) static inline int virtqueue_kick_prepare(struct virtqueue *vq) { + /* + * Ensure updated avail->idx is visible to vhost before reading + * the used->flags. + */ + virtio_mb(vq->hw->weak_barriers); return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY); } -static inline void -virtqueue_notify(struct virtqueue *vq) +static inline int +virtqueue_kick_prepare_packed(struct virtqueue *vq) { + uint16_t flags; + /* - * Ensure updated avail->idx is visible to host. - * For virtio on IA, the notificaiton is through io port operation - * which is a serialization instruction itself. + * Ensure updated data is visible to vhost before reading the flags. */ - vq->hw->vtpci_ops->notify_queue(vq->hw, vq); + virtio_mb(vq->hw->weak_barriers); + flags = vq->ring_packed.device_event->desc_event_flags; + + return flags != RING_EVENT_FLAGS_DISABLE; +} + +/* + * virtqueue_kick_prepare*() or the virtio_wmb() should be called + * before this function to be sure that all the data is visible to vhost. + */ +static inline void +virtqueue_notify(struct virtqueue *vq) +{ + VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq); } #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP @@ -329,6 +475,15 @@ virtqueue_notify(struct virtqueue *vq) uint16_t used_idx, nused; \ used_idx = (vq)->vq_ring.used->idx; \ nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \ + if (vtpci_packed_queue((vq)->hw)) { \ + PMD_INIT_LOG(DEBUG, \ + "VQ: - size=%d; free=%d; used_cons_idx=%d; avail_idx=%d;" \ + "VQ: - avail_wrap_counter=%d; used_wrap_counter=%d", \ + (vq)->vq_nentries, (vq)->vq_free_cnt, (vq)->vq_used_cons_idx, \ + (vq)->vq_avail_idx, (vq)->avail_wrap_counter, \ + (vq)->used_wrap_counter); \ + break; \ + } \ PMD_INIT_LOG(DEBUG, \ "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \ " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \