net/qede/base: fix macros to check chip revision/metal
[dpdk.git] / drivers / net / virtio / virtio_rxtx_simple_neon.c
1 /*
2  *   BSD LICENSE
3  *
4  *   Copyright (C) Cavium, Inc. 2016
5  *
6  *   Redistribution and use in source and binary forms, with or without
7  *   modification, are permitted provided that the following conditions
8  *   are met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above copyright
13  *       notice, this list of conditions and the following disclaimer in
14  *       the documentation and/or other materials provided with the
15  *       distribution.
16  *     * Neither the name of Cavium, Inc nor the names of its
17  *       contributors may be used to endorse or promote products derived
18  *       from this software without specific prior written permission.
19  *
20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include <stdint.h>
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <string.h>
37 #include <errno.h>
38
39 #include <rte_byteorder.h>
40 #include <rte_branch_prediction.h>
41 #include <rte_cycles.h>
42 #include <rte_ether.h>
43 #include <rte_ethdev.h>
44 #include <rte_errno.h>
45 #include <rte_memory.h>
46 #include <rte_memzone.h>
47 #include <rte_mempool.h>
48 #include <rte_malloc.h>
49 #include <rte_mbuf.h>
50 #include <rte_prefetch.h>
51 #include <rte_string_fns.h>
52 #include <rte_vect.h>
53
54 #include "virtio_rxtx_simple.h"
55
56 #define RTE_VIRTIO_VPMD_RX_BURST 32
57 #define RTE_VIRTIO_DESC_PER_LOOP 8
58 #define RTE_VIRTIO_VPMD_RX_REARM_THRESH RTE_VIRTIO_VPMD_RX_BURST
59
60 /* virtio vPMD receive routine, only accept(nb_pkts >= RTE_VIRTIO_DESC_PER_LOOP)
61  *
62  * This routine is for non-mergeable RX, one desc for each guest buffer.
63  * This routine is based on the RX ring layout optimization. Each entry in the
64  * avail ring points to the desc with the same index in the desc ring and this
65  * will never be changed in the driver.
66  *
67  * - nb_pkts < RTE_VIRTIO_DESC_PER_LOOP, just return no packet
68  */
69 uint16_t
70 virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
71         uint16_t nb_pkts)
72 {
73         struct virtnet_rx *rxvq = rx_queue;
74         struct virtqueue *vq = rxvq->vq;
75         struct virtio_hw *hw = vq->hw;
76         uint16_t nb_used;
77         uint16_t desc_idx;
78         struct vring_used_elem *rused;
79         struct rte_mbuf **sw_ring;
80         struct rte_mbuf **sw_ring_end;
81         uint16_t nb_pkts_received = 0;
82
83         uint8x16_t shuf_msk1 = {
84                 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */
85                 4, 5, 0xFF, 0xFF,       /* pkt len */
86                 4, 5,                   /* dat len */
87                 0xFF, 0xFF,             /* vlan tci */
88                 0xFF, 0xFF, 0xFF, 0xFF
89         };
90
91         uint8x16_t shuf_msk2 = {
92                 0xFF, 0xFF, 0xFF, 0xFF, /* packet type */
93                 12, 13, 0xFF, 0xFF,     /* pkt len */
94                 12, 13,                 /* dat len */
95                 0xFF, 0xFF,             /* vlan tci */
96                 0xFF, 0xFF, 0xFF, 0xFF
97         };
98
99         /* Subtract the header length.
100          *  In which case do we need the header length in used->len ?
101          */
102         uint16x8_t len_adjust = {
103                 0, 0,
104                 (uint16_t)vq->hw->vtnet_hdr_size, 0,
105                 (uint16_t)vq->hw->vtnet_hdr_size,
106                 0,
107                 0, 0
108         };
109
110         if (unlikely(hw->started == 0))
111                 return nb_pkts_received;
112
113         if (unlikely(nb_pkts < RTE_VIRTIO_DESC_PER_LOOP))
114                 return 0;
115
116         nb_used = VIRTQUEUE_NUSED(vq);
117
118         rte_rmb();
119
120         if (unlikely(nb_used == 0))
121                 return 0;
122
123         nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_VIRTIO_DESC_PER_LOOP);
124         nb_used = RTE_MIN(nb_used, nb_pkts);
125
126         desc_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
127         rused = &vq->vq_ring.used->ring[desc_idx];
128         sw_ring  = &vq->sw_ring[desc_idx];
129         sw_ring_end = &vq->sw_ring[vq->vq_nentries];
130
131         rte_prefetch_non_temporal(rused);
132
133         if (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
134                 virtio_rxq_rearm_vec(rxvq);
135                 if (unlikely(virtqueue_kick_prepare(vq)))
136                         virtqueue_notify(vq);
137         }
138
139         for (nb_pkts_received = 0;
140                 nb_pkts_received < nb_used;) {
141                 uint64x2_t desc[RTE_VIRTIO_DESC_PER_LOOP / 2];
142                 uint64x2_t mbp[RTE_VIRTIO_DESC_PER_LOOP / 2];
143                 uint64x2_t pkt_mb[RTE_VIRTIO_DESC_PER_LOOP];
144
145                 mbp[0] = vld1q_u64((uint64_t *)(sw_ring + 0));
146                 desc[0] = vld1q_u64((uint64_t *)(rused + 0));
147                 vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0]);
148
149                 mbp[1] = vld1q_u64((uint64_t *)(sw_ring + 2));
150                 desc[1] = vld1q_u64((uint64_t *)(rused + 2));
151                 vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1]);
152
153                 mbp[2] = vld1q_u64((uint64_t *)(sw_ring + 4));
154                 desc[2] = vld1q_u64((uint64_t *)(rused + 4));
155                 vst1q_u64((uint64_t *)&rx_pkts[4], mbp[2]);
156
157                 mbp[3] = vld1q_u64((uint64_t *)(sw_ring + 6));
158                 desc[3] = vld1q_u64((uint64_t *)(rused + 6));
159                 vst1q_u64((uint64_t *)&rx_pkts[6], mbp[3]);
160
161                 pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
162                                 vreinterpretq_u8_u64(desc[0]), shuf_msk2));
163                 pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
164                                 vreinterpretq_u8_u64(desc[0]), shuf_msk1));
165                 pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16(
166                                 vreinterpretq_u16_u64(pkt_mb[1]), len_adjust));
167                 pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16(
168                                 vreinterpretq_u16_u64(pkt_mb[0]), len_adjust));
169                 vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1,
170                         pkt_mb[1]);
171                 vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1,
172                         pkt_mb[0]);
173
174                 pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8(
175                                 vreinterpretq_u8_u64(desc[1]), shuf_msk2));
176                 pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
177                                 vreinterpretq_u8_u64(desc[1]), shuf_msk1));
178                 pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16(
179                                 vreinterpretq_u16_u64(pkt_mb[3]), len_adjust));
180                 pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16(
181                                 vreinterpretq_u16_u64(pkt_mb[2]), len_adjust));
182                 vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1,
183                         pkt_mb[3]);
184                 vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1,
185                         pkt_mb[2]);
186
187                 pkt_mb[5] = vreinterpretq_u64_u8(vqtbl1q_u8(
188                                 vreinterpretq_u8_u64(desc[2]), shuf_msk2));
189                 pkt_mb[4] = vreinterpretq_u64_u8(vqtbl1q_u8(
190                                 vreinterpretq_u8_u64(desc[2]), shuf_msk1));
191                 pkt_mb[5] = vreinterpretq_u64_u16(vsubq_u16(
192                                 vreinterpretq_u16_u64(pkt_mb[5]), len_adjust));
193                 pkt_mb[4] = vreinterpretq_u64_u16(vsubq_u16(
194                                 vreinterpretq_u16_u64(pkt_mb[4]), len_adjust));
195                 vst1q_u64((void *)&rx_pkts[5]->rx_descriptor_fields1,
196                         pkt_mb[5]);
197                 vst1q_u64((void *)&rx_pkts[4]->rx_descriptor_fields1,
198                         pkt_mb[4]);
199
200                 pkt_mb[7] = vreinterpretq_u64_u8(vqtbl1q_u8(
201                                 vreinterpretq_u8_u64(desc[3]), shuf_msk2));
202                 pkt_mb[6] = vreinterpretq_u64_u8(vqtbl1q_u8(
203                                 vreinterpretq_u8_u64(desc[3]), shuf_msk1));
204                 pkt_mb[7] = vreinterpretq_u64_u16(vsubq_u16(
205                                 vreinterpretq_u16_u64(pkt_mb[7]), len_adjust));
206                 pkt_mb[6] = vreinterpretq_u64_u16(vsubq_u16(
207                                 vreinterpretq_u16_u64(pkt_mb[6]), len_adjust));
208                 vst1q_u64((void *)&rx_pkts[7]->rx_descriptor_fields1,
209                         pkt_mb[7]);
210                 vst1q_u64((void *)&rx_pkts[6]->rx_descriptor_fields1,
211                         pkt_mb[6]);
212
213                 if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) {
214                         if (sw_ring + nb_used <= sw_ring_end)
215                                 nb_pkts_received += nb_used;
216                         else
217                                 nb_pkts_received += sw_ring_end - sw_ring;
218                         break;
219                 } else {
220                         if (unlikely(sw_ring + RTE_VIRTIO_DESC_PER_LOOP >=
221                                 sw_ring_end)) {
222                                 nb_pkts_received += sw_ring_end - sw_ring;
223                                 break;
224                         } else {
225                                 nb_pkts_received += RTE_VIRTIO_DESC_PER_LOOP;
226
227                                 rx_pkts += RTE_VIRTIO_DESC_PER_LOOP;
228                                 sw_ring += RTE_VIRTIO_DESC_PER_LOOP;
229                                 rused   += RTE_VIRTIO_DESC_PER_LOOP;
230                                 nb_used -= RTE_VIRTIO_DESC_PER_LOOP;
231                         }
232                 }
233         }
234
235         vq->vq_used_cons_idx += nb_pkts_received;
236         vq->vq_free_cnt += nb_pkts_received;
237         rxvq->stats.packets += nb_pkts_received;
238         return nb_pkts_received;
239 }