net/qede/base: fix macros to check chip revision/metal
[dpdk.git] / drivers / net / thunderx / nicvf_rxtx.c
1 /*
2  *   BSD LICENSE
3  *
4  *   Copyright (C) Cavium, Inc. 2016.
5  *
6  *   Redistribution and use in source and binary forms, with or without
7  *   modification, are permitted provided that the following conditions
8  *   are met:
9  *
10  *     * Redistributions of source code must retain the above copyright
11  *       notice, this list of conditions and the following disclaimer.
12  *     * Redistributions in binary form must reproduce the above copyright
13  *       notice, this list of conditions and the following disclaimer in
14  *       the documentation and/or other materials provided with the
15  *       distribution.
16  *     * Neither the name of Cavium, Inc nor the names of its
17  *       contributors may be used to endorse or promote products derived
18  *       from this software without specific prior written permission.
19  *
20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31  */
32
33 #include <unistd.h>
34 #include <stdint.h>
35 #include <stdio.h>
36 #include <stdlib.h>
37
38 #include <rte_atomic.h>
39 #include <rte_branch_prediction.h>
40 #include <rte_byteorder.h>
41 #include <rte_common.h>
42 #include <rte_cycles.h>
43 #include <rte_errno.h>
44 #include <rte_ethdev.h>
45 #include <rte_ether.h>
46 #include <rte_log.h>
47 #include <rte_mbuf.h>
48 #include <rte_prefetch.h>
49
50 #include "base/nicvf_plat.h"
51
52 #include "nicvf_ethdev.h"
53 #include "nicvf_rxtx.h"
54 #include "nicvf_logs.h"
55
56 static inline void __hot
57 fill_sq_desc_header(union sq_entry_t *entry, struct rte_mbuf *pkt)
58 {
59         /* Local variable sqe to avoid read from sq desc memory*/
60         union sq_entry_t sqe;
61         uint64_t ol_flags;
62
63         /* Fill SQ header descriptor */
64         sqe.buff[0] = 0;
65         sqe.hdr.subdesc_type = SQ_DESC_TYPE_HEADER;
66         /* Number of sub-descriptors following this one */
67         sqe.hdr.subdesc_cnt = pkt->nb_segs;
68         sqe.hdr.tot_len = pkt->pkt_len;
69
70         ol_flags = pkt->ol_flags & NICVF_TX_OFFLOAD_MASK;
71         if (unlikely(ol_flags)) {
72                 /* L4 cksum */
73                 uint64_t l4_flags = ol_flags & PKT_TX_L4_MASK;
74                 if (l4_flags == PKT_TX_TCP_CKSUM)
75                         sqe.hdr.csum_l4 = SEND_L4_CSUM_TCP;
76                 else if (l4_flags == PKT_TX_UDP_CKSUM)
77                         sqe.hdr.csum_l4 = SEND_L4_CSUM_UDP;
78                 else
79                         sqe.hdr.csum_l4 = SEND_L4_CSUM_DISABLE;
80
81                 sqe.hdr.l3_offset = pkt->l2_len;
82                 sqe.hdr.l4_offset = pkt->l3_len + pkt->l2_len;
83
84                 /* L3 cksum */
85                 if (ol_flags & PKT_TX_IP_CKSUM)
86                         sqe.hdr.csum_l3 = 1;
87         }
88
89         entry->buff[0] = sqe.buff[0];
90 }
91
92 void __hot
93 nicvf_single_pool_free_xmited_buffers(struct nicvf_txq *sq)
94 {
95         int j = 0;
96         uint32_t curr_head;
97         uint32_t head = sq->head;
98         struct rte_mbuf **txbuffs = sq->txbuffs;
99         void *obj_p[NICVF_MAX_TX_FREE_THRESH] __rte_cache_aligned;
100
101         curr_head = nicvf_addr_read(sq->sq_head) >> 4;
102         while (head != curr_head) {
103                 if (txbuffs[head])
104                         obj_p[j++] = txbuffs[head];
105
106                 head = (head + 1) & sq->qlen_mask;
107         }
108
109         rte_mempool_put_bulk(sq->pool, obj_p, j);
110         sq->head = curr_head;
111         sq->xmit_bufs -= j;
112         NICVF_TX_ASSERT(sq->xmit_bufs >= 0);
113 }
114
115 void __hot
116 nicvf_multi_pool_free_xmited_buffers(struct nicvf_txq *sq)
117 {
118         uint32_t n = 0;
119         uint32_t curr_head;
120         uint32_t head = sq->head;
121         struct rte_mbuf **txbuffs = sq->txbuffs;
122
123         curr_head = nicvf_addr_read(sq->sq_head) >> 4;
124         while (head != curr_head) {
125                 if (txbuffs[head]) {
126                         rte_pktmbuf_free_seg(txbuffs[head]);
127                         n++;
128                 }
129
130                 head = (head + 1) & sq->qlen_mask;
131         }
132
133         sq->head = curr_head;
134         sq->xmit_bufs -= n;
135         NICVF_TX_ASSERT(sq->xmit_bufs >= 0);
136 }
137
138 static inline uint32_t __hot
139 nicvf_free_tx_desc(struct nicvf_txq *sq)
140 {
141         return ((sq->head - sq->tail - 1) & sq->qlen_mask);
142 }
143
144 /* Send Header + Packet */
145 #define TX_DESC_PER_PKT 2
146
147 static inline uint32_t __hot
148 nicvf_free_xmitted_buffers(struct nicvf_txq *sq, struct rte_mbuf **tx_pkts,
149                             uint16_t nb_pkts)
150 {
151         uint32_t free_desc = nicvf_free_tx_desc(sq);
152
153         if (free_desc < nb_pkts * TX_DESC_PER_PKT ||
154                         sq->xmit_bufs > sq->tx_free_thresh) {
155                 if (unlikely(sq->pool == NULL))
156                         sq->pool = tx_pkts[0]->pool;
157
158                 sq->pool_free(sq);
159                 /* Freed now, let see the number of free descs again */
160                 free_desc = nicvf_free_tx_desc(sq);
161         }
162         return free_desc;
163 }
164
165 uint16_t __hot
166 nicvf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
167 {
168         int i;
169         uint32_t free_desc;
170         uint32_t tail;
171         struct nicvf_txq *sq = tx_queue;
172         union sq_entry_t *desc_ptr = sq->desc;
173         struct rte_mbuf **txbuffs = sq->txbuffs;
174         struct rte_mbuf *pkt;
175         uint32_t qlen_mask = sq->qlen_mask;
176
177         tail = sq->tail;
178         free_desc = nicvf_free_xmitted_buffers(sq, tx_pkts, nb_pkts);
179
180         for (i = 0; i < nb_pkts && (int)free_desc >= TX_DESC_PER_PKT; i++) {
181                 pkt = tx_pkts[i];
182
183                 txbuffs[tail] = NULL;
184                 fill_sq_desc_header(desc_ptr + tail, pkt);
185                 tail = (tail + 1) & qlen_mask;
186
187                 txbuffs[tail] = pkt;
188                 fill_sq_desc_gather(desc_ptr + tail, pkt);
189                 tail = (tail + 1) & qlen_mask;
190                 free_desc -= TX_DESC_PER_PKT;
191         }
192
193         sq->tail = tail;
194         sq->xmit_bufs += i;
195         rte_wmb();
196
197         /* Inform HW to xmit the packets */
198         nicvf_addr_write(sq->sq_door, i * TX_DESC_PER_PKT);
199         return i;
200 }
201
202 uint16_t __hot
203 nicvf_xmit_pkts_multiseg(void *tx_queue, struct rte_mbuf **tx_pkts,
204                          uint16_t nb_pkts)
205 {
206         int i, k;
207         uint32_t used_desc, next_used_desc, used_bufs, free_desc, tail;
208         struct nicvf_txq *sq = tx_queue;
209         union sq_entry_t *desc_ptr = sq->desc;
210         struct rte_mbuf **txbuffs = sq->txbuffs;
211         struct rte_mbuf *pkt, *seg;
212         uint32_t qlen_mask = sq->qlen_mask;
213         uint16_t nb_segs;
214
215         tail = sq->tail;
216         used_desc = 0;
217         used_bufs = 0;
218
219         free_desc = nicvf_free_xmitted_buffers(sq, tx_pkts, nb_pkts);
220
221         for (i = 0; i < nb_pkts; i++) {
222                 pkt = tx_pkts[i];
223
224                 nb_segs = pkt->nb_segs;
225
226                 next_used_desc = used_desc + nb_segs + 1;
227                 if (next_used_desc > free_desc)
228                         break;
229                 used_desc = next_used_desc;
230                 used_bufs += nb_segs;
231
232                 txbuffs[tail] = NULL;
233                 fill_sq_desc_header(desc_ptr + tail, pkt);
234                 tail = (tail + 1) & qlen_mask;
235
236                 txbuffs[tail] = pkt;
237                 fill_sq_desc_gather(desc_ptr + tail, pkt);
238                 tail = (tail + 1) & qlen_mask;
239
240                 seg = pkt->next;
241                 for (k = 1; k < nb_segs; k++) {
242                         txbuffs[tail] = seg;
243                         fill_sq_desc_gather(desc_ptr + tail, seg);
244                         tail = (tail + 1) & qlen_mask;
245                         seg = seg->next;
246                 }
247         }
248
249         sq->tail = tail;
250         sq->xmit_bufs += used_bufs;
251         rte_wmb();
252
253         /* Inform HW to xmit the packets */
254         nicvf_addr_write(sq->sq_door, used_desc);
255         return nb_pkts;
256 }
257
258 static const uint32_t ptype_table[16][16] __rte_cache_aligned = {
259         [L3_NONE][L4_NONE] = RTE_PTYPE_UNKNOWN,
260         [L3_NONE][L4_IPSEC_ESP] = RTE_PTYPE_UNKNOWN,
261         [L3_NONE][L4_IPFRAG] = RTE_PTYPE_L4_FRAG,
262         [L3_NONE][L4_IPCOMP] = RTE_PTYPE_UNKNOWN,
263         [L3_NONE][L4_TCP] = RTE_PTYPE_L4_TCP,
264         [L3_NONE][L4_UDP_PASS1] = RTE_PTYPE_L4_UDP,
265         [L3_NONE][L4_GRE] = RTE_PTYPE_TUNNEL_GRE,
266         [L3_NONE][L4_UDP_PASS2] = RTE_PTYPE_L4_UDP,
267         [L3_NONE][L4_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE,
268         [L3_NONE][L4_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
269         [L3_NONE][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE,
270
271         [L3_IPV4][L4_NONE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN,
272         [L3_IPV4][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV4,
273         [L3_IPV4][L4_IPFRAG] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG,
274         [L3_IPV4][L4_IPCOMP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN,
275         [L3_IPV4][L4_TCP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP,
276         [L3_IPV4][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP,
277         [L3_IPV4][L4_GRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GRE,
278         [L3_IPV4][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP,
279         [L3_IPV4][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GENEVE,
280         [L3_IPV4][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_VXLAN,
281         [L3_IPV4][L4_NVGRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE,
282
283         [L3_IPV4_OPT][L4_NONE] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN,
284         [L3_IPV4_OPT][L4_IPSEC_ESP] =  RTE_PTYPE_L3_IPV4_EXT |
285                                 RTE_PTYPE_L3_IPV4,
286         [L3_IPV4_OPT][L4_IPFRAG] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_FRAG,
287         [L3_IPV4_OPT][L4_IPCOMP] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN,
288         [L3_IPV4_OPT][L4_TCP] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP,
289         [L3_IPV4_OPT][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP,
290         [L3_IPV4_OPT][L4_GRE] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GRE,
291         [L3_IPV4_OPT][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP,
292         [L3_IPV4_OPT][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV4_EXT |
293                                 RTE_PTYPE_TUNNEL_GENEVE,
294         [L3_IPV4_OPT][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV4_EXT |
295                                 RTE_PTYPE_TUNNEL_VXLAN,
296         [L3_IPV4_OPT][L4_NVGRE] = RTE_PTYPE_L3_IPV4_EXT |
297                                 RTE_PTYPE_TUNNEL_NVGRE,
298
299         [L3_IPV6][L4_NONE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN,
300         [L3_IPV6][L4_IPSEC_ESP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L3_IPV4,
301         [L3_IPV6][L4_IPFRAG] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG,
302         [L3_IPV6][L4_IPCOMP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN,
303         [L3_IPV6][L4_TCP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP,
304         [L3_IPV6][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP,
305         [L3_IPV6][L4_GRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GRE,
306         [L3_IPV6][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP,
307         [L3_IPV6][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GENEVE,
308         [L3_IPV6][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_VXLAN,
309         [L3_IPV6][L4_NVGRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_NVGRE,
310
311         [L3_IPV6_OPT][L4_NONE] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN,
312         [L3_IPV6_OPT][L4_IPSEC_ESP] =  RTE_PTYPE_L3_IPV6_EXT |
313                                         RTE_PTYPE_L3_IPV4,
314         [L3_IPV6_OPT][L4_IPFRAG] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_FRAG,
315         [L3_IPV6_OPT][L4_IPCOMP] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN,
316         [L3_IPV6_OPT][L4_TCP] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP,
317         [L3_IPV6_OPT][L4_UDP_PASS1] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP,
318         [L3_IPV6_OPT][L4_GRE] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GRE,
319         [L3_IPV6_OPT][L4_UDP_PASS2] = RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP,
320         [L3_IPV6_OPT][L4_UDP_GENEVE] = RTE_PTYPE_L3_IPV6_EXT |
321                                         RTE_PTYPE_TUNNEL_GENEVE,
322         [L3_IPV6_OPT][L4_UDP_VXLAN] = RTE_PTYPE_L3_IPV6_EXT |
323                                         RTE_PTYPE_TUNNEL_VXLAN,
324         [L3_IPV6_OPT][L4_NVGRE] = RTE_PTYPE_L3_IPV6_EXT |
325                                         RTE_PTYPE_TUNNEL_NVGRE,
326
327         [L3_ET_STOP][L4_NONE] = RTE_PTYPE_UNKNOWN,
328         [L3_ET_STOP][L4_IPSEC_ESP] = RTE_PTYPE_UNKNOWN,
329         [L3_ET_STOP][L4_IPFRAG] = RTE_PTYPE_L4_FRAG,
330         [L3_ET_STOP][L4_IPCOMP] = RTE_PTYPE_UNKNOWN,
331         [L3_ET_STOP][L4_TCP] = RTE_PTYPE_L4_TCP,
332         [L3_ET_STOP][L4_UDP_PASS1] = RTE_PTYPE_L4_UDP,
333         [L3_ET_STOP][L4_GRE] = RTE_PTYPE_TUNNEL_GRE,
334         [L3_ET_STOP][L4_UDP_PASS2] = RTE_PTYPE_L4_UDP,
335         [L3_ET_STOP][L4_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE,
336         [L3_ET_STOP][L4_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
337         [L3_ET_STOP][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE,
338
339         [L3_OTHER][L4_NONE] = RTE_PTYPE_UNKNOWN,
340         [L3_OTHER][L4_IPSEC_ESP] = RTE_PTYPE_UNKNOWN,
341         [L3_OTHER][L4_IPFRAG] = RTE_PTYPE_L4_FRAG,
342         [L3_OTHER][L4_IPCOMP] = RTE_PTYPE_UNKNOWN,
343         [L3_OTHER][L4_TCP] = RTE_PTYPE_L4_TCP,
344         [L3_OTHER][L4_UDP_PASS1] = RTE_PTYPE_L4_UDP,
345         [L3_OTHER][L4_GRE] = RTE_PTYPE_TUNNEL_GRE,
346         [L3_OTHER][L4_UDP_PASS2] = RTE_PTYPE_L4_UDP,
347         [L3_OTHER][L4_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE,
348         [L3_OTHER][L4_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
349         [L3_OTHER][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE,
350 };
351
352 static inline uint32_t __hot
353 nicvf_rx_classify_pkt(cqe_rx_word0_t cqe_rx_w0)
354 {
355         return ptype_table[cqe_rx_w0.l3_type][cqe_rx_w0.l4_type];
356 }
357
358 static inline int __hot
359 nicvf_fill_rbdr(struct nicvf_rxq *rxq, int to_fill)
360 {
361         int i;
362         uint32_t ltail, next_tail;
363         struct nicvf_rbdr *rbdr = rxq->shared_rbdr;
364         uint64_t mbuf_phys_off = rxq->mbuf_phys_off;
365         struct rbdr_entry_t *desc = rbdr->desc;
366         uint32_t qlen_mask = rbdr->qlen_mask;
367         uintptr_t door = rbdr->rbdr_door;
368         void *obj_p[NICVF_MAX_RX_FREE_THRESH] __rte_cache_aligned;
369
370         if (unlikely(rte_mempool_get_bulk(rxq->pool, obj_p, to_fill) < 0)) {
371                 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
372                         to_fill;
373                 return 0;
374         }
375
376         NICVF_RX_ASSERT((unsigned int)to_fill <= (qlen_mask -
377                 (nicvf_addr_read(rbdr->rbdr_status) & NICVF_RBDR_COUNT_MASK)));
378
379         next_tail = __atomic_fetch_add(&rbdr->next_tail, to_fill,
380                                         __ATOMIC_ACQUIRE);
381         ltail = next_tail;
382         for (i = 0; i < to_fill; i++) {
383                 struct rbdr_entry_t *entry = desc + (ltail & qlen_mask);
384
385                 entry->full_addr = nicvf_mbuff_virt2phy((uintptr_t)obj_p[i],
386                                                         mbuf_phys_off);
387                 ltail++;
388         }
389
390         while (__atomic_load_n(&rbdr->tail, __ATOMIC_RELAXED) != next_tail)
391                 rte_pause();
392
393         __atomic_store_n(&rbdr->tail, ltail, __ATOMIC_RELEASE);
394         nicvf_addr_write(door, to_fill);
395         return to_fill;
396 }
397
398 static inline int32_t __hot
399 nicvf_rx_pkts_to_process(struct nicvf_rxq *rxq, uint16_t nb_pkts,
400                          int32_t available_space)
401 {
402         if (unlikely(available_space < nb_pkts))
403                 rxq->available_space = nicvf_addr_read(rxq->cq_status)
404                                                 & NICVF_CQ_CQE_COUNT_MASK;
405
406         return RTE_MIN(nb_pkts, available_space);
407 }
408
409 static inline void __hot
410 nicvf_rx_offload(cqe_rx_word0_t cqe_rx_w0, cqe_rx_word2_t cqe_rx_w2,
411                  struct rte_mbuf *pkt)
412 {
413         if (likely(cqe_rx_w0.rss_alg)) {
414                 pkt->hash.rss = cqe_rx_w2.rss_tag;
415                 pkt->ol_flags |= PKT_RX_RSS_HASH;
416         }
417 }
418
419 uint16_t __hot
420 nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
421 {
422         uint32_t i, to_process;
423         struct cqe_rx_t *cqe_rx;
424         struct rte_mbuf *pkt;
425         cqe_rx_word0_t cqe_rx_w0;
426         cqe_rx_word1_t cqe_rx_w1;
427         cqe_rx_word2_t cqe_rx_w2;
428         cqe_rx_word3_t cqe_rx_w3;
429         struct nicvf_rxq *rxq = rx_queue;
430         union cq_entry_t *desc = rxq->desc;
431         const uint64_t cqe_mask = rxq->qlen_mask;
432         uint64_t rb0_ptr, mbuf_phys_off = rxq->mbuf_phys_off;
433         const uint64_t mbuf_init = rxq->mbuf_initializer.value;
434         uint32_t cqe_head = rxq->head & cqe_mask;
435         int32_t available_space = rxq->available_space;
436         const uint8_t rbptr_offset = rxq->rbptr_offset;
437
438         to_process = nicvf_rx_pkts_to_process(rxq, nb_pkts, available_space);
439
440         for (i = 0; i < to_process; i++) {
441                 rte_prefetch_non_temporal(&desc[cqe_head + 2]);
442                 cqe_rx = (struct cqe_rx_t *)&desc[cqe_head];
443                 NICVF_RX_ASSERT(((struct cq_entry_type_t *)cqe_rx)->cqe_type
444                                                  == CQE_TYPE_RX);
445
446                 NICVF_LOAD_PAIR(cqe_rx_w0.u64, cqe_rx_w1.u64, cqe_rx);
447                 NICVF_LOAD_PAIR(cqe_rx_w2.u64, cqe_rx_w3.u64, &cqe_rx->word2);
448                 rb0_ptr = *((uint64_t *)cqe_rx + rbptr_offset);
449                 pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt
450                                 (rb0_ptr - cqe_rx_w1.align_pad, mbuf_phys_off);
451                 pkt->ol_flags = 0;
452                 pkt->data_len = cqe_rx_w3.rb0_sz;
453                 pkt->pkt_len = cqe_rx_w3.rb0_sz;
454                 pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0);
455                 nicvf_mbuff_init_update(pkt, mbuf_init, cqe_rx_w1.align_pad);
456                 nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt);
457                 rx_pkts[i] = pkt;
458                 cqe_head = (cqe_head + 1) & cqe_mask;
459                 nicvf_prefetch_store_keep(pkt);
460         }
461
462         if (likely(to_process)) {
463                 rxq->available_space -= to_process;
464                 rxq->head = cqe_head;
465                 nicvf_addr_write(rxq->cq_door, to_process);
466                 rxq->recv_buffers += to_process;
467         }
468         if (rxq->recv_buffers > rxq->rx_free_thresh) {
469                 rxq->recv_buffers -= nicvf_fill_rbdr(rxq, rxq->rx_free_thresh);
470                 NICVF_RX_ASSERT(rxq->recv_buffers >= 0);
471         }
472
473         return to_process;
474 }
475
476 static inline uint16_t __hot
477 nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx,
478                         uint64_t mbuf_phys_off,
479                         struct rte_mbuf **rx_pkt, uint8_t rbptr_offset,
480                         uint64_t mbuf_init)
481 {
482         struct rte_mbuf *pkt, *seg, *prev;
483         cqe_rx_word0_t cqe_rx_w0;
484         cqe_rx_word1_t cqe_rx_w1;
485         cqe_rx_word2_t cqe_rx_w2;
486         uint16_t *rb_sz, nb_segs, seg_idx;
487         uint64_t *rb_ptr;
488
489         NICVF_LOAD_PAIR(cqe_rx_w0.u64, cqe_rx_w1.u64, cqe_rx);
490         NICVF_RX_ASSERT(cqe_rx_w0.cqe_type == CQE_TYPE_RX);
491         cqe_rx_w2 = cqe_rx->word2;
492         rb_sz = &cqe_rx->word3.rb0_sz;
493         rb_ptr = (uint64_t *)cqe_rx + rbptr_offset;
494         nb_segs = cqe_rx_w0.rb_cnt;
495         pkt = (struct rte_mbuf *)nicvf_mbuff_phy2virt
496                         (rb_ptr[0] - cqe_rx_w1.align_pad, mbuf_phys_off);
497
498         pkt->ol_flags = 0;
499         pkt->pkt_len = cqe_rx_w1.pkt_len;
500         pkt->data_len = rb_sz[nicvf_frag_num(0)];
501         nicvf_mbuff_init_mseg_update(
502                                 pkt, mbuf_init, cqe_rx_w1.align_pad, nb_segs);
503         pkt->packet_type = nicvf_rx_classify_pkt(cqe_rx_w0);
504         nicvf_rx_offload(cqe_rx_w0, cqe_rx_w2, pkt);
505
506         *rx_pkt = pkt;
507         prev = pkt;
508         for (seg_idx = 1; seg_idx < nb_segs; seg_idx++) {
509                 seg = (struct rte_mbuf *)nicvf_mbuff_phy2virt
510                         (rb_ptr[seg_idx], mbuf_phys_off);
511
512                 prev->next = seg;
513                 seg->data_len = rb_sz[nicvf_frag_num(seg_idx)];
514                 nicvf_mbuff_init_update(seg, mbuf_init, 0);
515
516                 prev = seg;
517         }
518         prev->next = NULL;
519         return nb_segs;
520 }
521
522 uint16_t __hot
523 nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts,
524                          uint16_t nb_pkts)
525 {
526         union cq_entry_t *cq_entry;
527         struct cqe_rx_t *cqe_rx;
528         struct nicvf_rxq *rxq = rx_queue;
529         union cq_entry_t *desc = rxq->desc;
530         const uint64_t cqe_mask = rxq->qlen_mask;
531         uint64_t mbuf_phys_off = rxq->mbuf_phys_off;
532         uint32_t i, to_process, cqe_head, buffers_consumed = 0;
533         int32_t available_space = rxq->available_space;
534         uint16_t nb_segs;
535         const uint64_t mbuf_init = rxq->mbuf_initializer.value;
536         const uint8_t rbptr_offset = rxq->rbptr_offset;
537
538         cqe_head = rxq->head & cqe_mask;
539         to_process = nicvf_rx_pkts_to_process(rxq, nb_pkts, available_space);
540
541         for (i = 0; i < to_process; i++) {
542                 rte_prefetch_non_temporal(&desc[cqe_head + 2]);
543                 cq_entry = &desc[cqe_head];
544                 cqe_rx = (struct cqe_rx_t *)cq_entry;
545                 nb_segs = nicvf_process_cq_mseg_entry(cqe_rx, mbuf_phys_off,
546                         rx_pkts + i, rbptr_offset, mbuf_init);
547                 buffers_consumed += nb_segs;
548                 cqe_head = (cqe_head + 1) & cqe_mask;
549                 nicvf_prefetch_store_keep(rx_pkts[i]);
550         }
551
552         if (likely(to_process)) {
553                 rxq->available_space -= to_process;
554                 rxq->head = cqe_head;
555                 nicvf_addr_write(rxq->cq_door, to_process);
556                 rxq->recv_buffers += buffers_consumed;
557         }
558         if (rxq->recv_buffers > rxq->rx_free_thresh) {
559                 rxq->recv_buffers -= nicvf_fill_rbdr(rxq, rxq->rx_free_thresh);
560                 NICVF_RX_ASSERT(rxq->recv_buffers >= 0);
561         }
562
563         return to_process;
564 }
565
566 uint32_t
567 nicvf_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx)
568 {
569         struct nicvf_rxq *rxq;
570
571         rxq = dev->data->rx_queues[queue_idx];
572         return nicvf_addr_read(rxq->cq_status) & NICVF_CQ_CQE_COUNT_MASK;
573 }
574
575 uint32_t
576 nicvf_dev_rbdr_refill(struct rte_eth_dev *dev, uint16_t queue_idx)
577 {
578         struct nicvf_rxq *rxq;
579         uint32_t to_process;
580         uint32_t rx_free;
581
582         rxq = dev->data->rx_queues[queue_idx];
583         to_process = rxq->recv_buffers;
584         while (rxq->recv_buffers > 0) {
585                 rx_free = RTE_MIN(rxq->recv_buffers, NICVF_MAX_RX_FREE_THRESH);
586                 rxq->recv_buffers -= nicvf_fill_rbdr(rxq, rx_free);
587         }
588
589         assert(rxq->recv_buffers == 0);
590         return to_process;
591 }