net/bnxt: fix Rx ring producer index
[dpdk.git] / drivers / net / bnxt / bnxt_rxr.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2014-2018 Broadcom
3  * All rights reserved.
4  */
5
6 #include <inttypes.h>
7 #include <stdbool.h>
8
9 #include <rte_bitmap.h>
10 #include <rte_byteorder.h>
11 #include <rte_malloc.h>
12 #include <rte_memory.h>
13
14 #include "bnxt.h"
15 #include "bnxt_ring.h"
16 #include "bnxt_rxr.h"
17 #include "bnxt_rxq.h"
18 #include "hsi_struct_def_dpdk.h"
19 #ifdef RTE_LIBRTE_IEEE1588
20 #include "bnxt_hwrm.h"
21 #endif
22
23 #include <bnxt_tf_common.h>
24 #include <ulp_mark_mgr.h>
25
26 /*
27  * RX Ring handling
28  */
29
30 static inline struct rte_mbuf *__bnxt_alloc_rx_data(struct rte_mempool *mb)
31 {
32         struct rte_mbuf *data;
33
34         data = rte_mbuf_raw_alloc(mb);
35
36         return data;
37 }
38
39 static inline int bnxt_alloc_rx_data(struct bnxt_rx_queue *rxq,
40                                      struct bnxt_rx_ring_info *rxr,
41                                      uint16_t prod)
42 {
43         struct rx_prod_pkt_bd *rxbd = &rxr->rx_desc_ring[prod];
44         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
45         struct rte_mbuf *mbuf;
46
47         mbuf = __bnxt_alloc_rx_data(rxq->mb_pool);
48         if (!mbuf) {
49                 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail);
50                 return -ENOMEM;
51         }
52
53         rx_buf->mbuf = mbuf;
54         mbuf->data_off = RTE_PKTMBUF_HEADROOM;
55
56         rxbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
57
58         return 0;
59 }
60
61 static inline int bnxt_alloc_ag_data(struct bnxt_rx_queue *rxq,
62                                      struct bnxt_rx_ring_info *rxr,
63                                      uint16_t prod)
64 {
65         struct rx_prod_pkt_bd *rxbd = &rxr->ag_desc_ring[prod];
66         struct bnxt_sw_rx_bd *rx_buf = &rxr->ag_buf_ring[prod];
67         struct rte_mbuf *mbuf;
68
69         if (rxbd == NULL) {
70                 PMD_DRV_LOG(ERR, "Jumbo Frame. rxbd is NULL\n");
71                 return -EINVAL;
72         }
73
74         if (rx_buf == NULL) {
75                 PMD_DRV_LOG(ERR, "Jumbo Frame. rx_buf is NULL\n");
76                 return -EINVAL;
77         }
78
79         mbuf = __bnxt_alloc_rx_data(rxq->mb_pool);
80         if (!mbuf) {
81                 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail);
82                 return -ENOMEM;
83         }
84
85         rx_buf->mbuf = mbuf;
86         mbuf->data_off = RTE_PKTMBUF_HEADROOM;
87
88         rxbd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
89
90         return 0;
91 }
92
93 static inline void bnxt_reuse_rx_mbuf(struct bnxt_rx_ring_info *rxr,
94                                struct rte_mbuf *mbuf)
95 {
96         uint16_t prod = RING_NEXT(rxr->rx_ring_struct, rxr->rx_prod);
97         struct bnxt_sw_rx_bd *prod_rx_buf;
98         struct rx_prod_pkt_bd *prod_bd;
99
100         prod_rx_buf = &rxr->rx_buf_ring[prod];
101
102         RTE_ASSERT(prod_rx_buf->mbuf == NULL);
103         RTE_ASSERT(mbuf != NULL);
104
105         prod_rx_buf->mbuf = mbuf;
106
107         prod_bd = &rxr->rx_desc_ring[prod];
108
109         prod_bd->address = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
110
111         rxr->rx_prod = prod;
112 }
113
114 static inline
115 struct rte_mbuf *bnxt_consume_rx_buf(struct bnxt_rx_ring_info *rxr,
116                                      uint16_t cons)
117 {
118         struct bnxt_sw_rx_bd *cons_rx_buf;
119         struct rte_mbuf *mbuf;
120
121         cons_rx_buf = &rxr->rx_buf_ring[cons];
122         RTE_ASSERT(cons_rx_buf->mbuf != NULL);
123         mbuf = cons_rx_buf->mbuf;
124         cons_rx_buf->mbuf = NULL;
125         return mbuf;
126 }
127
128 static void bnxt_tpa_start(struct bnxt_rx_queue *rxq,
129                            struct rx_tpa_start_cmpl *tpa_start,
130                            struct rx_tpa_start_cmpl_hi *tpa_start1)
131 {
132         struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
133         uint16_t agg_id;
134         uint16_t data_cons;
135         struct bnxt_tpa_info *tpa_info;
136         struct rte_mbuf *mbuf;
137
138         agg_id = bnxt_tpa_start_agg_id(rxq->bp, tpa_start);
139
140         data_cons = tpa_start->opaque;
141         tpa_info = &rxr->tpa_info[agg_id];
142
143         mbuf = bnxt_consume_rx_buf(rxr, data_cons);
144
145         bnxt_reuse_rx_mbuf(rxr, tpa_info->mbuf);
146
147         tpa_info->agg_count = 0;
148         tpa_info->mbuf = mbuf;
149         tpa_info->len = rte_le_to_cpu_32(tpa_start->len);
150
151         mbuf->nb_segs = 1;
152         mbuf->next = NULL;
153         mbuf->pkt_len = rte_le_to_cpu_32(tpa_start->len);
154         mbuf->data_len = mbuf->pkt_len;
155         mbuf->port = rxq->port_id;
156         mbuf->ol_flags = PKT_RX_LRO;
157         if (likely(tpa_start->flags_type &
158                    rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS_RSS_VALID))) {
159                 mbuf->hash.rss = rte_le_to_cpu_32(tpa_start->rss_hash);
160                 mbuf->ol_flags |= PKT_RX_RSS_HASH;
161         } else {
162                 mbuf->hash.fdir.id = rte_le_to_cpu_16(tpa_start1->cfa_code);
163                 mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
164         }
165         if (tpa_start1->flags2 &
166             rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_META_FORMAT_VLAN)) {
167                 mbuf->vlan_tci = rte_le_to_cpu_32(tpa_start1->metadata);
168                 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
169         }
170         if (likely(tpa_start1->flags2 &
171                    rte_cpu_to_le_32(RX_TPA_START_CMPL_FLAGS2_L4_CS_CALC)))
172                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
173
174         /* recycle next mbuf */
175         data_cons = RING_NEXT(rxr->rx_ring_struct, data_cons);
176         bnxt_reuse_rx_mbuf(rxr, bnxt_consume_rx_buf(rxr, data_cons));
177 }
178
179 static int bnxt_agg_bufs_valid(struct bnxt_cp_ring_info *cpr,
180                 uint8_t agg_bufs, uint32_t raw_cp_cons)
181 {
182         uint16_t last_cp_cons;
183         struct rx_pkt_cmpl *agg_cmpl;
184
185         raw_cp_cons = ADV_RAW_CMP(raw_cp_cons, agg_bufs);
186         last_cp_cons = RING_CMP(cpr->cp_ring_struct, raw_cp_cons);
187         agg_cmpl = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[last_cp_cons];
188         cpr->valid = FLIP_VALID(raw_cp_cons,
189                                 cpr->cp_ring_struct->ring_mask,
190                                 cpr->valid);
191         return CMP_VALID(agg_cmpl, raw_cp_cons, cpr->cp_ring_struct);
192 }
193
194 /* TPA consume agg buffer out of order, allocate connected data only */
195 static int bnxt_prod_ag_mbuf(struct bnxt_rx_queue *rxq)
196 {
197         struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
198         uint16_t next = RING_NEXT(rxr->ag_ring_struct, rxr->ag_prod);
199
200         /* TODO batch allocation for better performance */
201         while (rte_bitmap_get(rxr->ag_bitmap, next)) {
202                 if (unlikely(bnxt_alloc_ag_data(rxq, rxr, next))) {
203                         PMD_DRV_LOG(ERR,
204                                 "agg mbuf alloc failed: prod=0x%x\n", next);
205                         break;
206                 }
207                 rte_bitmap_clear(rxr->ag_bitmap, next);
208                 rxr->ag_prod = next;
209                 next = RING_NEXT(rxr->ag_ring_struct, next);
210         }
211
212         return 0;
213 }
214
215 static int bnxt_rx_pages(struct bnxt_rx_queue *rxq,
216                          struct rte_mbuf *mbuf, uint32_t *tmp_raw_cons,
217                          uint8_t agg_buf, struct bnxt_tpa_info *tpa_info)
218 {
219         struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
220         struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
221         int i;
222         uint16_t cp_cons, ag_cons;
223         struct rx_pkt_cmpl *rxcmp;
224         struct rte_mbuf *last = mbuf;
225         bool is_thor_tpa = tpa_info && BNXT_CHIP_THOR(rxq->bp);
226
227         for (i = 0; i < agg_buf; i++) {
228                 struct bnxt_sw_rx_bd *ag_buf;
229                 struct rte_mbuf *ag_mbuf;
230
231                 if (is_thor_tpa) {
232                         rxcmp = (void *)&tpa_info->agg_arr[i];
233                 } else {
234                         *tmp_raw_cons = NEXT_RAW_CMP(*tmp_raw_cons);
235                         cp_cons = RING_CMP(cpr->cp_ring_struct, *tmp_raw_cons);
236                         rxcmp = (struct rx_pkt_cmpl *)
237                                         &cpr->cp_desc_ring[cp_cons];
238                 }
239
240 #ifdef BNXT_DEBUG
241                 bnxt_dump_cmpl(cp_cons, rxcmp);
242 #endif
243
244                 ag_cons = rxcmp->opaque;
245                 RTE_ASSERT(ag_cons <= rxr->ag_ring_struct->ring_mask);
246                 ag_buf = &rxr->ag_buf_ring[ag_cons];
247                 ag_mbuf = ag_buf->mbuf;
248                 RTE_ASSERT(ag_mbuf != NULL);
249
250                 ag_mbuf->data_len = rte_le_to_cpu_16(rxcmp->len);
251
252                 mbuf->nb_segs++;
253                 mbuf->pkt_len += ag_mbuf->data_len;
254
255                 last->next = ag_mbuf;
256                 last = ag_mbuf;
257
258                 ag_buf->mbuf = NULL;
259
260                 /*
261                  * As aggregation buffer consumed out of order in TPA module,
262                  * use bitmap to track freed slots to be allocated and notified
263                  * to NIC
264                  */
265                 rte_bitmap_set(rxr->ag_bitmap, ag_cons);
266         }
267         bnxt_prod_ag_mbuf(rxq);
268         return 0;
269 }
270
271 static inline struct rte_mbuf *bnxt_tpa_end(
272                 struct bnxt_rx_queue *rxq,
273                 uint32_t *raw_cp_cons,
274                 struct rx_tpa_end_cmpl *tpa_end,
275                 struct rx_tpa_end_cmpl_hi *tpa_end1)
276 {
277         struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
278         struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
279         uint16_t agg_id;
280         struct rte_mbuf *mbuf;
281         uint8_t agg_bufs;
282         uint8_t payload_offset;
283         struct bnxt_tpa_info *tpa_info;
284
285         if (BNXT_CHIP_THOR(rxq->bp)) {
286                 struct rx_tpa_v2_end_cmpl *th_tpa_end;
287                 struct rx_tpa_v2_end_cmpl_hi *th_tpa_end1;
288
289                 th_tpa_end = (void *)tpa_end;
290                 th_tpa_end1 = (void *)tpa_end1;
291                 agg_id = BNXT_TPA_END_AGG_ID_TH(th_tpa_end);
292                 agg_bufs = BNXT_TPA_END_AGG_BUFS_TH(th_tpa_end1);
293                 payload_offset = th_tpa_end1->payload_offset;
294         } else {
295                 agg_id = BNXT_TPA_END_AGG_ID(tpa_end);
296                 agg_bufs = BNXT_TPA_END_AGG_BUFS(tpa_end);
297                 if (!bnxt_agg_bufs_valid(cpr, agg_bufs, *raw_cp_cons))
298                         return NULL;
299                 payload_offset = tpa_end->payload_offset;
300         }
301
302         tpa_info = &rxr->tpa_info[agg_id];
303         mbuf = tpa_info->mbuf;
304         RTE_ASSERT(mbuf != NULL);
305
306         rte_prefetch0(mbuf);
307         if (agg_bufs) {
308                 bnxt_rx_pages(rxq, mbuf, raw_cp_cons, agg_bufs, tpa_info);
309         }
310         mbuf->l4_len = payload_offset;
311
312         struct rte_mbuf *new_data = __bnxt_alloc_rx_data(rxq->mb_pool);
313         RTE_ASSERT(new_data != NULL);
314         if (!new_data) {
315                 rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail);
316                 return NULL;
317         }
318         tpa_info->mbuf = new_data;
319
320         return mbuf;
321 }
322
323 static uint32_t
324 bnxt_parse_pkt_type(struct rx_pkt_cmpl *rxcmp, struct rx_pkt_cmpl_hi *rxcmp1)
325 {
326         uint32_t l3, pkt_type = 0;
327         uint32_t t_ipcs = 0, ip6 = 0, vlan = 0;
328         uint32_t flags_type;
329
330         vlan = !!(rxcmp1->flags2 &
331                 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN));
332         pkt_type |= vlan ? RTE_PTYPE_L2_ETHER_VLAN : RTE_PTYPE_L2_ETHER;
333
334         t_ipcs = !!(rxcmp1->flags2 &
335                 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC));
336         ip6 = !!(rxcmp1->flags2 &
337                  rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_TYPE));
338
339         flags_type = rxcmp->flags_type &
340                 rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS_ITYPE_MASK);
341
342         if (!t_ipcs && !ip6)
343                 l3 = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
344         else if (!t_ipcs && ip6)
345                 l3 = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
346         else if (t_ipcs && !ip6)
347                 l3 = RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN;
348         else
349                 l3 = RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN;
350
351         switch (flags_type) {
352         case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_ICMP):
353                 if (!t_ipcs)
354                         pkt_type |= l3 | RTE_PTYPE_L4_ICMP;
355                 else
356                         pkt_type |= l3 | RTE_PTYPE_INNER_L4_ICMP;
357                 break;
358
359         case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_TCP):
360                 if (!t_ipcs)
361                         pkt_type |= l3 | RTE_PTYPE_L4_TCP;
362                 else
363                         pkt_type |= l3 | RTE_PTYPE_INNER_L4_TCP;
364                 break;
365
366         case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_UDP):
367                 if (!t_ipcs)
368                         pkt_type |= l3 | RTE_PTYPE_L4_UDP;
369                 else
370                         pkt_type |= l3 | RTE_PTYPE_INNER_L4_UDP;
371                 break;
372
373         case RTE_LE32(RX_PKT_CMPL_FLAGS_ITYPE_IP):
374                 pkt_type |= l3;
375                 break;
376         }
377
378         return pkt_type;
379 }
380
381 #ifdef RTE_LIBRTE_IEEE1588
382 static void
383 bnxt_get_rx_ts_thor(struct bnxt *bp, uint32_t rx_ts_cmpl)
384 {
385         uint64_t systime_cycles = 0;
386
387         if (!BNXT_CHIP_THOR(bp))
388                 return;
389
390         /* On Thor, Rx timestamps are provided directly in the
391          * Rx completion records to the driver. Only 32 bits of
392          * the timestamp is present in the completion. Driver needs
393          * to read the current 48 bit free running timer using the
394          * HWRM_PORT_TS_QUERY command and combine the upper 16 bits
395          * from the HWRM response with the lower 32 bits in the
396          * Rx completion to produce the 48 bit timestamp for the Rx packet
397          */
398         bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME,
399                                 &systime_cycles);
400         bp->ptp_cfg->rx_timestamp = (systime_cycles & 0xFFFF00000000);
401         bp->ptp_cfg->rx_timestamp |= rx_ts_cmpl;
402 }
403 #endif
404
405 static void
406 bnxt_ulp_set_mark_in_mbuf(struct bnxt *bp, struct rx_pkt_cmpl_hi *rxcmp1,
407                           struct rte_mbuf *mbuf)
408 {
409         uint32_t cfa_code;
410         uint32_t meta_fmt;
411         uint32_t meta;
412         bool gfid = false;
413         uint32_t mark_id;
414         uint32_t flags2;
415         int rc;
416
417         cfa_code = rte_le_to_cpu_16(rxcmp1->cfa_code);
418         flags2 = rte_le_to_cpu_32(rxcmp1->flags2);
419         meta = rte_le_to_cpu_32(rxcmp1->metadata);
420
421         /*
422          * The flags field holds extra bits of info from [6:4]
423          * which indicate if the flow is in TCAM or EM or EEM
424          */
425         meta_fmt = (flags2 & BNXT_CFA_META_FMT_MASK) >>
426                 BNXT_CFA_META_FMT_SHFT;
427
428         switch (meta_fmt) {
429         case 0:
430                 /* Not an LFID or GFID, a flush cmd. */
431                 goto skip_mark;
432         case 4:
433         case 5:
434                 /*
435                  * EM/TCAM case
436                  * Assume that EM doesn't support Mark due to GFID
437                  * collisions with EEM.  Simply return without setting the mark
438                  * in the mbuf.
439                  */
440                 if (BNXT_CFA_META_EM_TEST(meta))
441                         goto skip_mark;
442                 /*
443                  * It is a TCAM entry, so it is an LFID. The TCAM IDX and Mode
444                  * can also be determined by decoding the meta_data.  We are not
445                  * using these for now.
446                  */
447                 break;
448         case 6:
449         case 7:
450                 /* EEM Case, only using gfid in EEM for now. */
451                 gfid = true;
452
453                 /*
454                  * For EEM flows, The first part of cfa_code is 16 bits.
455                  * The second part is embedded in the
456                  * metadata field from bit 19 onwards. The driver needs to
457                  * ignore the first 19 bits of metadata and use the next 12
458                  * bits as higher 12 bits of cfa_code.
459                  */
460                 meta >>= BNXT_RX_META_CFA_CODE_SHIFT;
461                 cfa_code |= meta << BNXT_CFA_CODE_META_SHIFT;
462                 break;
463         default:
464                 /* For other values, the cfa_code is assumed to be an LFID. */
465                 break;
466         }
467
468         if (cfa_code) {
469                 rc = ulp_mark_db_mark_get(bp->ulp_ctx, gfid,
470                                           cfa_code, &mark_id);
471                 if (!rc) {
472                         /* Got the mark, write it to the mbuf and return */
473                         mbuf->hash.fdir.hi = mark_id;
474                         mbuf->udata64 = (cfa_code & 0xffffffffull) << 32;
475                         mbuf->hash.fdir.id = rxcmp1->cfa_code;
476                         mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
477                         return;
478                 }
479         }
480
481 skip_mark:
482         mbuf->hash.fdir.hi = 0;
483         mbuf->hash.fdir.id = 0;
484 }
485
486 void bnxt_set_mark_in_mbuf(struct bnxt *bp,
487                            struct rx_pkt_cmpl_hi *rxcmp1,
488                            struct rte_mbuf *mbuf)
489 {
490         uint32_t cfa_code = 0;
491         uint8_t meta_fmt = 0;
492         uint16_t flags2 = 0;
493         uint32_t meta =  0;
494
495         cfa_code = rte_le_to_cpu_16(rxcmp1->cfa_code);
496         if (!cfa_code)
497                 return;
498
499         if (cfa_code && !bp->mark_table[cfa_code].valid)
500                 return;
501
502         flags2 = rte_le_to_cpu_16(rxcmp1->flags2);
503         meta = rte_le_to_cpu_32(rxcmp1->metadata);
504         if (meta) {
505                 meta >>= BNXT_RX_META_CFA_CODE_SHIFT;
506
507                 /* The flags field holds extra bits of info from [6:4]
508                  * which indicate if the flow is in TCAM or EM or EEM
509                  */
510                 meta_fmt = (flags2 & BNXT_CFA_META_FMT_MASK) >>
511                            BNXT_CFA_META_FMT_SHFT;
512
513                 /* meta_fmt == 4 => 'b100 => 'b10x => EM.
514                  * meta_fmt == 5 => 'b101 => 'b10x => EM + VLAN
515                  * meta_fmt == 6 => 'b110 => 'b11x => EEM
516                  * meta_fmt == 7 => 'b111 => 'b11x => EEM + VLAN.
517                  */
518                 meta_fmt >>= BNXT_CFA_META_FMT_EM_EEM_SHFT;
519         }
520
521         mbuf->hash.fdir.hi = bp->mark_table[cfa_code].mark_id;
522         mbuf->ol_flags |= PKT_RX_FDIR | PKT_RX_FDIR_ID;
523 }
524
525 static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt,
526                             struct bnxt_rx_queue *rxq, uint32_t *raw_cons)
527 {
528         struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
529         struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
530         struct rx_pkt_cmpl *rxcmp;
531         struct rx_pkt_cmpl_hi *rxcmp1;
532         uint32_t tmp_raw_cons = *raw_cons;
533         uint16_t cons, prod, cp_cons =
534             RING_CMP(cpr->cp_ring_struct, tmp_raw_cons);
535         struct rte_mbuf *mbuf;
536         int rc = 0;
537         uint8_t agg_buf = 0;
538         uint16_t cmp_type;
539         uint32_t flags2_f = 0;
540         uint16_t flags_type;
541         struct bnxt *bp = rxq->bp;
542
543         rxcmp = (struct rx_pkt_cmpl *)
544             &cpr->cp_desc_ring[cp_cons];
545
546         cmp_type = CMP_TYPE(rxcmp);
547
548         if (cmp_type == RX_TPA_V2_ABUF_CMPL_TYPE_RX_TPA_AGG) {
549                 struct rx_tpa_v2_abuf_cmpl *rx_agg = (void *)rxcmp;
550                 uint16_t agg_id = rte_cpu_to_le_16(rx_agg->agg_id);
551                 struct bnxt_tpa_info *tpa_info;
552
553                 tpa_info = &rxr->tpa_info[agg_id];
554                 RTE_ASSERT(tpa_info->agg_count < 16);
555                 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
556                 rc = -EINVAL; /* Continue w/o new mbuf */
557                 goto next_rx;
558         }
559
560         tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
561         cp_cons = RING_CMP(cpr->cp_ring_struct, tmp_raw_cons);
562         rxcmp1 = (struct rx_pkt_cmpl_hi *)&cpr->cp_desc_ring[cp_cons];
563
564         if (!CMP_VALID(rxcmp1, tmp_raw_cons, cpr->cp_ring_struct))
565                 return -EBUSY;
566
567         cpr->valid = FLIP_VALID(cp_cons,
568                                 cpr->cp_ring_struct->ring_mask,
569                                 cpr->valid);
570
571         if (cmp_type == RX_TPA_START_CMPL_TYPE_RX_TPA_START) {
572                 bnxt_tpa_start(rxq, (struct rx_tpa_start_cmpl *)rxcmp,
573                                (struct rx_tpa_start_cmpl_hi *)rxcmp1);
574                 rc = -EINVAL; /* Continue w/o new mbuf */
575                 goto next_rx;
576         } else if (cmp_type == RX_TPA_END_CMPL_TYPE_RX_TPA_END) {
577                 mbuf = bnxt_tpa_end(rxq, &tmp_raw_cons,
578                                    (struct rx_tpa_end_cmpl *)rxcmp,
579                                    (struct rx_tpa_end_cmpl_hi *)rxcmp1);
580                 if (unlikely(!mbuf))
581                         return -EBUSY;
582                 *rx_pkt = mbuf;
583                 goto next_rx;
584         } else if (cmp_type != 0x11) {
585                 rc = -EINVAL;
586                 goto next_rx;
587         }
588
589         agg_buf = (rxcmp->agg_bufs_v1 & RX_PKT_CMPL_AGG_BUFS_MASK)
590                         >> RX_PKT_CMPL_AGG_BUFS_SFT;
591         if (agg_buf && !bnxt_agg_bufs_valid(cpr, agg_buf, tmp_raw_cons))
592                 return -EBUSY;
593
594         prod = rxr->rx_prod;
595
596         cons = rxcmp->opaque;
597         mbuf = bnxt_consume_rx_buf(rxr, cons);
598         if (mbuf == NULL)
599                 return -EBUSY;
600
601         rte_prefetch0(mbuf);
602
603         mbuf->data_off = RTE_PKTMBUF_HEADROOM;
604         mbuf->nb_segs = 1;
605         mbuf->next = NULL;
606         mbuf->pkt_len = rxcmp->len;
607         mbuf->data_len = mbuf->pkt_len;
608         mbuf->port = rxq->port_id;
609         mbuf->ol_flags = 0;
610
611         flags_type = rte_le_to_cpu_16(rxcmp->flags_type);
612         if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) {
613                 mbuf->hash.rss = rxcmp->rss_hash;
614                 mbuf->ol_flags |= PKT_RX_RSS_HASH;
615         }
616
617         if (BNXT_TRUFLOW_EN(bp))
618                 bnxt_ulp_set_mark_in_mbuf(rxq->bp, rxcmp1, mbuf);
619         else
620                 bnxt_set_mark_in_mbuf(rxq->bp, rxcmp1, mbuf);
621
622 #ifdef RTE_LIBRTE_IEEE1588
623         if (unlikely((flags_type & RX_PKT_CMPL_FLAGS_MASK) ==
624                      RX_PKT_CMPL_FLAGS_ITYPE_PTP_W_TIMESTAMP)) {
625                 mbuf->ol_flags |= PKT_RX_IEEE1588_PTP | PKT_RX_IEEE1588_TMST;
626                 bnxt_get_rx_ts_thor(rxq->bp, rxcmp1->reorder);
627         }
628 #endif
629         if (agg_buf)
630                 bnxt_rx_pages(rxq, mbuf, &tmp_raw_cons, agg_buf, NULL);
631
632         if (rxcmp1->flags2 & RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN) {
633                 mbuf->vlan_tci = rxcmp1->metadata &
634                         (RX_PKT_CMPL_METADATA_VID_MASK |
635                         RX_PKT_CMPL_METADATA_DE |
636                         RX_PKT_CMPL_METADATA_PRI_MASK);
637                 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
638         }
639
640         flags2_f = flags2_0xf(rxcmp1);
641         /* IP Checksum */
642         if (likely(IS_IP_NONTUNNEL_PKT(flags2_f))) {
643                 if (unlikely(RX_CMP_IP_CS_ERROR(rxcmp1)))
644                         mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
645                 else if (unlikely(RX_CMP_IP_CS_UNKNOWN(rxcmp1)))
646                         mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
647                 else
648                         mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
649         } else if (IS_IP_TUNNEL_PKT(flags2_f)) {
650                 if (unlikely(RX_CMP_IP_OUTER_CS_ERROR(rxcmp1) ||
651                              RX_CMP_IP_CS_ERROR(rxcmp1)))
652                         mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
653                 else if (unlikely(RX_CMP_IP_CS_UNKNOWN(rxcmp1)))
654                         mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
655                 else
656                         mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD;
657         }
658
659         /* L4 Checksum */
660         if (likely(IS_L4_NONTUNNEL_PKT(flags2_f))) {
661                 if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1)))
662                         mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
663                 else
664                         mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
665         } else if (IS_L4_TUNNEL_PKT(flags2_f)) {
666                 if (unlikely(RX_CMP_L4_INNER_CS_ERR2(rxcmp1)))
667                         mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
668                 else
669                         mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
670                 if (unlikely(RX_CMP_L4_OUTER_CS_ERR2(rxcmp1))) {
671                         mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_BAD;
672                 } else if (unlikely(IS_L4_TUNNEL_PKT_ONLY_INNER_L4_CS
673                                     (flags2_f))) {
674                         mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_UNKNOWN;
675                 } else {
676                         mbuf->ol_flags |= PKT_RX_OUTER_L4_CKSUM_GOOD;
677                 }
678         } else if (unlikely(RX_CMP_L4_CS_UNKNOWN(rxcmp1))) {
679                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
680         }
681
682         mbuf->packet_type = bnxt_parse_pkt_type(rxcmp, rxcmp1);
683
684 #ifdef BNXT_DEBUG
685         if (rxcmp1->errors_v2 & RX_CMP_L2_ERRORS) {
686                 /* Re-install the mbuf back to the rx ring */
687                 bnxt_reuse_rx_mbuf(rxr, cons, mbuf);
688
689                 rc = -EIO;
690                 goto next_rx;
691         }
692 #endif
693         /*
694          * TODO: Redesign this....
695          * If the allocation fails, the packet does not get received.
696          * Simply returning this will result in slowly falling behind
697          * on the producer ring buffers.
698          * Instead, "filling up" the producer just before ringing the
699          * doorbell could be a better solution since it will let the
700          * producer ring starve until memory is available again pushing
701          * the drops into hardware and getting them out of the driver
702          * allowing recovery to a full producer ring.
703          *
704          * This could also help with cache usage by preventing per-packet
705          * calls in favour of a tight loop with the same function being called
706          * in it.
707          */
708         prod = RING_NEXT(rxr->rx_ring_struct, prod);
709         if (bnxt_alloc_rx_data(rxq, rxr, prod)) {
710                 PMD_DRV_LOG(ERR, "mbuf alloc failed with prod=0x%x\n", prod);
711                 rc = -ENOMEM;
712                 goto rx;
713         }
714         rxr->rx_prod = prod;
715         /*
716          * All MBUFs are allocated with the same size under DPDK,
717          * no optimization for rx_copy_thresh
718          */
719 rx:
720         *rx_pkt = mbuf;
721
722 next_rx:
723
724         *raw_cons = tmp_raw_cons;
725
726         return rc;
727 }
728
729 uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
730                                uint16_t nb_pkts)
731 {
732         struct bnxt_rx_queue *rxq = rx_queue;
733         struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
734         struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
735         uint32_t raw_cons = cpr->cp_raw_cons;
736         uint32_t cons;
737         int nb_rx_pkts = 0;
738         struct rx_pkt_cmpl *rxcmp;
739         uint16_t prod = rxr->rx_prod;
740         uint16_t ag_prod = rxr->ag_prod;
741         int rc = 0;
742         bool evt = false;
743
744         if (unlikely(is_bnxt_in_error(rxq->bp)))
745                 return 0;
746
747         /* If Rx Q was stopped return */
748         if (unlikely(!rxq->rx_started ||
749                      !rte_spinlock_trylock(&rxq->lock)))
750                 return 0;
751
752         /* Handle RX burst request */
753         while (1) {
754                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
755                 rte_prefetch0(&cpr->cp_desc_ring[cons]);
756                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
757
758                 if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct))
759                         break;
760                 cpr->valid = FLIP_VALID(cons,
761                                         cpr->cp_ring_struct->ring_mask,
762                                         cpr->valid);
763
764                 /* TODO: Avoid magic numbers... */
765                 if ((CMP_TYPE(rxcmp) & 0x30) == 0x10) {
766                         rc = bnxt_rx_pkt(&rx_pkts[nb_rx_pkts], rxq, &raw_cons);
767                         if (likely(!rc) || rc == -ENOMEM)
768                                 nb_rx_pkts++;
769                         if (rc == -EBUSY)       /* partial completion */
770                                 break;
771                 } else if (!BNXT_NUM_ASYNC_CPR(rxq->bp)) {
772                         evt =
773                         bnxt_event_hwrm_resp_handler(rxq->bp,
774                                                      (struct cmpl_base *)rxcmp);
775                         /* If the async event is Fatal error, return */
776                         if (unlikely(is_bnxt_in_error(rxq->bp)))
777                                 goto done;
778                 }
779
780                 raw_cons = NEXT_RAW_CMP(raw_cons);
781                 if (nb_rx_pkts == nb_pkts || evt)
782                         break;
783                 /* Post some Rx buf early in case of larger burst processing */
784                 if (nb_rx_pkts == BNXT_RX_POST_THRESH)
785                         bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
786         }
787
788         cpr->cp_raw_cons = raw_cons;
789         if (!nb_rx_pkts && !evt) {
790                 /*
791                  * For PMD, there is no need to keep on pushing to REARM
792                  * the doorbell if there are no new completions
793                  */
794                 goto done;
795         }
796
797         if (prod != rxr->rx_prod)
798                 bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
799
800         /* Ring the AGG ring DB */
801         if (ag_prod != rxr->ag_prod)
802                 bnxt_db_write(&rxr->ag_db, rxr->ag_prod);
803
804         bnxt_db_cq(cpr);
805
806         /* Attempt to alloc Rx buf in case of a previous allocation failure. */
807         if (rc == -ENOMEM) {
808                 int i = RING_NEXT(rxr->rx_ring_struct, prod);
809                 int cnt = nb_rx_pkts;
810
811                 for (; cnt;
812                         i = RING_NEXT(rxr->rx_ring_struct, i), cnt--) {
813                         struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
814
815                         /* Buffer already allocated for this index. */
816                         if (rx_buf->mbuf != NULL)
817                                 continue;
818
819                         /* This slot is empty. Alloc buffer for Rx */
820                         if (!bnxt_alloc_rx_data(rxq, rxr, i)) {
821                                 rxr->rx_prod = i;
822                                 bnxt_db_write(&rxr->rx_db, rxr->rx_prod);
823                         } else {
824                                 PMD_DRV_LOG(ERR, "Alloc  mbuf failed\n");
825                                 break;
826                         }
827                 }
828         }
829
830 done:
831         rte_spinlock_unlock(&rxq->lock);
832
833         return nb_rx_pkts;
834 }
835
836 /*
837  * Dummy DPDK callback for RX.
838  *
839  * This function is used to temporarily replace the real callback during
840  * unsafe control operations on the queue, or in case of error.
841  */
842 uint16_t
843 bnxt_dummy_recv_pkts(void *rx_queue __rte_unused,
844                      struct rte_mbuf **rx_pkts __rte_unused,
845                      uint16_t nb_pkts __rte_unused)
846 {
847         return 0;
848 }
849
850 void bnxt_free_rx_rings(struct bnxt *bp)
851 {
852         int i;
853         struct bnxt_rx_queue *rxq;
854
855         if (!bp->rx_queues)
856                 return;
857
858         for (i = 0; i < (int)bp->rx_nr_rings; i++) {
859                 rxq = bp->rx_queues[i];
860                 if (!rxq)
861                         continue;
862
863                 bnxt_free_ring(rxq->rx_ring->rx_ring_struct);
864                 rte_free(rxq->rx_ring->rx_ring_struct);
865
866                 /* Free the Aggregator ring */
867                 bnxt_free_ring(rxq->rx_ring->ag_ring_struct);
868                 rte_free(rxq->rx_ring->ag_ring_struct);
869                 rxq->rx_ring->ag_ring_struct = NULL;
870
871                 rte_free(rxq->rx_ring);
872
873                 bnxt_free_ring(rxq->cp_ring->cp_ring_struct);
874                 rte_free(rxq->cp_ring->cp_ring_struct);
875                 rte_free(rxq->cp_ring);
876
877                 rte_free(rxq);
878                 bp->rx_queues[i] = NULL;
879         }
880 }
881
882 int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id)
883 {
884         struct bnxt_cp_ring_info *cpr;
885         struct bnxt_rx_ring_info *rxr;
886         struct bnxt_ring *ring;
887
888         rxq->rx_buf_size = BNXT_MAX_PKT_LEN + sizeof(struct rte_mbuf);
889
890         rxr = rte_zmalloc_socket("bnxt_rx_ring",
891                                  sizeof(struct bnxt_rx_ring_info),
892                                  RTE_CACHE_LINE_SIZE, socket_id);
893         if (rxr == NULL)
894                 return -ENOMEM;
895         rxq->rx_ring = rxr;
896
897         ring = rte_zmalloc_socket("bnxt_rx_ring_struct",
898                                    sizeof(struct bnxt_ring),
899                                    RTE_CACHE_LINE_SIZE, socket_id);
900         if (ring == NULL)
901                 return -ENOMEM;
902         rxr->rx_ring_struct = ring;
903         ring->ring_size = rte_align32pow2(rxq->nb_rx_desc);
904         ring->ring_mask = ring->ring_size - 1;
905         ring->bd = (void *)rxr->rx_desc_ring;
906         ring->bd_dma = rxr->rx_desc_mapping;
907         ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd);
908         ring->vmem = (void **)&rxr->rx_buf_ring;
909
910         cpr = rte_zmalloc_socket("bnxt_rx_ring",
911                                  sizeof(struct bnxt_cp_ring_info),
912                                  RTE_CACHE_LINE_SIZE, socket_id);
913         if (cpr == NULL)
914                 return -ENOMEM;
915         rxq->cp_ring = cpr;
916
917         ring = rte_zmalloc_socket("bnxt_rx_ring_struct",
918                                    sizeof(struct bnxt_ring),
919                                    RTE_CACHE_LINE_SIZE, socket_id);
920         if (ring == NULL)
921                 return -ENOMEM;
922         cpr->cp_ring_struct = ring;
923         ring->ring_size = rte_align32pow2(rxr->rx_ring_struct->ring_size *
924                                           (2 + AGG_RING_SIZE_FACTOR));
925         ring->ring_mask = ring->ring_size - 1;
926         ring->bd = (void *)cpr->cp_desc_ring;
927         ring->bd_dma = cpr->cp_desc_mapping;
928         ring->vmem_size = 0;
929         ring->vmem = NULL;
930
931         /* Allocate Aggregator rings */
932         ring = rte_zmalloc_socket("bnxt_rx_ring_struct",
933                                    sizeof(struct bnxt_ring),
934                                    RTE_CACHE_LINE_SIZE, socket_id);
935         if (ring == NULL)
936                 return -ENOMEM;
937         rxr->ag_ring_struct = ring;
938         ring->ring_size = rte_align32pow2(rxq->nb_rx_desc *
939                                           AGG_RING_SIZE_FACTOR);
940         ring->ring_mask = ring->ring_size - 1;
941         ring->bd = (void *)rxr->ag_desc_ring;
942         ring->bd_dma = rxr->ag_desc_mapping;
943         ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd);
944         ring->vmem = (void **)&rxr->ag_buf_ring;
945
946         return 0;
947 }
948
949 static void bnxt_init_rxbds(struct bnxt_ring *ring, uint32_t type,
950                             uint16_t len)
951 {
952         uint32_t j;
953         struct rx_prod_pkt_bd *rx_bd_ring = (struct rx_prod_pkt_bd *)ring->bd;
954
955         if (!rx_bd_ring)
956                 return;
957         for (j = 0; j < ring->ring_size; j++) {
958                 rx_bd_ring[j].flags_type = rte_cpu_to_le_16(type);
959                 rx_bd_ring[j].len = rte_cpu_to_le_16(len);
960                 rx_bd_ring[j].opaque = j;
961         }
962 }
963
964 int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq)
965 {
966         struct bnxt_rx_ring_info *rxr;
967         struct bnxt_ring *ring;
968         uint32_t prod, type;
969         unsigned int i;
970         uint16_t size;
971
972         size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM;
973         size = RTE_MIN(BNXT_MAX_PKT_LEN, size);
974
975         type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT | RX_PROD_PKT_BD_FLAGS_EOP_PAD;
976
977         rxr = rxq->rx_ring;
978         ring = rxr->rx_ring_struct;
979         bnxt_init_rxbds(ring, type, size);
980
981         prod = rxr->rx_prod;
982         for (i = 0; i < ring->ring_size; i++) {
983                 if (unlikely(!rxr->rx_buf_ring[i].mbuf)) {
984                         if (bnxt_alloc_rx_data(rxq, rxr, prod) != 0) {
985                                 PMD_DRV_LOG(WARNING,
986                                             "init'ed rx ring %d with %d/%d mbufs only\n",
987                                             rxq->queue_id, i, ring->ring_size);
988                                 break;
989                         }
990                 }
991                 rxr->rx_prod = prod;
992                 prod = RING_NEXT(rxr->rx_ring_struct, prod);
993         }
994
995         ring = rxr->ag_ring_struct;
996         type = RX_PROD_AGG_BD_TYPE_RX_PROD_AGG;
997         bnxt_init_rxbds(ring, type, size);
998         prod = rxr->ag_prod;
999
1000         for (i = 0; i < ring->ring_size; i++) {
1001                 if (unlikely(!rxr->ag_buf_ring[i].mbuf)) {
1002                         if (bnxt_alloc_ag_data(rxq, rxr, prod) != 0) {
1003                                 PMD_DRV_LOG(WARNING,
1004                                             "init'ed AG ring %d with %d/%d mbufs only\n",
1005                                             rxq->queue_id, i, ring->ring_size);
1006                                 break;
1007                         }
1008                 }
1009                 rxr->ag_prod = prod;
1010                 prod = RING_NEXT(rxr->ag_ring_struct, prod);
1011         }
1012         PMD_DRV_LOG(DEBUG, "AGG Done!\n");
1013
1014         if (rxr->tpa_info) {
1015                 unsigned int max_aggs = BNXT_TPA_MAX_AGGS(rxq->bp);
1016
1017                 for (i = 0; i < max_aggs; i++) {
1018                         if (unlikely(!rxr->tpa_info[i].mbuf)) {
1019                                 rxr->tpa_info[i].mbuf =
1020                                         __bnxt_alloc_rx_data(rxq->mb_pool);
1021                                 if (!rxr->tpa_info[i].mbuf) {
1022                                         rte_atomic64_inc(&rxq->rx_mbuf_alloc_fail);
1023                                         return -ENOMEM;
1024                                 }
1025                         }
1026                 }
1027         }
1028         PMD_DRV_LOG(DEBUG, "TPA alloc Done!\n");
1029
1030         return 0;
1031 }