net/bnx2x: get cache line size from build configuration
[dpdk.git] / drivers / net / bnx2x / bnx2x_rxtx.c
1 /*
2  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
3  *
4  * Copyright (c) 2015 QLogic Corporation.
5  * All rights reserved.
6  * www.qlogic.com
7  *
8  * See LICENSE.bnx2x_pmd for copyright and licensing details.
9  */
10
11 #include "bnx2x.h"
12 #include "bnx2x_rxtx.h"
13
14 static const struct rte_memzone *
15 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
16                       uint16_t queue_id, uint32_t ring_size, int socket_id)
17 {
18         char z_name[RTE_MEMZONE_NAMESIZE];
19         const struct rte_memzone *mz;
20
21         snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
22                         dev->driver->pci_drv.driver.name, ring_name,
23                         dev->data->port_id, queue_id);
24
25         mz = rte_memzone_lookup(z_name);
26         if (mz)
27                 return mz;
28
29         return rte_memzone_reserve_aligned(z_name, ring_size, socket_id, 0, BNX2X_PAGE_SIZE);
30 }
31
32 static void
33 bnx2x_rx_queue_release(struct bnx2x_rx_queue *rx_queue)
34 {
35         uint16_t i;
36         struct rte_mbuf **sw_ring;
37
38         if (NULL != rx_queue) {
39
40                 sw_ring = rx_queue->sw_ring;
41                 if (NULL != sw_ring) {
42                         for (i = 0; i < rx_queue->nb_rx_desc; i++) {
43                                 if (NULL != sw_ring[i])
44                                         rte_pktmbuf_free(sw_ring[i]);
45                         }
46                         rte_free(sw_ring);
47                 }
48                 rte_free(rx_queue);
49         }
50 }
51
52 void
53 bnx2x_dev_rx_queue_release(void *rxq)
54 {
55         bnx2x_rx_queue_release(rxq);
56 }
57
58 int
59 bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
60                        uint16_t queue_idx,
61                        uint16_t nb_desc,
62                        unsigned int socket_id,
63                        const struct rte_eth_rxconf *rx_conf,
64                        struct rte_mempool *mp)
65 {
66         uint16_t j, idx;
67         const struct rte_memzone *dma;
68         struct bnx2x_rx_queue *rxq;
69         uint32_t dma_size;
70         struct rte_mbuf *mbuf;
71         struct bnx2x_softc *sc = dev->data->dev_private;
72         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
73         struct eth_rx_cqe_next_page *nextpg;
74         phys_addr_t *rx_bd;
75         phys_addr_t busaddr;
76
77         /* First allocate the rx queue data structure */
78         rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
79                                  RTE_CACHE_LINE_SIZE, socket_id);
80         if (NULL == rxq) {
81                 PMD_INIT_LOG(ERR, "rte_zmalloc for rxq failed!");
82                 return -ENOMEM;
83         }
84         rxq->sc = sc;
85         rxq->mb_pool = mp;
86         rxq->queue_id = queue_idx;
87         rxq->port_id = dev->data->port_id;
88         rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.hw_strip_crc) ? 0 : ETHER_CRC_LEN);
89
90         rxq->nb_rx_pages = 1;
91         while (USABLE_RX_BD(rxq) < nb_desc)
92                 rxq->nb_rx_pages <<= 1;
93
94         rxq->nb_rx_desc  = TOTAL_RX_BD(rxq);
95         sc->rx_ring_size = USABLE_RX_BD(rxq);
96         rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
97
98         rxq->rx_free_thresh = rx_conf->rx_free_thresh ?
99                 rx_conf->rx_free_thresh : DEFAULT_RX_FREE_THRESH;
100
101         PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
102                        "total_bd=%lu, rx_pages=%u, cq_pages=%u",
103                        queue_idx, nb_desc, rxq->rx_free_thresh,
104                        (unsigned long)USABLE_RX_BD(rxq),
105                        (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
106                        rxq->nb_cq_pages);
107
108         /* Allocate RX ring hardware descriptors */
109         dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd);
110         dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id);
111         if (NULL == dma) {
112                 PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!");
113                 bnx2x_rx_queue_release(rxq);
114                 return -ENOMEM;
115         }
116         fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->phys_addr;
117         rxq->rx_ring = (uint64_t*)dma->addr;
118         memset((void *)rxq->rx_ring, 0, dma_size);
119
120         /* Link the RX chain pages. */
121         for (j = 1; j <= rxq->nb_rx_pages; j++) {
122                 rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2];
123                 busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages);
124                 *rx_bd = busaddr;
125         }
126
127         /* Allocate software ring */
128         dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry);
129         rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size,
130                                           RTE_CACHE_LINE_SIZE,
131                                           socket_id);
132         if (NULL == rxq->sw_ring) {
133                 PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!");
134                 bnx2x_rx_queue_release(rxq);
135                 return -ENOMEM;
136         }
137
138         /* Initialize software ring entries */
139         rxq->rx_mbuf_alloc = 0;
140         for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) {
141                 mbuf = rte_mbuf_raw_alloc(mp);
142                 if (NULL == mbuf) {
143                         PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d",
144                                    (unsigned)rxq->queue_id, idx);
145                         bnx2x_rx_queue_release(rxq);
146                         return -ENOMEM;
147                 }
148                 rxq->sw_ring[idx] = mbuf;
149                 rxq->rx_ring[idx] = mbuf->buf_physaddr;
150                 rxq->rx_mbuf_alloc++;
151         }
152         rxq->pkt_first_seg = NULL;
153         rxq->pkt_last_seg = NULL;
154         rxq->rx_bd_head = 0;
155         rxq->rx_bd_tail = rxq->nb_rx_desc;
156
157         /* Allocate CQ chain. */
158         dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages;
159         dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id);
160         if (NULL == dma) {
161                 PMD_RX_LOG(ERR, "RCQ  alloc failed");
162                 return -ENOMEM;
163         }
164         fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->phys_addr;
165         rxq->cq_ring = (union eth_rx_cqe*)dma->addr;
166
167         /* Link the CQ chain pages. */
168         for (j = 1; j <= rxq->nb_cq_pages; j++) {
169                 nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe;
170                 busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages);
171                 nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
172                 nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
173         }
174         rxq->rx_cq_head = 0;
175         rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq);
176
177         dev->data->rx_queues[queue_idx] = rxq;
178         if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues;
179
180         return 0;
181 }
182
183 static void
184 bnx2x_tx_queue_release(struct bnx2x_tx_queue *tx_queue)
185 {
186         uint16_t i;
187         struct rte_mbuf **sw_ring;
188
189         if (NULL != tx_queue) {
190
191                 sw_ring = tx_queue->sw_ring;
192                 if (NULL != sw_ring) {
193                         for (i = 0; i < tx_queue->nb_tx_desc; i++) {
194                                 if (NULL != sw_ring[i])
195                                         rte_pktmbuf_free(sw_ring[i]);
196                         }
197                         rte_free(sw_ring);
198                 }
199                 rte_free(tx_queue);
200         }
201 }
202
203 void
204 bnx2x_dev_tx_queue_release(void *txq)
205 {
206         bnx2x_tx_queue_release(txq);
207 }
208
209 static uint16_t
210 bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
211 {
212         struct bnx2x_tx_queue *txq;
213         struct bnx2x_softc *sc;
214         struct bnx2x_fastpath *fp;
215         uint16_t nb_tx_pkts;
216         uint16_t nb_pkt_sent = 0;
217         uint32_t ret;
218
219         txq = p_txq;
220         sc = txq->sc;
221         fp = &sc->fp[txq->queue_id];
222
223         if ((unlikely((txq->nb_tx_desc - txq->nb_tx_avail) >
224                                 txq->tx_free_thresh)))
225                 bnx2x_txeof(sc, fp);
226
227         nb_tx_pkts = RTE_MIN(nb_pkts, txq->nb_tx_avail / BDS_PER_TX_PKT);
228         if (unlikely(nb_tx_pkts == 0))
229                 return 0;
230
231         while (nb_tx_pkts--) {
232                 struct rte_mbuf *m = *tx_pkts++;
233                 assert(m != NULL);
234                 ret = bnx2x_tx_encap(txq, m);
235                 fp->tx_db.data.prod += ret;
236                 nb_pkt_sent++;
237         }
238
239         bnx2x_update_fp_sb_idx(fp);
240         mb();
241         DOORBELL(sc, txq->queue_id, fp->tx_db.raw);
242         mb();
243
244         if ((txq->nb_tx_desc - txq->nb_tx_avail) >
245                                 txq->tx_free_thresh)
246                 bnx2x_txeof(sc, fp);
247
248         return nb_pkt_sent;
249 }
250
251 int
252 bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
253                        uint16_t queue_idx,
254                        uint16_t nb_desc,
255                        unsigned int socket_id,
256                        const struct rte_eth_txconf *tx_conf)
257 {
258         uint16_t i;
259         unsigned int tsize;
260         const struct rte_memzone *tz;
261         struct bnx2x_tx_queue *txq;
262         struct eth_tx_next_bd *tx_n_bd;
263         uint64_t busaddr;
264         struct bnx2x_softc *sc = dev->data->dev_private;
265         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
266
267         /* First allocate the tx queue data structure */
268         txq = rte_zmalloc("ethdev TX queue", sizeof(struct bnx2x_tx_queue),
269                           RTE_CACHE_LINE_SIZE);
270         if (txq == NULL)
271                 return -ENOMEM;
272         txq->sc = sc;
273
274         txq->nb_tx_pages = 1;
275         while (USABLE_TX_BD(txq) < nb_desc)
276                 txq->nb_tx_pages <<= 1;
277
278         txq->nb_tx_desc  = TOTAL_TX_BD(txq);
279         sc->tx_ring_size = TOTAL_TX_BD(txq);
280
281         txq->tx_free_thresh = tx_conf->tx_free_thresh ?
282                 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH;
283
284         PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
285                      "total_bd=%lu, tx_pages=%u",
286                      queue_idx, nb_desc, txq->tx_free_thresh,
287                      (unsigned long)USABLE_TX_BD(txq),
288                      (unsigned long)TOTAL_TX_BD(txq), txq->nb_tx_pages);
289
290         /* Allocate TX ring hardware descriptors */
291         tsize = txq->nb_tx_desc * sizeof(union eth_tx_bd_types);
292         tz = ring_dma_zone_reserve(dev, "tx_hw_ring", queue_idx, tsize, socket_id);
293         if (tz == NULL) {
294                 bnx2x_tx_queue_release(txq);
295                 return -ENOMEM;
296         }
297         fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->phys_addr;
298         txq->tx_ring = (union eth_tx_bd_types *) tz->addr;
299         memset(txq->tx_ring, 0, tsize);
300
301         /* Allocate software ring */
302         tsize = txq->nb_tx_desc * sizeof(struct rte_mbuf *);
303         txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize,
304                                    RTE_CACHE_LINE_SIZE);
305         if (txq->sw_ring == NULL) {
306                 bnx2x_tx_queue_release(txq);
307                 return -ENOMEM;
308         }
309
310         /* PMD_DRV_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
311            txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */
312
313         /* Link TX pages */
314         for (i = 1; i <= txq->nb_tx_pages; i++) {
315                 tx_n_bd = &txq->tx_ring[TOTAL_TX_BD_PER_PAGE * i - 1].next_bd;
316                 busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages);
317                 tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
318                 tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
319                 /* PMD_DRV_LOG(DEBUG, "link tx page %lu", (TOTAL_TX_BD_PER_PAGE * i - 1)); */
320         }
321
322         txq->queue_id = queue_idx;
323         txq->port_id = dev->data->port_id;
324         txq->tx_pkt_tail = 0;
325         txq->tx_pkt_head = 0;
326         txq->tx_bd_tail = 0;
327         txq->tx_bd_head = 0;
328         txq->nb_tx_avail = txq->nb_tx_desc;
329         dev->tx_pkt_burst = bnx2x_xmit_pkts;
330         dev->data->tx_queues[queue_idx] = txq;
331         if (!sc->tx_queues) sc->tx_queues = dev->data->tx_queues;
332
333         return 0;
334 }
335
336 static inline void
337 bnx2x_upd_rx_prod_fast(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
338                 uint16_t rx_bd_prod, uint16_t rx_cq_prod)
339 {
340         union ustorm_eth_rx_producers rx_prods;
341
342         rx_prods.prod.bd_prod  = rx_bd_prod;
343         rx_prods.prod.cqe_prod = rx_cq_prod;
344
345         REG_WR(sc, fp->ustorm_rx_prods_offset, rx_prods.raw_data[0]);
346 }
347
348 static uint16_t
349 bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
350 {
351         struct bnx2x_rx_queue *rxq = p_rxq;
352         struct bnx2x_softc *sc = rxq->sc;
353         struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id];
354         uint32_t nb_rx = 0;
355         uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod;
356         uint16_t bd_cons, bd_prod;
357         struct rte_mbuf *new_mb;
358         uint16_t rx_pref;
359         struct eth_fast_path_rx_cqe *cqe_fp;
360         uint16_t len, pad;
361         struct rte_mbuf *rx_mb = NULL;
362
363         hw_cq_cons = le16toh(*fp->rx_cq_cons_sb);
364         if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) ==
365                         USABLE_RCQ_ENTRIES_PER_PAGE) {
366                 ++hw_cq_cons;
367         }
368
369         bd_cons = rxq->rx_bd_head;
370         bd_prod = rxq->rx_bd_tail;
371         sw_cq_cons = rxq->rx_cq_head;
372         sw_cq_prod = rxq->rx_cq_tail;
373
374         if (sw_cq_cons == hw_cq_cons)
375                 return 0;
376
377         while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) {
378
379                 bd_prod &= MAX_RX_BD(rxq);
380                 bd_cons &= MAX_RX_BD(rxq);
381
382                 cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe;
383
384                 if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) {
385                         PMD_RX_LOG(ERR, "slowpath event during traffic processing");
386                         break;
387                 }
388
389                 if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) {
390                         PMD_RX_LOG(ERR, "flags 0x%x rx packet %u",
391                                         cqe_fp->type_error_flags, sw_cq_cons);
392                         goto next_rx;
393                 }
394
395                 len = cqe_fp->pkt_len_or_gro_seg_len;
396                 pad = cqe_fp->placement_offset;
397
398                 new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
399                 if (unlikely(!new_mb)) {
400                         PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index);
401                         rte_eth_devices[rxq->port_id].data->
402                                         rx_mbuf_alloc_failed++;
403                         goto next_rx;
404                 }
405
406                 rx_mb = rxq->sw_ring[bd_cons];
407                 rxq->sw_ring[bd_cons] = new_mb;
408                 rxq->rx_ring[bd_prod] = new_mb->buf_physaddr;
409
410                 rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq);
411                 rte_prefetch0(rxq->sw_ring[rx_pref]);
412                 if ((rx_pref & 0x3) == 0) {
413                         rte_prefetch0(&rxq->rx_ring[rx_pref]);
414                         rte_prefetch0(&rxq->sw_ring[rx_pref]);
415                 }
416
417                 rx_mb->data_off = pad;
418                 rx_mb->nb_segs = 1;
419                 rx_mb->next = NULL;
420                 rx_mb->pkt_len = rx_mb->data_len = len;
421                 rx_mb->port = rxq->port_id;
422                 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
423
424                 /*
425                  * If we received a packet with a vlan tag,
426                  * attach that information to the packet.
427                  */
428                 if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) {
429                         rx_mb->vlan_tci = cqe_fp->vlan_tag;
430                         rx_mb->ol_flags |= PKT_RX_VLAN_PKT;
431                 }
432
433                 rx_pkts[nb_rx] = rx_mb;
434                 nb_rx++;
435
436                 /* limit spinning on the queue */
437                 if (unlikely(nb_rx == sc->rx_budget)) {
438                         PMD_RX_LOG(ERR, "Limit spinning on the queue");
439                         break;
440                 }
441
442 next_rx:
443                 bd_cons    = NEXT_RX_BD(bd_cons);
444                 bd_prod    = NEXT_RX_BD(bd_prod);
445                 sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod);
446                 sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons);
447         }
448         rxq->rx_bd_head = bd_cons;
449         rxq->rx_bd_tail = bd_prod;
450         rxq->rx_cq_head = sw_cq_cons;
451         rxq->rx_cq_tail = sw_cq_prod;
452
453         bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod);
454
455         return nb_rx;
456 }
457
458 int
459 bnx2x_dev_rx_init(struct rte_eth_dev *dev)
460 {
461         dev->rx_pkt_burst = bnx2x_recv_pkts;
462
463         return 0;
464 }
465
466 void
467 bnx2x_dev_clear_queues(struct rte_eth_dev *dev)
468 {
469         uint8_t i;
470
471         PMD_INIT_FUNC_TRACE();
472
473         for (i = 0; i < dev->data->nb_tx_queues; i++) {
474                 struct bnx2x_tx_queue *txq = dev->data->tx_queues[i];
475                 if (txq != NULL) {
476                         bnx2x_tx_queue_release(txq);
477                         dev->data->tx_queues[i] = NULL;
478                 }
479         }
480
481         for (i = 0; i < dev->data->nb_rx_queues; i++) {
482                 struct bnx2x_rx_queue *rxq = dev->data->rx_queues[i];
483                 if (rxq != NULL) {
484                         bnx2x_rx_queue_release(rxq);
485                         dev->data->rx_queues[i] = NULL;
486                 }
487         }
488 }