drivers/net/bnx2x/bnx2x_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
   3  * Copyright (c) 2015-2018 Cavium Inc.
   4  * All rights reserved.
   5  * www.cavium.com
   6  */
   7
   8 #include "bnx2x.h"
   9 #include "bnx2x_rxtx.h"
  10
  11 static const struct rte_memzone *
  12 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
  13                       uint16_t queue_id, uint32_t ring_size, int socket_id)
  14 {
  15         return rte_eth_dma_zone_reserve(dev, ring_name, queue_id,
  16                         ring_size, BNX2X_PAGE_SIZE, socket_id);
  17 }
  18
  19 static void
  20 bnx2x_rx_queue_release(struct bnx2x_rx_queue *rx_queue)
  21 {
  22         uint16_t i;
  23         struct rte_mbuf **sw_ring;
  24
  25         if (NULL != rx_queue) {
  26
  27                 sw_ring = rx_queue->sw_ring;
  28                 if (NULL != sw_ring) {
  29                         for (i = 0; i < rx_queue->nb_rx_desc; i++) {
  30                                 if (NULL != sw_ring[i])
  31                                         rte_pktmbuf_free(sw_ring[i]);
  32                         }
  33                         rte_free(sw_ring);
  34                 }
  35                 rte_free(rx_queue);
  36         }
  37 }
  38
  39 void
  40 bnx2x_dev_rx_queue_release(void *rxq)
  41 {
  42         bnx2x_rx_queue_release(rxq);
  43 }
  44
  45 int
  46 bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
  47                        uint16_t queue_idx,
  48                        uint16_t nb_desc,
  49                        unsigned int socket_id,
  50                        __rte_unused const struct rte_eth_rxconf *rx_conf,
  51                        struct rte_mempool *mp)
  52 {
  53         uint16_t j, idx;
  54         const struct rte_memzone *dma;
  55         struct bnx2x_rx_queue *rxq;
  56         uint32_t dma_size;
  57         struct rte_mbuf *mbuf;
  58         struct bnx2x_softc *sc = dev->data->dev_private;
  59         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
  60         struct eth_rx_cqe_next_page *nextpg;
  61         rte_iova_t *rx_bd;
  62         rte_iova_t busaddr;
  63
  64         /* First allocate the rx queue data structure */
  65         rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
  66                                  RTE_CACHE_LINE_SIZE, socket_id);
  67         if (NULL == rxq) {
  68                 PMD_DRV_LOG(ERR, sc, "rte_zmalloc for rxq failed!");
  69                 return -ENOMEM;
  70         }
  71         rxq->sc = sc;
  72         rxq->mb_pool = mp;
  73         rxq->queue_id = queue_idx;
  74         rxq->port_id = dev->data->port_id;
  75
  76         rxq->nb_rx_pages = 1;
  77         while (USABLE_RX_BD(rxq) < nb_desc)
  78                 rxq->nb_rx_pages <<= 1;
  79
  80         rxq->nb_rx_desc  = TOTAL_RX_BD(rxq);
  81         sc->rx_ring_size = USABLE_RX_BD(rxq);
  82         rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
  83
  84         PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, usable_bd=%lu, "
  85                        "total_bd=%lu, rx_pages=%u, cq_pages=%u",
  86                        queue_idx, nb_desc, (unsigned long)USABLE_RX_BD(rxq),
  87                        (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
  88                        rxq->nb_cq_pages);
  89
  90         /* Allocate RX ring hardware descriptors */
  91         dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd);
  92         dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id);
  93         if (NULL == dma) {
  94                 PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!");
  95                 bnx2x_rx_queue_release(rxq);
  96                 return -ENOMEM;
  97         }
  98         fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->iova;
  99         rxq->rx_ring = (uint64_t*)dma->addr;
 100         memset((void *)rxq->rx_ring, 0, dma_size);
 101
 102         /* Link the RX chain pages. */
 103         for (j = 1; j <= rxq->nb_rx_pages; j++) {
 104                 rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2];
 105                 busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages);
 106                 *rx_bd = busaddr;
 107         }
 108
 109         /* Allocate software ring */
 110         dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry);
 111         rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size,
 112                                           RTE_CACHE_LINE_SIZE,
 113                                           socket_id);
 114         if (NULL == rxq->sw_ring) {
 115                 PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!");
 116                 bnx2x_rx_queue_release(rxq);
 117                 return -ENOMEM;
 118         }
 119
 120         /* Initialize software ring entries */
 121         for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) {
 122                 mbuf = rte_mbuf_raw_alloc(mp);
 123                 if (NULL == mbuf) {
 124                         PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d",
 125                                    (unsigned)rxq->queue_id, idx);
 126                         bnx2x_rx_queue_release(rxq);
 127                         return -ENOMEM;
 128                 }
 129                 rxq->sw_ring[idx] = mbuf;
 130                 rxq->rx_ring[idx] =
 131                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
 132         }
 133         rxq->pkt_first_seg = NULL;
 134         rxq->pkt_last_seg = NULL;
 135         rxq->rx_bd_head = 0;
 136         rxq->rx_bd_tail = rxq->nb_rx_desc;
 137
 138         /* Allocate CQ chain. */
 139         dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages;
 140         dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id);
 141         if (NULL == dma) {
 142                 PMD_RX_LOG(ERR, "RCQ  alloc failed");
 143                 return -ENOMEM;
 144         }
 145         fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->iova;
 146         rxq->cq_ring = (union eth_rx_cqe*)dma->addr;
 147
 148         /* Link the CQ chain pages. */
 149         for (j = 1; j <= rxq->nb_cq_pages; j++) {
 150                 nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe;
 151                 busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages);
 152                 nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
 153                 nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
 154         }
 155         rxq->rx_cq_head = 0;
 156         rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq);
 157
 158         dev->data->rx_queues[queue_idx] = rxq;
 159         if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues;
 160
 161         return 0;
 162 }
 163
 164 static void
 165 bnx2x_tx_queue_release(struct bnx2x_tx_queue *tx_queue)
 166 {
 167         uint16_t i;
 168         struct rte_mbuf **sw_ring;
 169
 170         if (NULL != tx_queue) {
 171
 172                 sw_ring = tx_queue->sw_ring;
 173                 if (NULL != sw_ring) {
 174                         for (i = 0; i < tx_queue->nb_tx_desc; i++) {
 175                                 if (NULL != sw_ring[i])
 176                                         rte_pktmbuf_free(sw_ring[i]);
 177                         }
 178                         rte_free(sw_ring);
 179                 }
 180                 rte_free(tx_queue);
 181         }
 182 }
 183
 184 void
 185 bnx2x_dev_tx_queue_release(void *txq)
 186 {
 187         bnx2x_tx_queue_release(txq);
 188 }
 189
 190 static uint16_t
 191 bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 192 {
 193         struct bnx2x_tx_queue *txq;
 194         struct bnx2x_softc *sc;
 195         struct bnx2x_fastpath *fp;
 196         uint16_t nb_tx_pkts;
 197         uint16_t nb_pkt_sent = 0;
 198         uint32_t ret;
 199
 200         txq = p_txq;
 201         sc = txq->sc;
 202         fp = &sc->fp[txq->queue_id];
 203
 204         if ((unlikely((txq->nb_tx_desc - txq->nb_tx_avail) >
 205                                 txq->tx_free_thresh)))
 206                 bnx2x_txeof(sc, fp);
 207
 208         nb_tx_pkts = RTE_MIN(nb_pkts, txq->nb_tx_avail / BDS_PER_TX_PKT);
 209         if (unlikely(nb_tx_pkts == 0))
 210                 return 0;
 211
 212         while (nb_tx_pkts--) {
 213                 struct rte_mbuf *m = *tx_pkts++;
 214                 assert(m != NULL);
 215                 ret = bnx2x_tx_encap(txq, m);
 216                 fp->tx_db.data.prod += ret;
 217                 nb_pkt_sent++;
 218         }
 219
 220         bnx2x_update_fp_sb_idx(fp);
 221         mb();
 222         DOORBELL(sc, txq->queue_id, fp->tx_db.raw);
 223         mb();
 224
 225         if ((txq->nb_tx_desc - txq->nb_tx_avail) >
 226                                 txq->tx_free_thresh)
 227                 bnx2x_txeof(sc, fp);
 228
 229         return nb_pkt_sent;
 230 }
 231
 232 int
 233 bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
 234                        uint16_t queue_idx,
 235                        uint16_t nb_desc,
 236                        unsigned int socket_id,
 237                        const struct rte_eth_txconf *tx_conf)
 238 {
 239         uint16_t i;
 240         unsigned int tsize;
 241         const struct rte_memzone *tz;
 242         struct bnx2x_tx_queue *txq;
 243         struct eth_tx_next_bd *tx_n_bd;
 244         uint64_t busaddr;
 245         struct bnx2x_softc *sc = dev->data->dev_private;
 246         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
 247
 248         /* First allocate the tx queue data structure */
 249         txq = rte_zmalloc("ethdev TX queue", sizeof(struct bnx2x_tx_queue),
 250                           RTE_CACHE_LINE_SIZE);
 251         if (txq == NULL)
 252                 return -ENOMEM;
 253         txq->sc = sc;
 254
 255         txq->nb_tx_pages = 1;
 256         while (USABLE_TX_BD(txq) < nb_desc)
 257                 txq->nb_tx_pages <<= 1;
 258
 259         txq->nb_tx_desc  = TOTAL_TX_BD(txq);
 260         sc->tx_ring_size = TOTAL_TX_BD(txq);
 261
 262         txq->tx_free_thresh = tx_conf->tx_free_thresh ?
 263                 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH;
 264         txq->tx_free_thresh = min(txq->tx_free_thresh,
 265                                   txq->nb_tx_desc - BDS_PER_TX_PKT);
 266
 267         PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
 268                      "total_bd=%lu, tx_pages=%u",
 269                      queue_idx, nb_desc, txq->tx_free_thresh,
 270                      (unsigned long)USABLE_TX_BD(txq),
 271                      (unsigned long)TOTAL_TX_BD(txq), txq->nb_tx_pages);
 272
 273         /* Allocate TX ring hardware descriptors */
 274         tsize = txq->nb_tx_desc * sizeof(union eth_tx_bd_types);
 275         tz = ring_dma_zone_reserve(dev, "tx_hw_ring", queue_idx, tsize, socket_id);
 276         if (tz == NULL) {
 277                 bnx2x_tx_queue_release(txq);
 278                 return -ENOMEM;
 279         }
 280         fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->iova;
 281         txq->tx_ring = (union eth_tx_bd_types *) tz->addr;
 282         memset(txq->tx_ring, 0, tsize);
 283
 284         /* Allocate software ring */
 285         tsize = txq->nb_tx_desc * sizeof(struct rte_mbuf *);
 286         txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize,
 287                                    RTE_CACHE_LINE_SIZE);
 288         if (txq->sw_ring == NULL) {
 289                 bnx2x_tx_queue_release(txq);
 290                 return -ENOMEM;
 291         }
 292
 293         /* PMD_DRV_LOG(DEBUG, sc, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
 294            txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */
 295
 296         /* Link TX pages */
 297         for (i = 1; i <= txq->nb_tx_pages; i++) {
 298                 tx_n_bd = &txq->tx_ring[TOTAL_TX_BD_PER_PAGE * i - 1].next_bd;
 299                 busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages);
 300                 tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
 301                 tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
 302                 /* PMD_DRV_LOG(DEBUG, sc, "link tx page %lu",
 303                  *          (TOTAL_TX_BD_PER_PAGE * i - 1));
 304                  */
 305         }
 306
 307         txq->queue_id = queue_idx;
 308         txq->port_id = dev->data->port_id;
 309         txq->tx_pkt_tail = 0;
 310         txq->tx_pkt_head = 0;
 311         txq->tx_bd_tail = 0;
 312         txq->tx_bd_head = 0;
 313         txq->nb_tx_avail = txq->nb_tx_desc;
 314         dev->data->tx_queues[queue_idx] = txq;
 315         if (!sc->tx_queues) sc->tx_queues = dev->data->tx_queues;
 316
 317         return 0;
 318 }
 319
 320 static inline void
 321 bnx2x_upd_rx_prod_fast(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
 322                 uint16_t rx_bd_prod, uint16_t rx_cq_prod)
 323 {
 324         struct ustorm_eth_rx_producers rx_prods = { 0 };
 325         uint32_t *val = NULL;
 326
 327         rx_prods.bd_prod  = rx_bd_prod;
 328         rx_prods.cqe_prod = rx_cq_prod;
 329
 330         val = (uint32_t *)&rx_prods;
 331         REG_WR(sc, fp->ustorm_rx_prods_offset, val[0]);
 332 }
 333
 334 static uint16_t
 335 bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 336 {
 337         struct bnx2x_rx_queue *rxq = p_rxq;
 338         struct bnx2x_softc *sc = rxq->sc;
 339         struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id];
 340         uint32_t nb_rx = 0;
 341         uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod;
 342         uint16_t bd_cons, bd_prod;
 343         struct rte_mbuf *new_mb;
 344         uint16_t rx_pref;
 345         struct eth_fast_path_rx_cqe *cqe_fp;
 346         uint16_t len, pad, bd_len, buf_len;
 347         struct rte_mbuf *rx_mb = NULL;
 348         static bool log_once = true;
 349
 350         rte_spinlock_lock(&(fp)->rx_mtx);
 351
 352         hw_cq_cons = le16toh(*fp->rx_cq_cons_sb);
 353         if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) ==
 354                         USABLE_RCQ_ENTRIES_PER_PAGE) {
 355                 ++hw_cq_cons;
 356         }
 357
 358         bd_cons = rxq->rx_bd_head;
 359         bd_prod = rxq->rx_bd_tail;
 360         sw_cq_cons = rxq->rx_cq_head;
 361         sw_cq_prod = rxq->rx_cq_tail;
 362
 363         if (sw_cq_cons == hw_cq_cons) {
 364                 rte_spinlock_unlock(&(fp)->rx_mtx);
 365                 return 0;
 366         }
 367
 368         while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) {
 369
 370                 bd_prod &= MAX_RX_BD(rxq);
 371                 bd_cons &= MAX_RX_BD(rxq);
 372
 373                 cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe;
 374
 375                 if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) {
 376                         PMD_RX_LOG(ERR, "slowpath event during traffic processing");
 377                         break;
 378                 }
 379
 380                 if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) {
 381                         PMD_RX_LOG(ERR, "flags 0x%x rx packet %u",
 382                                         cqe_fp->type_error_flags, sw_cq_cons);
 383                         goto next_rx;
 384                 }
 385
 386                 len = cqe_fp->pkt_len_or_gro_seg_len;
 387                 pad = cqe_fp->placement_offset;
 388                 bd_len = cqe_fp->len_on_bd;
 389                 buf_len = rxq->sw_ring[bd_cons]->buf_len;
 390
 391                 /* Check for sufficient buffer length */
 392                 if (unlikely(buf_len < len + (pad + RTE_PKTMBUF_HEADROOM))) {
 393                         if (unlikely(log_once)) {
 394                                 PMD_DRV_LOG(ERR, sc, "mbuf size %d is not enough to hold Rx packet length more than %d",
 395                                             buf_len - RTE_PKTMBUF_HEADROOM,
 396                                             buf_len -
 397                                             (pad + RTE_PKTMBUF_HEADROOM));
 398                                 log_once = false;
 399                         }
 400                         goto next_rx;
 401                 }
 402
 403                 new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
 404                 if (unlikely(!new_mb)) {
 405                         PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index);
 406                         rte_eth_devices[rxq->port_id].data->
 407                                         rx_mbuf_alloc_failed++;
 408                         goto next_rx;
 409                 }
 410
 411                 rx_mb = rxq->sw_ring[bd_cons];
 412                 rxq->sw_ring[bd_cons] = new_mb;
 413                 rxq->rx_ring[bd_prod] =
 414                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(new_mb));
 415
 416                 rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq);
 417                 rte_prefetch0(rxq->sw_ring[rx_pref]);
 418                 if ((rx_pref & 0x3) == 0) {
 419                         rte_prefetch0(&rxq->rx_ring[rx_pref]);
 420                         rte_prefetch0(&rxq->sw_ring[rx_pref]);
 421                 }
 422
 423                 rx_mb->data_off = pad + RTE_PKTMBUF_HEADROOM;
 424                 rx_mb->nb_segs = 1;
 425                 rx_mb->next = NULL;
 426                 rx_mb->pkt_len = len;
 427                 rx_mb->data_len = bd_len;
 428                 rx_mb->port = rxq->port_id;
 429                 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
 430
 431                 /*
 432                  * If we received a packet with a vlan tag,
 433                  * attach that information to the packet.
 434                  */
 435                 if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) {
 436                         rx_mb->vlan_tci = cqe_fp->vlan_tag;
 437                         rx_mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 438                 }
 439
 440                 rx_pkts[nb_rx] = rx_mb;
 441                 nb_rx++;
 442
 443                 /* limit spinning on the queue */
 444                 if (unlikely(nb_rx == sc->rx_budget)) {
 445                         PMD_RX_LOG(ERR, "Limit spinning on the queue");
 446                         break;
 447                 }
 448
 449 next_rx:
 450                 bd_cons    = NEXT_RX_BD(bd_cons);
 451                 bd_prod    = NEXT_RX_BD(bd_prod);
 452                 sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod);
 453                 sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons);
 454         }
 455         rxq->rx_bd_head = bd_cons;
 456         rxq->rx_bd_tail = bd_prod;
 457         rxq->rx_cq_head = sw_cq_cons;
 458         rxq->rx_cq_tail = sw_cq_prod;
 459
 460         bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod);
 461
 462         rte_spinlock_unlock(&(fp)->rx_mtx);
 463
 464         return nb_rx;
 465 }
 466
 467 static uint16_t
 468 bnx2x_rxtx_pkts_dummy(__rte_unused void *p_rxq,
 469                       __rte_unused struct rte_mbuf **rx_pkts,
 470                       __rte_unused uint16_t nb_pkts)
 471 {
 472         return 0;
 473 }
 474
 475 void bnx2x_dev_rxtx_init_dummy(struct rte_eth_dev *dev)
 476 {
 477         dev->rx_pkt_burst = bnx2x_rxtx_pkts_dummy;
 478         dev->tx_pkt_burst = bnx2x_rxtx_pkts_dummy;
 479 }
 480
 481 void bnx2x_dev_rxtx_init(struct rte_eth_dev *dev)
 482 {
 483         dev->rx_pkt_burst = bnx2x_recv_pkts;
 484         dev->tx_pkt_burst = bnx2x_xmit_pkts;
 485 }
 486
 487 void
 488 bnx2x_dev_clear_queues(struct rte_eth_dev *dev)
 489 {
 490         struct bnx2x_softc *sc = dev->data->dev_private;
 491         uint8_t i;
 492
 493         PMD_INIT_FUNC_TRACE(sc);
 494
 495         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 496                 struct bnx2x_tx_queue *txq = dev->data->tx_queues[i];
 497                 if (txq != NULL) {
 498                         bnx2x_tx_queue_release(txq);
 499                         dev->data->tx_queues[i] = NULL;
 500                 }
 501         }
 502
 503         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 504                 struct bnx2x_rx_queue *rxq = dev->data->rx_queues[i];
 505                 if (rxq != NULL) {
 506                         bnx2x_rx_queue_release(rxq);
 507                         dev->data->rx_queues[i] = NULL;
 508                 }
 509         }
 510 }