drivers/net/bnx2x/bnx2x_rxtx.c

   1 /*
   2  * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
   3  *
   4  * Copyright (c) 2015 QLogic Corporation.
   5  * All rights reserved.
   6  * www.qlogic.com
   7  *
   8  * See LICENSE.bnx2x_pmd for copyright and licensing details.
   9  */
  10
  11 #include "bnx2x.h"
  12 #include "bnx2x_rxtx.h"
  13
  14 static inline struct rte_mbuf *
  15 bnx2x_rxmbuf_alloc(struct rte_mempool *mp)
  16 {
  17         struct rte_mbuf *m;
  18
  19         m = __rte_mbuf_raw_alloc(mp);
  20         __rte_mbuf_sanity_check(m, 0);
  21
  22         return m;
  23 }
  24
  25 static const struct rte_memzone *
  26 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
  27                       uint16_t queue_id, uint32_t ring_size, int socket_id)
  28 {
  29         char z_name[RTE_MEMZONE_NAMESIZE];
  30         const struct rte_memzone *mz;
  31
  32         snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
  33                         dev->driver->pci_drv.name, ring_name, dev->data->port_id, queue_id);
  34
  35         mz = rte_memzone_lookup(z_name);
  36         if (mz)
  37                 return mz;
  38
  39         return rte_memzone_reserve_aligned(z_name, ring_size, socket_id, 0, BNX2X_PAGE_SIZE);
  40 }
  41
  42 static void
  43 bnx2x_rx_queue_release(struct bnx2x_rx_queue *rx_queue)
  44 {
  45         uint16_t i;
  46         struct rte_mbuf **sw_ring;
  47
  48         if (NULL != rx_queue) {
  49
  50                 sw_ring = rx_queue->sw_ring;
  51                 if (NULL != sw_ring) {
  52                         for (i = 0; i < rx_queue->nb_rx_desc; i++) {
  53                                 if (NULL != sw_ring[i])
  54                                         rte_pktmbuf_free(sw_ring[i]);
  55                         }
  56                         rte_free(sw_ring);
  57                 }
  58                 rte_free(rx_queue);
  59         }
  60 }
  61
  62 void
  63 bnx2x_dev_rx_queue_release(void *rxq)
  64 {
  65         bnx2x_rx_queue_release(rxq);
  66 }
  67
  68 int
  69 bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
  70                        uint16_t queue_idx,
  71                        uint16_t nb_desc,
  72                        unsigned int socket_id,
  73                        const struct rte_eth_rxconf *rx_conf,
  74                        struct rte_mempool *mp)
  75 {
  76         uint16_t j, idx;
  77         const struct rte_memzone *dma;
  78         struct bnx2x_rx_queue *rxq;
  79         uint32_t dma_size;
  80         struct rte_mbuf *mbuf;
  81         struct bnx2x_softc *sc = dev->data->dev_private;
  82         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
  83         struct eth_rx_cqe_next_page *nextpg;
  84         phys_addr_t *rx_bd;
  85         phys_addr_t busaddr;
  86
  87         /* First allocate the rx queue data structure */
  88         rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
  89                                  RTE_CACHE_LINE_SIZE, socket_id);
  90         if (NULL == rxq) {
  91                 PMD_INIT_LOG(ERR, "rte_zmalloc for rxq failed!");
  92                 return -ENOMEM;
  93         }
  94         rxq->sc = sc;
  95         rxq->mb_pool = mp;
  96         rxq->queue_id = queue_idx;
  97         rxq->port_id = dev->data->port_id;
  98         rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.hw_strip_crc) ? 0 : ETHER_CRC_LEN);
  99
 100         rxq->nb_rx_pages = 1;
 101         while (USABLE_RX_BD(rxq) < nb_desc)
 102                 rxq->nb_rx_pages <<= 1;
 103
 104         rxq->nb_rx_desc  = TOTAL_RX_BD(rxq);
 105         sc->rx_ring_size = USABLE_RX_BD(rxq);
 106         rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
 107
 108         rxq->rx_free_thresh = rx_conf->rx_free_thresh ?
 109                 rx_conf->rx_free_thresh : DEFAULT_RX_FREE_THRESH;
 110
 111         PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
 112                        "total_bd=%lu, rx_pages=%u, cq_pages=%u",
 113                        queue_idx, nb_desc, rxq->rx_free_thresh,
 114                        (unsigned long)USABLE_RX_BD(rxq),
 115                        (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
 116                        rxq->nb_cq_pages);
 117
 118         /* Allocate RX ring hardware descriptors */
 119         dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd);
 120         dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id);
 121         if (NULL == dma) {
 122                 PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!");
 123                 bnx2x_rx_queue_release(rxq);
 124                 return -ENOMEM;
 125         }
 126         fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->phys_addr;
 127         rxq->rx_ring = (uint64_t*)dma->addr;
 128         memset((void *)rxq->rx_ring, 0, dma_size);
 129
 130         /* Link the RX chain pages. */
 131         for (j = 1; j <= rxq->nb_rx_pages; j++) {
 132                 rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2];
 133                 busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages);
 134                 *rx_bd = busaddr;
 135         }
 136
 137         /* Allocate software ring */
 138         dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry);
 139         rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size,
 140                                           RTE_CACHE_LINE_SIZE,
 141                                           socket_id);
 142         if (NULL == rxq->sw_ring) {
 143                 PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!");
 144                 bnx2x_rx_queue_release(rxq);
 145                 return -ENOMEM;
 146         }
 147
 148         /* Initialize software ring entries */
 149         rxq->rx_mbuf_alloc = 0;
 150         for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) {
 151                 mbuf = bnx2x_rxmbuf_alloc(mp);
 152                 if (NULL == mbuf) {
 153                         PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d",
 154                                    (unsigned)rxq->queue_id, idx);
 155                         bnx2x_rx_queue_release(rxq);
 156                         return -ENOMEM;
 157                 }
 158                 rxq->sw_ring[idx] = mbuf;
 159                 rxq->rx_ring[idx] = mbuf->buf_physaddr;
 160                 rxq->rx_mbuf_alloc++;
 161         }
 162         rxq->pkt_first_seg = NULL;
 163         rxq->pkt_last_seg = NULL;
 164         rxq->rx_bd_head = 0;
 165         rxq->rx_bd_tail = rxq->nb_rx_desc;
 166
 167         /* Allocate CQ chain. */
 168         dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages;
 169         dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id);
 170         if (NULL == dma) {
 171                 PMD_RX_LOG(ERR, "RCQ  alloc failed");
 172                 return -ENOMEM;
 173         }
 174         fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->phys_addr;
 175         rxq->cq_ring = (union eth_rx_cqe*)dma->addr;
 176
 177         /* Link the CQ chain pages. */
 178         for (j = 1; j <= rxq->nb_cq_pages; j++) {
 179                 nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe;
 180                 busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages);
 181                 nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
 182                 nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
 183         }
 184         rxq->rx_cq_head = 0;
 185         rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq);
 186
 187         dev->data->rx_queues[queue_idx] = rxq;
 188         if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues;
 189
 190         return 0;
 191 }
 192
 193 static void
 194 bnx2x_tx_queue_release(struct bnx2x_tx_queue *tx_queue)
 195 {
 196         uint16_t i;
 197         struct rte_mbuf **sw_ring;
 198
 199         if (NULL != tx_queue) {
 200
 201                 sw_ring = tx_queue->sw_ring;
 202                 if (NULL != sw_ring) {
 203                         for (i = 0; i < tx_queue->nb_tx_desc; i++) {
 204                                 if (NULL != sw_ring[i])
 205                                         rte_pktmbuf_free(sw_ring[i]);
 206                         }
 207                         rte_free(sw_ring);
 208                 }
 209                 rte_free(tx_queue);
 210         }
 211 }
 212
 213 void
 214 bnx2x_dev_tx_queue_release(void *txq)
 215 {
 216         bnx2x_tx_queue_release(txq);
 217 }
 218
 219 static uint16_t
 220 bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 221 {
 222         struct bnx2x_tx_queue *txq;
 223         struct bnx2x_softc *sc;
 224         struct bnx2x_fastpath *fp;
 225         uint32_t burst, nb_tx;
 226         struct rte_mbuf **m = tx_pkts;
 227         int ret;
 228
 229         txq = p_txq;
 230         sc = txq->sc;
 231         fp = &sc->fp[txq->queue_id];
 232
 233         nb_tx = nb_pkts;
 234
 235         do {
 236                 burst = RTE_MIN(nb_pkts, RTE_PMD_BNX2X_TX_MAX_BURST);
 237
 238                 ret = bnx2x_tx_encap(txq, m, burst);
 239                 if (unlikely(ret)) {
 240                         PMD_TX_LOG(ERR, "tx_encap failed!");
 241                 }
 242
 243                 bnx2x_update_fp_sb_idx(fp);
 244
 245                 if ((txq->nb_tx_desc - txq->nb_tx_avail) > txq->tx_free_thresh) {
 246                         bnx2x_txeof(sc, fp);
 247                 }
 248
 249                 if (unlikely(ret == -ENOMEM)) {
 250                         break;
 251                 }
 252
 253                 m += burst;
 254                 nb_pkts -= burst;
 255
 256         } while (nb_pkts);
 257
 258         return nb_tx - nb_pkts;
 259 }
 260
 261 int
 262 bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
 263                        uint16_t queue_idx,
 264                        uint16_t nb_desc,
 265                        unsigned int socket_id,
 266                        const struct rte_eth_txconf *tx_conf)
 267 {
 268         uint16_t i;
 269         unsigned int tsize;
 270         const struct rte_memzone *tz;
 271         struct bnx2x_tx_queue *txq;
 272         struct eth_tx_next_bd *tx_n_bd;
 273         uint64_t busaddr;
 274         struct bnx2x_softc *sc = dev->data->dev_private;
 275         struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
 276
 277         /* First allocate the tx queue data structure */
 278         txq = rte_zmalloc("ethdev TX queue", sizeof(struct bnx2x_tx_queue),
 279                           RTE_CACHE_LINE_SIZE);
 280         if (txq == NULL)
 281                 return -ENOMEM;
 282         txq->sc = sc;
 283
 284         txq->nb_tx_pages = 1;
 285         while (USABLE_TX_BD(txq) < nb_desc)
 286                 txq->nb_tx_pages <<= 1;
 287
 288         txq->nb_tx_desc  = TOTAL_TX_BD(txq);
 289         sc->tx_ring_size = TOTAL_TX_BD(txq);
 290
 291         txq->tx_free_thresh = tx_conf->tx_free_thresh ?
 292                 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH;
 293
 294         PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
 295                      "total_bd=%lu, tx_pages=%u",
 296                      queue_idx, nb_desc, txq->tx_free_thresh,
 297                      (unsigned long)USABLE_TX_BD(txq),
 298                      (unsigned long)TOTAL_TX_BD(txq), txq->nb_tx_pages);
 299
 300         /* Allocate TX ring hardware descriptors */
 301         tsize = txq->nb_tx_desc * sizeof(union eth_tx_bd_types);
 302         tz = ring_dma_zone_reserve(dev, "tx_hw_ring", queue_idx, tsize, socket_id);
 303         if (tz == NULL) {
 304                 bnx2x_tx_queue_release(txq);
 305                 return -ENOMEM;
 306         }
 307         fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->phys_addr;
 308         txq->tx_ring = (union eth_tx_bd_types *) tz->addr;
 309         memset(txq->tx_ring, 0, tsize);
 310
 311         /* Allocate software ring */
 312         tsize = txq->nb_tx_desc * sizeof(struct rte_mbuf *);
 313         txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize,
 314                                    RTE_CACHE_LINE_SIZE);
 315         if (txq->sw_ring == NULL) {
 316                 bnx2x_tx_queue_release(txq);
 317                 return -ENOMEM;
 318         }
 319
 320         /* PMD_DRV_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
 321            txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */
 322
 323         /* Link TX pages */
 324         for (i = 1; i <= txq->nb_tx_pages; i++) {
 325                 tx_n_bd = &txq->tx_ring[TOTAL_TX_BD_PER_PAGE * i - 1].next_bd;
 326                 busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages);
 327                 tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
 328                 tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
 329                 /* PMD_DRV_LOG(DEBUG, "link tx page %lu", (TOTAL_TX_BD_PER_PAGE * i - 1)); */
 330         }
 331
 332         txq->queue_id = queue_idx;
 333         txq->port_id = dev->data->port_id;
 334         txq->tx_pkt_tail = 0;
 335         txq->tx_pkt_head = 0;
 336         txq->tx_bd_tail = 0;
 337         txq->tx_bd_head = 0;
 338         txq->nb_tx_avail = txq->nb_tx_desc;
 339         dev->tx_pkt_burst = bnx2x_xmit_pkts;
 340         dev->data->tx_queues[queue_idx] = txq;
 341         if (!sc->tx_queues) sc->tx_queues = dev->data->tx_queues;
 342
 343         return 0;
 344 }
 345
 346 static inline void
 347 bnx2x_upd_rx_prod_fast(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
 348                 uint16_t rx_bd_prod, uint16_t rx_cq_prod)
 349 {
 350         union ustorm_eth_rx_producers rx_prods;
 351
 352         rx_prods.prod.bd_prod  = rx_bd_prod;
 353         rx_prods.prod.cqe_prod = rx_cq_prod;
 354
 355         REG_WR(sc, fp->ustorm_rx_prods_offset, rx_prods.raw_data[0]);
 356 }
 357
 358 static uint16_t
 359 bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 360 {
 361         struct bnx2x_rx_queue *rxq = p_rxq;
 362         struct bnx2x_softc *sc = rxq->sc;
 363         struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id];
 364         uint32_t nb_rx = 0;
 365         uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod;
 366         uint16_t bd_cons, bd_prod;
 367         struct rte_mbuf *new_mb;
 368         uint16_t rx_pref;
 369         struct eth_fast_path_rx_cqe *cqe_fp;
 370         uint16_t len, pad;
 371         struct rte_mbuf *rx_mb = NULL;
 372
 373         hw_cq_cons = le16toh(*fp->rx_cq_cons_sb);
 374         if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) ==
 375                         USABLE_RCQ_ENTRIES_PER_PAGE) {
 376                 ++hw_cq_cons;
 377         }
 378
 379         bd_cons = rxq->rx_bd_head;
 380         bd_prod = rxq->rx_bd_tail;
 381         sw_cq_cons = rxq->rx_cq_head;
 382         sw_cq_prod = rxq->rx_cq_tail;
 383
 384         if (sw_cq_cons == hw_cq_cons)
 385                 return 0;
 386
 387         while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) {
 388
 389                 bd_prod &= MAX_RX_BD(rxq);
 390                 bd_cons &= MAX_RX_BD(rxq);
 391
 392                 cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe;
 393
 394                 if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) {
 395                         PMD_RX_LOG(ERR, "slowpath event during traffic processing");
 396                         break;
 397                 }
 398
 399                 if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) {
 400                         PMD_RX_LOG(ERR, "flags 0x%x rx packet %u",
 401                                         cqe_fp->type_error_flags, sw_cq_cons);
 402                         goto next_rx;
 403                 }
 404
 405                 len = cqe_fp->pkt_len_or_gro_seg_len;
 406                 pad = cqe_fp->placement_offset;
 407
 408                 new_mb = bnx2x_rxmbuf_alloc(rxq->mb_pool);
 409                 if (unlikely(!new_mb)) {
 410                         PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index);
 411                         goto next_rx;
 412                 }
 413
 414                 rx_mb = rxq->sw_ring[bd_cons];
 415                 rxq->sw_ring[bd_cons] = new_mb;
 416                 rxq->rx_ring[bd_prod] = new_mb->buf_physaddr;
 417
 418                 rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq);
 419                 rte_prefetch0(rxq->sw_ring[rx_pref]);
 420                 if ((rx_pref & 0x3) == 0) {
 421                         rte_prefetch0(&rxq->rx_ring[rx_pref]);
 422                         rte_prefetch0(&rxq->sw_ring[rx_pref]);
 423                 }
 424
 425                 rx_mb->data_off = pad;
 426                 rx_mb->nb_segs = 1;
 427                 rx_mb->next = NULL;
 428                 rx_mb->pkt_len = rx_mb->data_len = len;
 429                 rx_mb->port = rxq->port_id;
 430                 rx_mb->buf_len = len + pad;
 431                 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
 432
 433                 /*
 434                  * If we received a packet with a vlan tag,
 435                  * attach that information to the packet.
 436                  */
 437                 if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) {
 438                         rx_mb->vlan_tci = cqe_fp->vlan_tag;
 439                         rx_mb->ol_flags |= PKT_RX_VLAN_PKT;
 440                 }
 441
 442                 rx_pkts[nb_rx] = rx_mb;
 443                 nb_rx++;
 444
 445                 /* limit spinning on the queue */
 446                 if (unlikely(nb_rx == sc->rx_budget)) {
 447                         PMD_RX_LOG(ERR, "Limit spinning on the queue");
 448                         break;
 449                 }
 450
 451 next_rx:
 452                 bd_cons    = NEXT_RX_BD(bd_cons);
 453                 bd_prod    = NEXT_RX_BD(bd_prod);
 454                 sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod);
 455                 sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons);
 456         }
 457         rxq->rx_bd_head = bd_cons;
 458         rxq->rx_bd_tail = bd_prod;
 459         rxq->rx_cq_head = sw_cq_cons;
 460         rxq->rx_cq_tail = sw_cq_prod;
 461
 462         bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod);
 463
 464         return nb_rx;
 465 }
 466
 467 int
 468 bnx2x_dev_rx_init(struct rte_eth_dev *dev)
 469 {
 470         dev->rx_pkt_burst = bnx2x_recv_pkts;
 471
 472         return 0;
 473 }
 474
 475 void
 476 bnx2x_dev_clear_queues(struct rte_eth_dev *dev)
 477 {
 478         uint8_t i;
 479
 480         PMD_INIT_FUNC_TRACE();
 481
 482         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 483                 struct bnx2x_tx_queue *txq = dev->data->tx_queues[i];
 484                 if (txq != NULL) {
 485                         bnx2x_tx_queue_release(txq);
 486                         dev->data->tx_queues[i] = NULL;
 487                 }
 488         }
 489
 490         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 491                 struct bnx2x_rx_queue *rxq = dev->data->rx_queues[i];
 492                 if (rxq != NULL) {
 493                         bnx2x_rx_queue_release(rxq);
 494                         dev->data->rx_queues[i] = NULL;
 495                 }
 496         }
 497 }