drivers/net/atlantic/atl_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Aquantia Corporation
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "atl_ethdev.h"
   9 #include "atl_hw_regs.h"
  10
  11 #include "atl_logs.h"
  12 #include "hw_atl/hw_atl_llh.h"
  13 #include "hw_atl/hw_atl_b0.h"
  14 #include "hw_atl/hw_atl_b0_internal.h"
  15
  16 /**
  17  * Structure associated with each descriptor of the RX ring of a RX queue.
  18  */
  19 struct atl_rx_entry {
  20         struct rte_mbuf *mbuf;
  21 };
  22
  23 /**
  24  * Structure associated with each RX queue.
  25  */
  26 struct atl_rx_queue {
  27         struct rte_mempool      *mb_pool;
  28         struct hw_atl_rxd_s     *hw_ring;
  29         uint64_t                hw_ring_phys_addr;
  30         struct atl_rx_entry     *sw_ring;
  31         uint16_t                nb_rx_desc;
  32         uint16_t                rx_tail;
  33         uint16_t                nb_rx_hold;
  34         uint16_t                rx_free_thresh;
  35         uint16_t                queue_id;
  36         uint16_t                port_id;
  37         uint16_t                buff_size;
  38         bool                    l3_csum_enabled;
  39         bool                    l4_csum_enabled;
  40 };
  41
  42 static inline void
  43 atl_reset_rx_queue(struct atl_rx_queue *rxq)
  44 {
  45         struct hw_atl_rxd_s *rxd = NULL;
  46         int i;
  47
  48         PMD_INIT_FUNC_TRACE();
  49
  50         for (i = 0; i < rxq->nb_rx_desc; i++) {
  51                 rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i];
  52                 rxd->buf_addr = 0;
  53                 rxd->hdr_addr = 0;
  54         }
  55
  56         rxq->rx_tail = 0;
  57 }
  58
  59 int
  60 atl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
  61                    uint16_t nb_rx_desc, unsigned int socket_id,
  62                    const struct rte_eth_rxconf *rx_conf,
  63                    struct rte_mempool *mb_pool)
  64 {
  65         struct atl_rx_queue *rxq;
  66         const struct rte_memzone *mz;
  67
  68         PMD_INIT_FUNC_TRACE();
  69
  70         /* make sure a valid number of descriptors have been requested */
  71         if (nb_rx_desc < AQ_HW_MIN_RX_RING_SIZE ||
  72                         nb_rx_desc > AQ_HW_MAX_RX_RING_SIZE) {
  73                 PMD_INIT_LOG(ERR, "Number of Rx descriptors must be "
  74                 "less than or equal to %d, "
  75                 "greater than or equal to %d", AQ_HW_MAX_RX_RING_SIZE,
  76                 AQ_HW_MIN_RX_RING_SIZE);
  77                 return -EINVAL;
  78         }
  79
  80         /*
  81          * if this queue existed already, free the associated memory. The
  82          * queue cannot be reused in case we need to allocate memory on
  83          * different socket than was previously used.
  84          */
  85         if (dev->data->rx_queues[rx_queue_id] != NULL) {
  86                 atl_rx_queue_release(dev->data->rx_queues[rx_queue_id]);
  87                 dev->data->rx_queues[rx_queue_id] = NULL;
  88         }
  89
  90         /* allocate memory for the queue structure */
  91         rxq = rte_zmalloc_socket("atlantic Rx queue", sizeof(*rxq),
  92                                  RTE_CACHE_LINE_SIZE, socket_id);
  93         if (rxq == NULL) {
  94                 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
  95                 return -ENOMEM;
  96         }
  97
  98         /* setup queue */
  99         rxq->mb_pool = mb_pool;
 100         rxq->nb_rx_desc = nb_rx_desc;
 101         rxq->port_id = dev->data->port_id;
 102         rxq->queue_id = rx_queue_id;
 103         rxq->rx_free_thresh = rx_conf->rx_free_thresh;
 104
 105         rxq->l3_csum_enabled = dev->data->dev_conf.rxmode.offloads &
 106                 DEV_RX_OFFLOAD_IPV4_CKSUM;
 107         rxq->l4_csum_enabled = dev->data->dev_conf.rxmode.offloads &
 108                 (DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM);
 109         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC)
 110                 PMD_DRV_LOG(ERR, "PMD does not support KEEP_CRC offload");
 111
 112         /* allocate memory for the software ring */
 113         rxq->sw_ring = rte_zmalloc_socket("atlantic sw rx ring",
 114                                 nb_rx_desc * sizeof(struct atl_rx_entry),
 115                                 RTE_CACHE_LINE_SIZE, socket_id);
 116         if (rxq->sw_ring == NULL) {
 117                 PMD_INIT_LOG(ERR,
 118                         "Port %d: Cannot allocate software ring for queue %d",
 119                         rxq->port_id, rxq->queue_id);
 120                 rte_free(rxq);
 121                 return -ENOMEM;
 122         }
 123
 124         /*
 125          * allocate memory for the hardware descriptor ring. A memzone large
 126          * enough to hold the maximum ring size is requested to allow for
 127          * resizing in later calls to the queue setup function.
 128          */
 129         mz = rte_eth_dma_zone_reserve(dev, "rx hw_ring", rx_queue_id,
 130                                       HW_ATL_B0_MAX_RXD *
 131                                         sizeof(struct hw_atl_rxd_s),
 132                                       128, socket_id);
 133         if (mz == NULL) {
 134                 PMD_INIT_LOG(ERR,
 135                         "Port %d: Cannot allocate hardware ring for queue %d",
 136                         rxq->port_id, rxq->queue_id);
 137                 rte_free(rxq->sw_ring);
 138                 rte_free(rxq);
 139                 return -ENOMEM;
 140         }
 141         rxq->hw_ring = mz->addr;
 142         rxq->hw_ring_phys_addr = mz->iova;
 143
 144         atl_reset_rx_queue(rxq);
 145
 146         dev->data->rx_queues[rx_queue_id] = rxq;
 147         return 0;
 148 }
 149
 150 int
 151 atl_tx_init(struct rte_eth_dev *eth_dev __rte_unused)
 152 {
 153         return 0;
 154 }
 155
 156 int
 157 atl_rx_init(struct rte_eth_dev *eth_dev)
 158 {
 159         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 160         struct atl_rx_queue *rxq;
 161         uint64_t base_addr = 0;
 162         int i = 0;
 163         int err = 0;
 164
 165         PMD_INIT_FUNC_TRACE();
 166
 167         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 168                 rxq = eth_dev->data->rx_queues[i];
 169                 base_addr = rxq->hw_ring_phys_addr;
 170
 171                 /* Take requested pool mbuf size and adapt
 172                  * descriptor buffer to best fit
 173                  */
 174                 int buff_size = rte_pktmbuf_data_room_size(rxq->mb_pool) -
 175                                 RTE_PKTMBUF_HEADROOM;
 176
 177                 buff_size = RTE_ALIGN_FLOOR(buff_size, 1024);
 178                 if (buff_size > HW_ATL_B0_RXD_BUF_SIZE_MAX) {
 179                         PMD_INIT_LOG(WARNING,
 180                                 "Port %d queue %d: mem pool buff size is too big\n",
 181                                 rxq->port_id, rxq->queue_id);
 182                         buff_size = HW_ATL_B0_RXD_BUF_SIZE_MAX;
 183                 }
 184                 if (buff_size < 1024) {
 185                         PMD_INIT_LOG(ERR,
 186                                 "Port %d queue %d: mem pool buff size is too small\n",
 187                                 rxq->port_id, rxq->queue_id);
 188                         return -EINVAL;
 189                 }
 190                 rxq->buff_size = buff_size;
 191
 192                 err = hw_atl_b0_hw_ring_rx_init(hw, base_addr, rxq->queue_id,
 193                                                 rxq->nb_rx_desc, buff_size, 0,
 194                                                 rxq->port_id);
 195
 196                 if (err) {
 197                         PMD_INIT_LOG(ERR, "Port %d: Cannot init RX queue %d",
 198                                      rxq->port_id, rxq->queue_id);
 199                         break;
 200                 }
 201         }
 202
 203         return err;
 204 }
 205
 206 static int
 207 atl_alloc_rx_queue_mbufs(struct atl_rx_queue *rxq)
 208 {
 209         struct atl_rx_entry *rx_entry = rxq->sw_ring;
 210         struct hw_atl_rxd_s *rxd;
 211         uint64_t dma_addr = 0;
 212         uint32_t i = 0;
 213
 214         PMD_INIT_FUNC_TRACE();
 215
 216         /* fill Rx ring */
 217         for (i = 0; i < rxq->nb_rx_desc; i++) {
 218                 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
 219
 220                 if (mbuf == NULL) {
 221                         PMD_INIT_LOG(ERR,
 222                                 "Port %d: mbuf alloc failed for rx queue %d",
 223                                 rxq->port_id, rxq->queue_id);
 224                         return -ENOMEM;
 225                 }
 226
 227                 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
 228                 mbuf->port = rxq->port_id;
 229
 230                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
 231                 rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i];
 232                 rxd->buf_addr = dma_addr;
 233                 rxd->hdr_addr = 0;
 234                 rx_entry[i].mbuf = mbuf;
 235         }
 236
 237         return 0;
 238 }
 239
 240 static void
 241 atl_rx_queue_release_mbufs(struct atl_rx_queue *rxq)
 242 {
 243         int i;
 244
 245         PMD_INIT_FUNC_TRACE();
 246
 247         if (rxq->sw_ring != NULL) {
 248                 for (i = 0; i < rxq->nb_rx_desc; i++) {
 249                         if (rxq->sw_ring[i].mbuf != NULL) {
 250                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
 251                                 rxq->sw_ring[i].mbuf = NULL;
 252                         }
 253                 }
 254         }
 255 }
 256
 257 int
 258 atl_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 259 {
 260         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 261         struct atl_rx_queue *rxq = NULL;
 262
 263         PMD_INIT_FUNC_TRACE();
 264
 265         if (rx_queue_id < dev->data->nb_rx_queues) {
 266                 rxq = dev->data->rx_queues[rx_queue_id];
 267
 268                 if (atl_alloc_rx_queue_mbufs(rxq) != 0) {
 269                         PMD_INIT_LOG(ERR,
 270                                 "Port %d: Allocate mbufs for queue %d failed",
 271                                 rxq->port_id, rxq->queue_id);
 272                         return -1;
 273                 }
 274
 275                 hw_atl_b0_hw_ring_rx_start(hw, rx_queue_id);
 276
 277                 rte_wmb();
 278                 hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, rxq->nb_rx_desc - 1,
 279                                                     rx_queue_id);
 280                 dev->data->rx_queue_state[rx_queue_id] =
 281                         RTE_ETH_QUEUE_STATE_STARTED;
 282         } else {
 283                 return -1;
 284         }
 285
 286         return 0;
 287 }
 288
 289 int
 290 atl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 291 {
 292         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 293         struct atl_rx_queue *rxq = NULL;
 294
 295         PMD_INIT_FUNC_TRACE();
 296
 297         if (rx_queue_id < dev->data->nb_rx_queues) {
 298                 rxq = dev->data->rx_queues[rx_queue_id];
 299
 300                 hw_atl_b0_hw_ring_rx_stop(hw, rx_queue_id);
 301
 302                 atl_rx_queue_release_mbufs(rxq);
 303                 atl_reset_rx_queue(rxq);
 304
 305                 dev->data->rx_queue_state[rx_queue_id] =
 306                         RTE_ETH_QUEUE_STATE_STOPPED;
 307         } else {
 308                 return -1;
 309         }
 310
 311         return 0;
 312 }
 313
 314 void
 315 atl_rx_queue_release(void *rx_queue)
 316 {
 317         PMD_INIT_FUNC_TRACE();
 318
 319         if (rx_queue != NULL) {
 320                 struct atl_rx_queue *rxq = (struct atl_rx_queue *)rx_queue;
 321
 322                 atl_rx_queue_release_mbufs(rxq);
 323                 rte_free(rxq->sw_ring);
 324                 rte_free(rxq);
 325         }
 326 }
 327
 328 uint16_t
 329 atl_prep_pkts(void *tx_queue __rte_unused,
 330               struct rte_mbuf **tx_pkts __rte_unused,
 331               uint16_t nb_pkts __rte_unused)
 332 {
 333         return 0;
 334 }
 335
 336 void
 337 atl_free_queues(struct rte_eth_dev *dev)
 338 {
 339         unsigned int i;
 340
 341         PMD_INIT_FUNC_TRACE();
 342
 343         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 344                 atl_rx_queue_release(dev->data->rx_queues[i]);
 345                 dev->data->rx_queues[i] = 0;
 346         }
 347         dev->data->nb_rx_queues = 0;
 348 }
 349
 350 int
 351 atl_start_queues(struct rte_eth_dev *dev)
 352 {
 353         int i;
 354
 355         PMD_INIT_FUNC_TRACE();
 356
 357         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 358                 if (atl_rx_queue_start(dev, i) != 0) {
 359                         PMD_DRV_LOG(ERR,
 360                                 "Port %d: Start Rx queue %d failed",
 361                                 dev->data->port_id, i);
 362                         return -1;
 363                 }
 364         }
 365
 366         return 0;
 367 }
 368
 369 int
 370 atl_stop_queues(struct rte_eth_dev *dev)
 371 {
 372         int i;
 373
 374         PMD_INIT_FUNC_TRACE();
 375
 376         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 377                 if (atl_rx_queue_stop(dev, i) != 0) {
 378                         PMD_DRV_LOG(ERR,
 379                                 "Port %d: Stop Rx queue %d failed",
 380                                 dev->data->port_id, i);
 381                         return -1;
 382                 }
 383         }
 384
 385         return 0;
 386 }
 387
 388 static uint64_t
 389 atl_desc_to_offload_flags(struct atl_rx_queue *rxq,
 390                           struct hw_atl_rxd_wb_s *rxd_wb)
 391 {
 392         uint64_t mbuf_flags = 0;
 393
 394         PMD_INIT_FUNC_TRACE();
 395
 396         /* IPv4 ? */
 397         if (rxq->l3_csum_enabled && ((rxd_wb->pkt_type & 0x3) == 0)) {
 398                 /* IPv4 csum error ? */
 399                 if (rxd_wb->rx_stat & BIT(1))
 400                         mbuf_flags |= PKT_RX_IP_CKSUM_BAD;
 401                 else
 402                         mbuf_flags |= PKT_RX_IP_CKSUM_GOOD;
 403         } else {
 404                 mbuf_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
 405         }
 406
 407         /* CSUM calculated ? */
 408         if (rxq->l4_csum_enabled && (rxd_wb->rx_stat & BIT(3))) {
 409                 if (rxd_wb->rx_stat & BIT(2))
 410                         mbuf_flags |= PKT_RX_L4_CKSUM_BAD;
 411                 else
 412                         mbuf_flags |= PKT_RX_L4_CKSUM_GOOD;
 413         } else {
 414                 mbuf_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
 415         }
 416
 417         return mbuf_flags;
 418 }
 419
 420 static uint32_t
 421 atl_desc_to_pkt_type(struct hw_atl_rxd_wb_s *rxd_wb)
 422 {
 423         uint32_t type = RTE_PTYPE_UNKNOWN;
 424         uint16_t l2_l3_type = rxd_wb->pkt_type & 0x3;
 425         uint16_t l4_type = (rxd_wb->pkt_type & 0x1C) >> 2;
 426
 427         switch (l2_l3_type) {
 428         case 0:
 429                 type = RTE_PTYPE_L3_IPV4;
 430                 break;
 431         case 1:
 432                 type = RTE_PTYPE_L3_IPV6;
 433                 break;
 434         case 2:
 435                 type = RTE_PTYPE_L2_ETHER;
 436                 break;
 437         case 3:
 438                 type = RTE_PTYPE_L2_ETHER_ARP;
 439                 break;
 440         }
 441
 442         switch (l4_type) {
 443         case 0:
 444                 type |= RTE_PTYPE_L4_TCP;
 445                 break;
 446         case 1:
 447                 type |= RTE_PTYPE_L4_UDP;
 448                 break;
 449         case 2:
 450                 type |= RTE_PTYPE_L4_SCTP;
 451                 break;
 452         case 3:
 453                 type |= RTE_PTYPE_L4_ICMP;
 454                 break;
 455         }
 456
 457         if (rxd_wb->pkt_type & BIT(5))
 458                 type |= RTE_PTYPE_L2_ETHER_VLAN;
 459
 460         return type;
 461 }
 462
 463 uint16_t
 464 atl_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 465 {
 466         struct atl_rx_queue *rxq = (struct atl_rx_queue *)rx_queue;
 467         struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
 468         struct atl_adapter *adapter =
 469                 ATL_DEV_TO_ADAPTER(&rte_eth_devices[rxq->port_id]);
 470         struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(adapter);
 471         struct atl_rx_entry *sw_ring = rxq->sw_ring;
 472
 473         struct rte_mbuf *new_mbuf;
 474         struct rte_mbuf *rx_mbuf, *rx_mbuf_prev, *rx_mbuf_first;
 475         struct atl_rx_entry *rx_entry;
 476         uint16_t nb_rx = 0;
 477         uint16_t nb_hold = 0;
 478         struct hw_atl_rxd_wb_s rxd_wb;
 479         struct hw_atl_rxd_s *rxd = NULL;
 480         uint16_t tail = rxq->rx_tail;
 481         uint64_t dma_addr;
 482         uint16_t pkt_len = 0;
 483
 484         while (nb_rx < nb_pkts) {
 485                 uint16_t eop_tail = tail;
 486
 487                 rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail];
 488                 rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd;
 489
 490                 if (!rxd_wb.dd) { /* RxD is not done */
 491                         break;
 492                 }
 493
 494                 PMD_RX_LOG(ERR, "port_id=%u queue_id=%u tail=%u "
 495                            "eop=0x%x pkt_len=%u hash=0x%x hash_type=0x%x",
 496                            (unsigned int)rxq->port_id,
 497                            (unsigned int)rxq->queue_id,
 498                            (unsigned int)tail, (unsigned int)rxd_wb.eop,
 499                            (unsigned int)rte_le_to_cpu_16(rxd_wb.pkt_len),
 500                         rxd_wb.rss_hash, rxd_wb.rss_type);
 501
 502                 /* RxD is not done */
 503                 if (!rxd_wb.eop) {
 504                         while (true) {
 505                                 struct hw_atl_rxd_wb_s *eop_rxwbd;
 506
 507                                 eop_tail = (eop_tail + 1) % rxq->nb_rx_desc;
 508                                 eop_rxwbd = (struct hw_atl_rxd_wb_s *)
 509                                         &rxq->hw_ring[eop_tail];
 510                                 if (!eop_rxwbd->dd) {
 511                                         /* no EOP received yet */
 512                                         eop_tail = tail;
 513                                         break;
 514                                 }
 515                                 if (eop_rxwbd->dd && eop_rxwbd->eop)
 516                                         break;
 517                         }
 518                         /* No EOP in ring */
 519                         if (eop_tail == tail)
 520                                 break;
 521                 }
 522                 rx_mbuf_prev = NULL;
 523                 rx_mbuf_first = NULL;
 524
 525                 /* Run through packet segments */
 526                 while (true) {
 527                         new_mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
 528                         if (new_mbuf == NULL) {
 529                                 PMD_RX_LOG(ERR,
 530                                    "RX mbuf alloc failed port_id=%u "
 531                                    "queue_id=%u", (unsigned int)rxq->port_id,
 532                                    (unsigned int)rxq->queue_id);
 533                                 dev->data->rx_mbuf_alloc_failed++;
 534                                                 goto err_stop;
 535                         }
 536
 537                         nb_hold++;
 538                         rx_entry = &sw_ring[tail];
 539
 540                         rx_mbuf = rx_entry->mbuf;
 541                         rx_entry->mbuf = new_mbuf;
 542                         dma_addr = rte_cpu_to_le_64(
 543                                 rte_mbuf_data_iova_default(new_mbuf));
 544
 545                         /* setup RX descriptor */
 546                         rxd->hdr_addr = 0;
 547                         rxd->buf_addr = dma_addr;
 548
 549                         /*
 550                          * Initialize the returned mbuf.
 551                          * 1) setup generic mbuf fields:
 552                          *        - number of segments,
 553                          *        - next segment,
 554                          *        - packet length,
 555                          *        - RX port identifier.
 556                          * 2) integrate hardware offload data, if any:
 557                          *      <  - RSS flag & hash,
 558                          *        - IP checksum flag,
 559                          *        - VLAN TCI, if any,
 560                          *        - error flags.
 561                          */
 562                         pkt_len = (uint16_t)rte_le_to_cpu_16(rxd_wb.pkt_len);
 563                         rx_mbuf->data_off = RTE_PKTMBUF_HEADROOM;
 564                         rte_prefetch1((char *)rx_mbuf->buf_addr +
 565                                 rx_mbuf->data_off);
 566                         rx_mbuf->nb_segs = 0;
 567                         rx_mbuf->next = NULL;
 568                         rx_mbuf->pkt_len = pkt_len;
 569                         rx_mbuf->data_len = pkt_len;
 570                         if (rxd_wb.eop) {
 571                                 u16 remainder_len = pkt_len % rxq->buff_size;
 572                                 if (!remainder_len)
 573                                         remainder_len = rxq->buff_size;
 574                                 rx_mbuf->data_len = remainder_len;
 575                         } else {
 576                                 rx_mbuf->data_len = pkt_len > rxq->buff_size ?
 577                                                 rxq->buff_size : pkt_len;
 578                         }
 579                         rx_mbuf->port = rxq->port_id;
 580
 581                         rx_mbuf->hash.rss = rxd_wb.rss_hash;
 582
 583                         rx_mbuf->vlan_tci = rxd_wb.vlan;
 584
 585                         rx_mbuf->ol_flags =
 586                                 atl_desc_to_offload_flags(rxq, &rxd_wb);
 587                         rx_mbuf->packet_type = atl_desc_to_pkt_type(&rxd_wb);
 588
 589                         if (!rx_mbuf_first)
 590                                 rx_mbuf_first = rx_mbuf;
 591                         rx_mbuf_first->nb_segs++;
 592
 593                         if (rx_mbuf_prev)
 594                                 rx_mbuf_prev->next = rx_mbuf;
 595                         rx_mbuf_prev = rx_mbuf;
 596
 597                         tail = (tail + 1) % rxq->nb_rx_desc;
 598                         /* Prefetch next mbufs */
 599                         rte_prefetch0(sw_ring[tail].mbuf);
 600                         if ((tail & 0x3) == 0) {
 601                                 rte_prefetch0(&sw_ring[tail]);
 602                                 rte_prefetch0(&sw_ring[tail]);
 603                         }
 604
 605                         /* filled mbuf_first */
 606                         if (rxd_wb.eop)
 607                                 break;
 608                         rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail];
 609                         rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd;
 610                 };
 611
 612                 /*
 613                  * Store the mbuf address into the next entry of the array
 614                  * of returned packets.
 615                  */
 616                 rx_pkts[nb_rx++] = rx_mbuf_first;
 617
 618                 PMD_RX_LOG(ERR, "add mbuf segs=%d pkt_len=%d",
 619                         rx_mbuf_first->nb_segs,
 620                         rx_mbuf_first->pkt_len);
 621         }
 622
 623 err_stop:
 624
 625         rxq->rx_tail = tail;
 626
 627         /*
 628          * If the number of free RX descriptors is greater than the RX free
 629          * threshold of the queue, advance the Receive Descriptor Tail (RDT)
 630          * register.
 631          * Update the RDT with the value of the last processed RX descriptor
 632          * minus 1, to guarantee that the RDT register is never equal to the
 633          * RDH register, which creates a "full" ring situtation from the
 634          * hardware point of view...
 635          */
 636         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
 637         if (nb_hold > rxq->rx_free_thresh) {
 638                 PMD_RX_LOG(ERR, "port_id=%u queue_id=%u rx_tail=%u "
 639                         "nb_hold=%u nb_rx=%u",
 640                         (unsigned int)rxq->port_id, (unsigned int)rxq->queue_id,
 641                         (unsigned int)tail, (unsigned int)nb_hold,
 642                         (unsigned int)nb_rx);
 643                 tail = (uint16_t)((tail == 0) ?
 644                         (rxq->nb_rx_desc - 1) : (tail - 1));
 645
 646                 hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, tail, rxq->queue_id);
 647
 648                 nb_hold = 0;
 649         }
 650
 651         rxq->nb_rx_hold = nb_hold;
 652
 653         return nb_rx;
 654 }
 655
 656
 657 uint16_t
 658 atl_xmit_pkts(void *tx_queue __rte_unused,
 659               struct rte_mbuf **tx_pkts __rte_unused,
 660               uint16_t nb_pkts __rte_unused)
 661 {
 662         return 0;
 663 }
 664