drivers/net/avf/avf_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <errno.h>
   9 #include <stdint.h>
  10 #include <stdarg.h>
  11 #include <unistd.h>
  12 #include <inttypes.h>
  13 #include <sys/queue.h>
  14
  15 #include <rte_string_fns.h>
  16 #include <rte_memzone.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_malloc.h>
  19 #include <rte_ether.h>
  20 #include <rte_ethdev.h>
  21 #include <rte_tcp.h>
  22 #include <rte_sctp.h>
  23 #include <rte_udp.h>
  24 #include <rte_ip.h>
  25 #include <rte_net.h>
  26
  27 #include "avf_log.h"
  28 #include "base/avf_prototype.h"
  29 #include "base/avf_type.h"
  30 #include "avf.h"
  31 #include "avf_rxtx.h"
  32
  33 static inline int
  34 check_rx_thresh(uint16_t nb_desc, uint16_t thresh)
  35 {
  36         /* The following constraints must be satisfied:
  37          *   thresh < rxq->nb_rx_desc
  38          */
  39         if (thresh >= nb_desc) {
  40                 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be less than %u",
  41                              thresh, nb_desc);
  42                 return -EINVAL;
  43         }
  44         return 0;
  45 }
  46
  47 static inline int
  48 check_tx_thresh(uint16_t nb_desc, uint16_t tx_rs_thresh,
  49                 uint16_t tx_free_thresh)
  50 {
  51         /* TX descriptors will have their RS bit set after tx_rs_thresh
  52          * descriptors have been used. The TX descriptor ring will be cleaned
  53          * after tx_free_thresh descriptors are used or if the number of
  54          * descriptors required to transmit a packet is greater than the
  55          * number of free TX descriptors.
  56          *
  57          * The following constraints must be satisfied:
  58          *  - tx_rs_thresh must be less than the size of the ring minus 2.
  59          *  - tx_free_thresh must be less than the size of the ring minus 3.
  60          *  - tx_rs_thresh must be less than or equal to tx_free_thresh.
  61          *  - tx_rs_thresh must be a divisor of the ring size.
  62          *
  63          * One descriptor in the TX ring is used as a sentinel to avoid a H/W
  64          * race condition, hence the maximum threshold constraints. When set
  65          * to zero use default values.
  66          */
  67         if (tx_rs_thresh >= (nb_desc - 2)) {
  68                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be less than the "
  69                              "number of TX descriptors (%u) minus 2",
  70                              tx_rs_thresh, nb_desc);
  71                 return -EINVAL;
  72         }
  73         if (tx_free_thresh >= (nb_desc - 3)) {
  74                 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be less than the "
  75                              "number of TX descriptors (%u) minus 3.",
  76                              tx_free_thresh, nb_desc);
  77                 return -EINVAL;
  78         }
  79         if (tx_rs_thresh > tx_free_thresh) {
  80                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be less than or "
  81                              "equal to tx_free_thresh (%u).",
  82                              tx_rs_thresh, tx_free_thresh);
  83                 return -EINVAL;
  84         }
  85         if ((nb_desc % tx_rs_thresh) != 0) {
  86                 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be a divisor of the "
  87                              "number of TX descriptors (%u).",
  88                              tx_rs_thresh, nb_desc);
  89                 return -EINVAL;
  90         }
  91
  92         return 0;
  93 }
  94
  95 static inline void
  96 reset_rx_queue(struct avf_rx_queue *rxq)
  97 {
  98         uint16_t len, i;
  99
 100         if (!rxq)
 101                 return;
 102
 103         len = rxq->nb_rx_desc + AVF_RX_MAX_BURST;
 104
 105         for (i = 0; i < len * sizeof(union avf_rx_desc); i++)
 106                 ((volatile char *)rxq->rx_ring)[i] = 0;
 107
 108         memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
 109
 110         for (i = 0; i < AVF_RX_MAX_BURST; i++)
 111                 rxq->sw_ring[rxq->nb_rx_desc + i] = &rxq->fake_mbuf;
 112
 113         rxq->rx_tail = 0;
 114         rxq->nb_rx_hold = 0;
 115         rxq->pkt_first_seg = NULL;
 116         rxq->pkt_last_seg = NULL;
 117 }
 118
 119 static inline void
 120 reset_tx_queue(struct avf_tx_queue *txq)
 121 {
 122         struct avf_tx_entry *txe;
 123         uint16_t i, prev, size;
 124
 125         if (!txq) {
 126                 PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
 127                 return;
 128         }
 129
 130         txe = txq->sw_ring;
 131         size = sizeof(struct avf_tx_desc) * txq->nb_tx_desc;
 132         for (i = 0; i < size; i++)
 133                 ((volatile char *)txq->tx_ring)[i] = 0;
 134
 135         prev = (uint16_t)(txq->nb_tx_desc - 1);
 136         for (i = 0; i < txq->nb_tx_desc; i++) {
 137                 txq->tx_ring[i].cmd_type_offset_bsz =
 138                         rte_cpu_to_le_64(AVF_TX_DESC_DTYPE_DESC_DONE);
 139                 txe[i].mbuf =  NULL;
 140                 txe[i].last_id = i;
 141                 txe[prev].next_id = i;
 142                 prev = i;
 143         }
 144
 145         txq->tx_tail = 0;
 146         txq->nb_used = 0;
 147
 148         txq->last_desc_cleaned = txq->nb_tx_desc - 1;
 149         txq->nb_free = txq->nb_tx_desc - 1;
 150
 151         txq->next_dd = txq->rs_thresh - 1;
 152         txq->next_rs = txq->rs_thresh - 1;
 153 }
 154
 155 static int
 156 alloc_rxq_mbufs(struct avf_rx_queue *rxq)
 157 {
 158         volatile union avf_rx_desc *rxd;
 159         struct rte_mbuf *mbuf = NULL;
 160         uint64_t dma_addr;
 161         uint16_t i;
 162
 163         for (i = 0; i < rxq->nb_rx_desc; i++) {
 164                 mbuf = rte_mbuf_raw_alloc(rxq->mp);
 165                 if (unlikely(!mbuf)) {
 166                         PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX");
 167                         return -ENOMEM;
 168                 }
 169
 170                 rte_mbuf_refcnt_set(mbuf, 1);
 171                 mbuf->next = NULL;
 172                 mbuf->data_off = RTE_PKTMBUF_HEADROOM;
 173                 mbuf->nb_segs = 1;
 174                 mbuf->port = rxq->port_id;
 175
 176                 dma_addr =
 177                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
 178
 179                 rxd = &rxq->rx_ring[i];
 180                 rxd->read.pkt_addr = dma_addr;
 181                 rxd->read.hdr_addr = 0;
 182 #ifndef RTE_LIBRTE_AVF_16BYTE_RX_DESC
 183                 rxd->read.rsvd1 = 0;
 184                 rxd->read.rsvd2 = 0;
 185 #endif
 186
 187                 rxq->sw_ring[i] = mbuf;
 188         }
 189
 190         return 0;
 191 }
 192
 193 static inline void
 194 release_rxq_mbufs(struct avf_rx_queue *rxq)
 195 {
 196         struct rte_mbuf *mbuf;
 197         uint16_t i;
 198
 199         if (!rxq->sw_ring)
 200                 return;
 201
 202         for (i = 0; i < rxq->nb_rx_desc; i++) {
 203                 if (rxq->sw_ring[i]) {
 204                         rte_pktmbuf_free_seg(rxq->sw_ring[i]);
 205                         rxq->sw_ring[i] = NULL;
 206                 }
 207         }
 208 }
 209
 210 static inline void
 211 release_txq_mbufs(struct avf_tx_queue *txq)
 212 {
 213         uint16_t i;
 214
 215         if (!txq || !txq->sw_ring) {
 216                 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
 217                 return;
 218         }
 219
 220         for (i = 0; i < txq->nb_tx_desc; i++) {
 221                 if (txq->sw_ring[i].mbuf) {
 222                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
 223                         txq->sw_ring[i].mbuf = NULL;
 224                 }
 225         }
 226 }
 227
 228 int
 229 avf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 230                        uint16_t nb_desc, unsigned int socket_id,
 231                        const struct rte_eth_rxconf *rx_conf,
 232                        struct rte_mempool *mp)
 233 {
 234         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 235         struct avf_adapter *ad =
 236                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 237         struct avf_rx_queue *rxq;
 238         const struct rte_memzone *mz;
 239         uint32_t ring_size;
 240         uint16_t len, i;
 241         uint16_t rx_free_thresh;
 242         uint16_t base, bsf, tc_mapping;
 243
 244         PMD_INIT_FUNC_TRACE();
 245
 246         if (nb_desc % AVF_ALIGN_RING_DESC != 0 ||
 247             nb_desc > AVF_MAX_RING_DESC ||
 248             nb_desc < AVF_MIN_RING_DESC) {
 249                 PMD_INIT_LOG(ERR, "Number (%u) of receive descriptors is "
 250                              "invalid", nb_desc);
 251                 return -EINVAL;
 252         }
 253
 254         /* Check free threshold */
 255         rx_free_thresh = (rx_conf->rx_free_thresh == 0) ?
 256                          AVF_DEFAULT_RX_FREE_THRESH :
 257                          rx_conf->rx_free_thresh;
 258         if (check_rx_thresh(nb_desc, rx_free_thresh) != 0)
 259                 return -EINVAL;
 260
 261         /* Free memory if needed */
 262         if (dev->data->rx_queues[queue_idx]) {
 263                 avf_dev_rx_queue_release(dev->data->rx_queues[queue_idx]);
 264                 dev->data->rx_queues[queue_idx] = NULL;
 265         }
 266
 267         /* Allocate the rx queue data structure */
 268         rxq = rte_zmalloc_socket("avf rxq",
 269                                  sizeof(struct avf_rx_queue),
 270                                  RTE_CACHE_LINE_SIZE,
 271                                  socket_id);
 272         if (!rxq) {
 273                 PMD_INIT_LOG(ERR, "Failed to allocate memory for "
 274                              "rx queue data structure");
 275                 return -ENOMEM;
 276         }
 277
 278         rxq->mp = mp;
 279         rxq->nb_rx_desc = nb_desc;
 280         rxq->rx_free_thresh = rx_free_thresh;
 281         rxq->queue_id = queue_idx;
 282         rxq->port_id = dev->data->port_id;
 283         rxq->crc_len = 0; /* crc stripping by default */
 284         rxq->rx_deferred_start = rx_conf->rx_deferred_start;
 285         rxq->rx_hdr_len = 0;
 286
 287         len = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM;
 288         rxq->rx_buf_len = RTE_ALIGN(len, (1 << AVF_RXQ_CTX_DBUFF_SHIFT));
 289
 290         /* Allocate the software ring. */
 291         len = nb_desc + AVF_RX_MAX_BURST;
 292         rxq->sw_ring =
 293                 rte_zmalloc_socket("avf rx sw ring",
 294                                    sizeof(struct rte_mbuf *) * len,
 295                                    RTE_CACHE_LINE_SIZE,
 296                                    socket_id);
 297         if (!rxq->sw_ring) {
 298                 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW ring");
 299                 rte_free(rxq);
 300                 return -ENOMEM;
 301         }
 302
 303         /* Allocate the maximun number of RX ring hardware descriptor with
 304          * a liitle more to support bulk allocate.
 305          */
 306         len = AVF_MAX_RING_DESC + AVF_RX_MAX_BURST;
 307         ring_size = RTE_ALIGN(len * sizeof(union avf_rx_desc),
 308                               AVF_DMA_MEM_ALIGN);
 309         mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
 310                                       ring_size, AVF_RING_BASE_ALIGN,
 311                                       socket_id);
 312         if (!mz) {
 313                 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for RX");
 314                 rte_free(rxq->sw_ring);
 315                 rte_free(rxq);
 316                 return -ENOMEM;
 317         }
 318         /* Zero all the descriptors in the ring. */
 319         memset(mz->addr, 0, ring_size);
 320         rxq->rx_ring_phys_addr = mz->iova;
 321         rxq->rx_ring = (union avf_rx_desc *)mz->addr;
 322
 323         rxq->mz = mz;
 324         reset_rx_queue(rxq);
 325         rxq->q_set = TRUE;
 326         dev->data->rx_queues[queue_idx] = rxq;
 327         rxq->qrx_tail = hw->hw_addr + AVF_QRX_TAIL1(rxq->queue_id);
 328
 329         return 0;
 330 }
 331
 332 int
 333 avf_dev_tx_queue_setup(struct rte_eth_dev *dev,
 334                        uint16_t queue_idx,
 335                        uint16_t nb_desc,
 336                        unsigned int socket_id,
 337                        const struct rte_eth_txconf *tx_conf)
 338 {
 339         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 340         struct avf_tx_queue *txq;
 341         const struct rte_memzone *mz;
 342         uint32_t ring_size;
 343         uint16_t tx_rs_thresh, tx_free_thresh;
 344         uint16_t i, base, bsf, tc_mapping;
 345
 346         PMD_INIT_FUNC_TRACE();
 347
 348         if (nb_desc % AVF_ALIGN_RING_DESC != 0 ||
 349             nb_desc > AVF_MAX_RING_DESC ||
 350             nb_desc < AVF_MIN_RING_DESC) {
 351                 PMD_INIT_LOG(ERR, "Number (%u) of transmit descriptors is "
 352                             "invalid", nb_desc);
 353                 return -EINVAL;
 354         }
 355
 356         tx_rs_thresh = (uint16_t)((tx_conf->tx_rs_thresh) ?
 357                 tx_conf->tx_rs_thresh : DEFAULT_TX_RS_THRESH);
 358         tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
 359                 tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
 360         check_tx_thresh(nb_desc, tx_rs_thresh, tx_rs_thresh);
 361
 362         /* Free memory if needed. */
 363         if (dev->data->tx_queues[queue_idx]) {
 364                 avf_dev_tx_queue_release(dev->data->tx_queues[queue_idx]);
 365                 dev->data->tx_queues[queue_idx] = NULL;
 366         }
 367
 368         /* Allocate the TX queue data structure. */
 369         txq = rte_zmalloc_socket("avf txq",
 370                                  sizeof(struct avf_tx_queue),
 371                                  RTE_CACHE_LINE_SIZE,
 372                                  socket_id);
 373         if (!txq) {
 374                 PMD_INIT_LOG(ERR, "Failed to allocate memory for "
 375                              "tx queue structure");
 376                 return -ENOMEM;
 377         }
 378
 379         txq->nb_tx_desc = nb_desc;
 380         txq->rs_thresh = tx_rs_thresh;
 381         txq->free_thresh = tx_free_thresh;
 382         txq->queue_id = queue_idx;
 383         txq->port_id = dev->data->port_id;
 384         txq->txq_flags = tx_conf->txq_flags;
 385         txq->tx_deferred_start = tx_conf->tx_deferred_start;
 386
 387         /* Allocate software ring */
 388         txq->sw_ring =
 389                 rte_zmalloc_socket("avf tx sw ring",
 390                                    sizeof(struct avf_tx_entry) * nb_desc,
 391                                    RTE_CACHE_LINE_SIZE,
 392                                    socket_id);
 393         if (!txq->sw_ring) {
 394                 PMD_INIT_LOG(ERR, "Failed to allocate memory for SW TX ring");
 395                 rte_free(txq);
 396                 return -ENOMEM;
 397         }
 398
 399         /* Allocate TX hardware ring descriptors. */
 400         ring_size = sizeof(struct avf_tx_desc) * AVF_MAX_RING_DESC;
 401         ring_size = RTE_ALIGN(ring_size, AVF_DMA_MEM_ALIGN);
 402         mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
 403                                       ring_size, AVF_RING_BASE_ALIGN,
 404                                       socket_id);
 405         if (!mz) {
 406                 PMD_INIT_LOG(ERR, "Failed to reserve DMA memory for TX");
 407                 rte_free(txq->sw_ring);
 408                 rte_free(txq);
 409                 return -ENOMEM;
 410         }
 411         txq->tx_ring_phys_addr = mz->iova;
 412         txq->tx_ring = (struct avf_tx_desc *)mz->addr;
 413
 414         txq->mz = mz;
 415         reset_tx_queue(txq);
 416         txq->q_set = TRUE;
 417         dev->data->tx_queues[queue_idx] = txq;
 418         txq->qtx_tail = hw->hw_addr + AVF_QTX_TAIL1(queue_idx);
 419
 420         return 0;
 421 }
 422
 423 int
 424 avf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 425 {
 426         struct avf_adapter *adapter =
 427                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 428         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 429         struct avf_rx_queue *rxq;
 430         int err = 0;
 431
 432         PMD_DRV_FUNC_TRACE();
 433
 434         if (rx_queue_id >= dev->data->nb_rx_queues)
 435                 return -EINVAL;
 436
 437         rxq = dev->data->rx_queues[rx_queue_id];
 438
 439         err = alloc_rxq_mbufs(rxq);
 440         if (err) {
 441                 PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
 442                 return err;
 443         }
 444
 445         rte_wmb();
 446
 447         /* Init the RX tail register. */
 448         AVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
 449         AVF_WRITE_FLUSH(hw);
 450
 451         /* Ready to switch the queue on */
 452         err = avf_switch_queue(adapter, rx_queue_id, TRUE, TRUE);
 453         if (err)
 454                 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 455                             rx_queue_id);
 456         else
 457                 dev->data->rx_queue_state[rx_queue_id] =
 458                         RTE_ETH_QUEUE_STATE_STARTED;
 459
 460         return err;
 461 }
 462
 463 int
 464 avf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 465 {
 466         struct avf_adapter *adapter =
 467                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 468         struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 469         struct avf_tx_queue *txq;
 470         int err = 0;
 471
 472         PMD_DRV_FUNC_TRACE();
 473
 474         if (tx_queue_id >= dev->data->nb_tx_queues)
 475                 return -EINVAL;
 476
 477         txq = dev->data->tx_queues[tx_queue_id];
 478
 479         /* Init the RX tail register. */
 480         AVF_PCI_REG_WRITE(txq->qtx_tail, 0);
 481         AVF_WRITE_FLUSH(hw);
 482
 483         /* Ready to switch the queue on */
 484         err = avf_switch_queue(adapter, tx_queue_id, FALSE, TRUE);
 485
 486         if (err)
 487                 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
 488                             tx_queue_id);
 489         else
 490                 dev->data->tx_queue_state[tx_queue_id] =
 491                         RTE_ETH_QUEUE_STATE_STARTED;
 492
 493         return err;
 494 }
 495
 496 int
 497 avf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 498 {
 499         struct avf_adapter *adapter =
 500                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 501         struct avf_rx_queue *rxq;
 502         int err;
 503
 504         PMD_DRV_FUNC_TRACE();
 505
 506         if (rx_queue_id >= dev->data->nb_rx_queues)
 507                 return -EINVAL;
 508
 509         err = avf_switch_queue(adapter, rx_queue_id, TRUE, FALSE);
 510         if (err) {
 511                 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
 512                             rx_queue_id);
 513                 return err;
 514         }
 515
 516         rxq = dev->data->rx_queues[rx_queue_id];
 517         release_rxq_mbufs(rxq);
 518         reset_rx_queue(rxq);
 519         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 520
 521         return 0;
 522 }
 523
 524 int
 525 avf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 526 {
 527         struct avf_adapter *adapter =
 528                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 529         struct avf_tx_queue *txq;
 530         int err;
 531
 532         PMD_DRV_FUNC_TRACE();
 533
 534         if (tx_queue_id >= dev->data->nb_tx_queues)
 535                 return -EINVAL;
 536
 537         err = avf_switch_queue(adapter, tx_queue_id, FALSE, FALSE);
 538         if (err) {
 539                 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
 540                             tx_queue_id);
 541                 return err;
 542         }
 543
 544         txq = dev->data->tx_queues[tx_queue_id];
 545         release_txq_mbufs(txq);
 546         reset_tx_queue(txq);
 547         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 548
 549         return 0;
 550 }
 551
 552 void
 553 avf_dev_rx_queue_release(void *rxq)
 554 {
 555         struct avf_rx_queue *q = (struct avf_rx_queue *)rxq;
 556
 557         if (!q)
 558                 return;
 559
 560         release_rxq_mbufs(q);
 561         rte_free(q->sw_ring);
 562         rte_memzone_free(q->mz);
 563         rte_free(q);
 564 }
 565
 566 void
 567 avf_dev_tx_queue_release(void *txq)
 568 {
 569         struct avf_tx_queue *q = (struct avf_tx_queue *)txq;
 570
 571         if (!q)
 572                 return;
 573
 574         release_txq_mbufs(q);
 575         rte_free(q->sw_ring);
 576         rte_memzone_free(q->mz);
 577         rte_free(q);
 578 }
 579
 580 void
 581 avf_stop_queues(struct rte_eth_dev *dev)
 582 {
 583         struct avf_adapter *adapter =
 584                 AVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 585         struct avf_rx_queue *rxq;
 586         struct avf_tx_queue *txq;
 587         int ret, i;
 588
 589         /* Stop All queues */
 590         ret = avf_disable_queues(adapter);
 591         if (ret)
 592                 PMD_DRV_LOG(WARNING, "Fail to stop queues");
 593
 594         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 595                 txq = dev->data->tx_queues[i];
 596                 if (!txq)
 597                         continue;
 598                 release_txq_mbufs(txq);
 599                 reset_tx_queue(txq);
 600                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
 601         }
 602         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 603                 rxq = dev->data->rx_queues[i];
 604                 if (!rxq)
 605                         continue;
 606                 release_rxq_mbufs(rxq);
 607                 reset_rx_queue(rxq);
 608                 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
 609         }
 610 }
 611
 612 static inline void
 613 avf_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union avf_rx_desc *rxdp)
 614 {
 615         if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
 616                 (1 << AVF_RX_DESC_STATUS_L2TAG1P_SHIFT)) {
 617                 mb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 618                 mb->vlan_tci =
 619                         rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1);
 620         } else {
 621                 mb->vlan_tci = 0;
 622         }
 623 }
 624
 625 /* Translate the rx descriptor status and error fields to pkt flags */
 626 static inline uint64_t
 627 avf_rxd_to_pkt_flags(uint64_t qword)
 628 {
 629         uint64_t flags;
 630         uint64_t error_bits = (qword >> AVF_RXD_QW1_ERROR_SHIFT);
 631
 632 #define AVF_RX_ERR_BITS 0x3f
 633
 634         /* Check if RSS_HASH */
 635         flags = (((qword >> AVF_RX_DESC_STATUS_FLTSTAT_SHIFT) &
 636                                         AVF_RX_DESC_FLTSTAT_RSS_HASH) ==
 637                         AVF_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0;
 638
 639         if (likely((error_bits & AVF_RX_ERR_BITS) == 0)) {
 640                 flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
 641                 return flags;
 642         }
 643
 644         if (unlikely(error_bits & (1 << AVF_RX_DESC_ERROR_IPE_SHIFT)))
 645                 flags |= PKT_RX_IP_CKSUM_BAD;
 646         else
 647                 flags |= PKT_RX_IP_CKSUM_GOOD;
 648
 649         if (unlikely(error_bits & (1 << AVF_RX_DESC_ERROR_L4E_SHIFT)))
 650                 flags |= PKT_RX_L4_CKSUM_BAD;
 651         else
 652                 flags |= PKT_RX_L4_CKSUM_GOOD;
 653
 654         /* TODO: Oversize error bit is not processed here */
 655
 656         return flags;
 657 }
 658
 659 /* implement recv_pkts */
 660 uint16_t
 661 avf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 662 {
 663         volatile union avf_rx_desc *rx_ring;
 664         volatile union avf_rx_desc *rxdp;
 665         struct avf_rx_queue *rxq;
 666         union avf_rx_desc rxd;
 667         struct rte_mbuf *rxe;
 668         struct rte_eth_dev *dev;
 669         struct rte_mbuf *rxm;
 670         struct rte_mbuf *nmb;
 671         uint16_t nb_rx;
 672         uint32_t rx_status;
 673         uint64_t qword1;
 674         uint16_t rx_packet_len;
 675         uint16_t rx_id, nb_hold;
 676         uint64_t dma_addr;
 677         uint64_t pkt_flags;
 678         static const uint32_t ptype_tbl[UINT8_MAX + 1] __rte_cache_aligned = {
 679                 /* [0] reserved */
 680                 [1] = RTE_PTYPE_L2_ETHER,
 681                 /* [2] - [21] reserved */
 682                 [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 683                         RTE_PTYPE_L4_FRAG,
 684                 [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 685                         RTE_PTYPE_L4_NONFRAG,
 686                 [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 687                         RTE_PTYPE_L4_UDP,
 688                 /* [25] reserved */
 689                 [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 690                         RTE_PTYPE_L4_TCP,
 691                 [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 692                         RTE_PTYPE_L4_SCTP,
 693                 [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 694                         RTE_PTYPE_L4_ICMP,
 695                 /* All others reserved */
 696         };
 697
 698         nb_rx = 0;
 699         nb_hold = 0;
 700         rxq = rx_queue;
 701         rx_id = rxq->rx_tail;
 702         rx_ring = rxq->rx_ring;
 703
 704         while (nb_rx < nb_pkts) {
 705                 rxdp = &rx_ring[rx_id];
 706                 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
 707                 rx_status = (qword1 & AVF_RXD_QW1_STATUS_MASK) >>
 708                             AVF_RXD_QW1_STATUS_SHIFT;
 709
 710                 /* Check the DD bit first */
 711                 if (!(rx_status & (1 << AVF_RX_DESC_STATUS_DD_SHIFT)))
 712                         break;
 713                 AVF_DUMP_RX_DESC(rxq, rxdp, rx_id);
 714
 715                 nmb = rte_mbuf_raw_alloc(rxq->mp);
 716                 if (unlikely(!nmb)) {
 717                         dev = &rte_eth_devices[rxq->port_id];
 718                         dev->data->rx_mbuf_alloc_failed++;
 719                         PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
 720                                    "queue_id=%u", rxq->port_id, rxq->queue_id);
 721                         break;
 722                 }
 723
 724                 rxd = *rxdp;
 725                 nb_hold++;
 726                 rxe = rxq->sw_ring[rx_id];
 727                 rx_id++;
 728                 if (unlikely(rx_id == rxq->nb_rx_desc))
 729                         rx_id = 0;
 730
 731                 /* Prefetch next mbuf */
 732                 rte_prefetch0(rxq->sw_ring[rx_id]);
 733
 734                 /* When next RX descriptor is on a cache line boundary,
 735                  * prefetch the next 4 RX descriptors and next 8 pointers
 736                  * to mbufs.
 737                  */
 738                 if ((rx_id & 0x3) == 0) {
 739                         rte_prefetch0(&rx_ring[rx_id]);
 740                         rte_prefetch0(rxq->sw_ring[rx_id]);
 741                 }
 742                 rxm = rxe;
 743                 rxe = nmb;
 744                 dma_addr =
 745                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
 746                 rxdp->read.hdr_addr = 0;
 747                 rxdp->read.pkt_addr = dma_addr;
 748
 749                 rx_packet_len = ((qword1 & AVF_RXD_QW1_LENGTH_PBUF_MASK) >>
 750                                 AVF_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len;
 751
 752                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 753                 rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
 754                 rxm->nb_segs = 1;
 755                 rxm->next = NULL;
 756                 rxm->pkt_len = rx_packet_len;
 757                 rxm->data_len = rx_packet_len;
 758                 rxm->port = rxq->port_id;
 759                 rxm->ol_flags = 0;
 760                 avf_rxd_to_vlan_tci(rxm, &rxd);
 761                 pkt_flags = avf_rxd_to_pkt_flags(qword1);
 762                 rxm->packet_type =
 763                         ptype_tbl[(uint8_t)((qword1 &
 764                         AVF_RXD_QW1_PTYPE_MASK) >> AVF_RXD_QW1_PTYPE_SHIFT)];
 765
 766                 if (pkt_flags & PKT_RX_RSS_HASH)
 767                         rxm->hash.rss =
 768                                 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 769
 770                 rxm->ol_flags |= pkt_flags;
 771
 772                 rx_pkts[nb_rx++] = rxm;
 773         }
 774         rxq->rx_tail = rx_id;
 775
 776         /* If the number of free RX descriptors is greater than the RX free
 777          * threshold of the queue, advance the receive tail register of queue.
 778          * Update that register with the value of the last processed RX
 779          * descriptor minus 1.
 780          */
 781         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
 782         if (nb_hold > rxq->rx_free_thresh) {
 783                 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
 784                            "nb_hold=%u nb_rx=%u",
 785                            rxq->port_id, rxq->queue_id,
 786                            rx_id, nb_hold, nb_rx);
 787                 rx_id = (uint16_t)((rx_id == 0) ?
 788                         (rxq->nb_rx_desc - 1) : (rx_id - 1));
 789                 AVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
 790                 nb_hold = 0;
 791         }
 792         rxq->nb_rx_hold = nb_hold;
 793
 794         return nb_rx;
 795 }
 796
 797 /* implement recv_scattered_pkts  */
 798 uint16_t
 799 avf_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 800                         uint16_t nb_pkts)
 801 {
 802         struct avf_rx_queue *rxq = rx_queue;
 803         union avf_rx_desc rxd;
 804         struct rte_mbuf *rxe;
 805         struct rte_mbuf *first_seg = rxq->pkt_first_seg;
 806         struct rte_mbuf *last_seg = rxq->pkt_last_seg;
 807         struct rte_mbuf *nmb, *rxm;
 808         uint16_t rx_id = rxq->rx_tail;
 809         uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len;
 810         struct rte_eth_dev *dev;
 811         uint32_t rx_status;
 812         uint64_t qword1;
 813         uint64_t dma_addr;
 814         uint64_t pkt_flags;
 815
 816         volatile union avf_rx_desc *rx_ring = rxq->rx_ring;
 817         volatile union avf_rx_desc *rxdp;
 818         static const uint32_t ptype_tbl[UINT8_MAX + 1] __rte_cache_aligned = {
 819                 /* [0] reserved */
 820                 [1] = RTE_PTYPE_L2_ETHER,
 821                 /* [2] - [21] reserved */
 822                 [22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 823                         RTE_PTYPE_L4_FRAG,
 824                 [23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 825                         RTE_PTYPE_L4_NONFRAG,
 826                 [24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 827                         RTE_PTYPE_L4_UDP,
 828                 /* [25] reserved */
 829                 [26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 830                         RTE_PTYPE_L4_TCP,
 831                 [27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 832                         RTE_PTYPE_L4_SCTP,
 833                 [28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 834                         RTE_PTYPE_L4_ICMP,
 835                 /* All others reserved */
 836         };
 837
 838         while (nb_rx < nb_pkts) {
 839                 rxdp = &rx_ring[rx_id];
 840                 qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
 841                 rx_status = (qword1 & AVF_RXD_QW1_STATUS_MASK) >>
 842                             AVF_RXD_QW1_STATUS_SHIFT;
 843
 844                 /* Check the DD bit */
 845                 if (!(rx_status & (1 << AVF_RX_DESC_STATUS_DD_SHIFT)))
 846                         break;
 847                 AVF_DUMP_RX_DESC(rxq, rxdp, rx_id);
 848
 849                 nmb = rte_mbuf_raw_alloc(rxq->mp);
 850                 if (unlikely(!nmb)) {
 851                         PMD_RX_LOG(DEBUG, "RX mbuf alloc failed port_id=%u "
 852                                    "queue_id=%u", rxq->port_id, rxq->queue_id);
 853                         dev = &rte_eth_devices[rxq->port_id];
 854                         dev->data->rx_mbuf_alloc_failed++;
 855                         break;
 856                 }
 857
 858                 rxd = *rxdp;
 859                 nb_hold++;
 860                 rxe = rxq->sw_ring[rx_id];
 861                 rx_id++;
 862                 if (rx_id == rxq->nb_rx_desc)
 863                         rx_id = 0;
 864
 865                 /* Prefetch next mbuf */
 866                 rte_prefetch0(rxq->sw_ring[rx_id]);
 867
 868                 /* When next RX descriptor is on a cache line boundary,
 869                  * prefetch the next 4 RX descriptors and next 8 pointers
 870                  * to mbufs.
 871                  */
 872                 if ((rx_id & 0x3) == 0) {
 873                         rte_prefetch0(&rx_ring[rx_id]);
 874                         rte_prefetch0(rxq->sw_ring[rx_id]);
 875                 }
 876
 877                 rxm = rxe;
 878                 rxe = nmb;
 879                 dma_addr =
 880                         rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
 881
 882                 /* Set data buffer address and data length of the mbuf */
 883                 rxdp->read.hdr_addr = 0;
 884                 rxdp->read.pkt_addr = dma_addr;
 885                 rx_packet_len = (qword1 & AVF_RXD_QW1_LENGTH_PBUF_MASK) >>
 886                                  AVF_RXD_QW1_LENGTH_PBUF_SHIFT;
 887                 rxm->data_len = rx_packet_len;
 888                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 889
 890                 /* If this is the first buffer of the received packet, set the
 891                  * pointer to the first mbuf of the packet and initialize its
 892                  * context. Otherwise, update the total length and the number
 893                  * of segments of the current scattered packet, and update the
 894                  * pointer to the last mbuf of the current packet.
 895                  */
 896                 if (!first_seg) {
 897                         first_seg = rxm;
 898                         first_seg->nb_segs = 1;
 899                         first_seg->pkt_len = rx_packet_len;
 900                 } else {
 901                         first_seg->pkt_len =
 902                                 (uint16_t)(first_seg->pkt_len +
 903                                                 rx_packet_len);
 904                         first_seg->nb_segs++;
 905                         last_seg->next = rxm;
 906                 }
 907
 908                 /* If this is not the last buffer of the received packet,
 909                  * update the pointer to the last mbuf of the current scattered
 910                  * packet and continue to parse the RX ring.
 911                  */
 912                 if (!(rx_status & (1 << AVF_RX_DESC_STATUS_EOF_SHIFT))) {
 913                         last_seg = rxm;
 914                         continue;
 915                 }
 916
 917                 /* This is the last buffer of the received packet. If the CRC
 918                  * is not stripped by the hardware:
 919                  *  - Subtract the CRC length from the total packet length.
 920                  *  - If the last buffer only contains the whole CRC or a part
 921                  *  of it, free the mbuf associated to the last buffer. If part
 922                  *  of the CRC is also contained in the previous mbuf, subtract
 923                  *  the length of that CRC part from the data length of the
 924                  *  previous mbuf.
 925                  */
 926                 rxm->next = NULL;
 927                 if (unlikely(rxq->crc_len > 0)) {
 928                         first_seg->pkt_len -= ETHER_CRC_LEN;
 929                         if (rx_packet_len <= ETHER_CRC_LEN) {
 930                                 rte_pktmbuf_free_seg(rxm);
 931                                 first_seg->nb_segs--;
 932                                 last_seg->data_len =
 933                                         (uint16_t)(last_seg->data_len -
 934                                         (ETHER_CRC_LEN - rx_packet_len));
 935                                 last_seg->next = NULL;
 936                         } else
 937                                 rxm->data_len = (uint16_t)(rx_packet_len -
 938                                                                 ETHER_CRC_LEN);
 939                 }
 940
 941                 first_seg->port = rxq->port_id;
 942                 first_seg->ol_flags = 0;
 943                 avf_rxd_to_vlan_tci(first_seg, &rxd);
 944                 pkt_flags = avf_rxd_to_pkt_flags(qword1);
 945                 first_seg->packet_type =
 946                         ptype_tbl[(uint8_t)((qword1 &
 947                         AVF_RXD_QW1_PTYPE_MASK) >> AVF_RXD_QW1_PTYPE_SHIFT)];
 948
 949                 if (pkt_flags & PKT_RX_RSS_HASH)
 950                         first_seg->hash.rss =
 951                                 rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
 952
 953                 first_seg->ol_flags |= pkt_flags;
 954
 955                 /* Prefetch data of first segment, if configured to do so. */
 956                 rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr,
 957                                           first_seg->data_off));
 958                 rx_pkts[nb_rx++] = first_seg;
 959                 first_seg = NULL;
 960         }
 961
 962         /* Record index of the next RX descriptor to probe. */
 963         rxq->rx_tail = rx_id;
 964         rxq->pkt_first_seg = first_seg;
 965         rxq->pkt_last_seg = last_seg;
 966
 967         /* If the number of free RX descriptors is greater than the RX free
 968          * threshold of the queue, advance the Receive Descriptor Tail (RDT)
 969          * register. Update the RDT with the value of the last processed RX
 970          * descriptor minus 1, to guarantee that the RDT register is never
 971          * equal to the RDH register, which creates a "full" ring situtation
 972          * from the hardware point of view.
 973          */
 974         nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
 975         if (nb_hold > rxq->rx_free_thresh) {
 976                 PMD_RX_LOG(DEBUG, "port_id=%u queue_id=%u rx_tail=%u "
 977                            "nb_hold=%u nb_rx=%u",
 978                            rxq->port_id, rxq->queue_id,
 979                            rx_id, nb_hold, nb_rx);
 980                 rx_id = (uint16_t)(rx_id == 0 ?
 981                         (rxq->nb_rx_desc - 1) : (rx_id - 1));
 982                 AVF_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
 983                 nb_hold = 0;
 984         }
 985         rxq->nb_rx_hold = nb_hold;
 986
 987         return nb_rx;
 988 }
 989
 990 static inline int
 991 avf_xmit_cleanup(struct avf_tx_queue *txq)
 992 {
 993         struct avf_tx_entry *sw_ring = txq->sw_ring;
 994         uint16_t last_desc_cleaned = txq->last_desc_cleaned;
 995         uint16_t nb_tx_desc = txq->nb_tx_desc;
 996         uint16_t desc_to_clean_to;
 997         uint16_t nb_tx_to_clean;
 998
 999         volatile struct avf_tx_desc *txd = txq->tx_ring;
1000
1001         desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->rs_thresh);
1002         if (desc_to_clean_to >= nb_tx_desc)
1003                 desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
1004
1005         desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
1006         if ((txd[desc_to_clean_to].cmd_type_offset_bsz &
1007                         rte_cpu_to_le_64(AVF_TXD_QW1_DTYPE_MASK)) !=
1008                         rte_cpu_to_le_64(AVF_TX_DESC_DTYPE_DESC_DONE)) {
1009                 PMD_TX_FREE_LOG(DEBUG, "TX descriptor %4u is not done "
1010                                 "(port=%d queue=%d)", desc_to_clean_to,
1011                                 txq->port_id, txq->queue_id);
1012                 return -1;
1013         }
1014
1015         if (last_desc_cleaned > desc_to_clean_to)
1016                 nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
1017                                                         desc_to_clean_to);
1018         else
1019                 nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
1020                                         last_desc_cleaned);
1021
1022         txd[desc_to_clean_to].cmd_type_offset_bsz = 0;
1023
1024         txq->last_desc_cleaned = desc_to_clean_to;
1025         txq->nb_free = (uint16_t)(txq->nb_free + nb_tx_to_clean);
1026
1027         return 0;
1028 }
1029
1030 /* Check if the context descriptor is needed for TX offloading */
1031 static inline uint16_t
1032 avf_calc_context_desc(uint64_t flags)
1033 {
1034         static uint64_t mask = PKT_TX_TCP_SEG;
1035
1036         return (flags & mask) ? 1 : 0;
1037 }
1038
1039 static inline void
1040 avf_txd_enable_checksum(uint64_t ol_flags,
1041                         uint32_t *td_cmd,
1042                         uint32_t *td_offset,
1043                         union avf_tx_offload tx_offload)
1044 {
1045         /* Set MACLEN */
1046         *td_offset |= (tx_offload.l2_len >> 1) <<
1047                       AVF_TX_DESC_LENGTH_MACLEN_SHIFT;
1048
1049         /* Enable L3 checksum offloads */
1050         if (ol_flags & PKT_TX_IP_CKSUM) {
1051                 *td_cmd |= AVF_TX_DESC_CMD_IIPT_IPV4_CSUM;
1052                 *td_offset |= (tx_offload.l3_len >> 2) <<
1053                               AVF_TX_DESC_LENGTH_IPLEN_SHIFT;
1054         } else if (ol_flags & PKT_TX_IPV4) {
1055                 *td_cmd |= AVF_TX_DESC_CMD_IIPT_IPV4;
1056                 *td_offset |= (tx_offload.l3_len >> 2) <<
1057                               AVF_TX_DESC_LENGTH_IPLEN_SHIFT;
1058         } else if (ol_flags & PKT_TX_IPV6) {
1059                 *td_cmd |= AVF_TX_DESC_CMD_IIPT_IPV6;
1060                 *td_offset |= (tx_offload.l3_len >> 2) <<
1061                               AVF_TX_DESC_LENGTH_IPLEN_SHIFT;
1062         }
1063
1064         if (ol_flags & PKT_TX_TCP_SEG) {
1065                 *td_cmd |= AVF_TX_DESC_CMD_L4T_EOFT_TCP;
1066                 *td_offset |= (tx_offload.l4_len >> 2) <<
1067                               AVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1068                 return;
1069         }
1070
1071         /* Enable L4 checksum offloads */
1072         switch (ol_flags & PKT_TX_L4_MASK) {
1073         case PKT_TX_TCP_CKSUM:
1074                 *td_cmd |= AVF_TX_DESC_CMD_L4T_EOFT_TCP;
1075                 *td_offset |= (sizeof(struct tcp_hdr) >> 2) <<
1076                               AVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1077                 break;
1078         case PKT_TX_SCTP_CKSUM:
1079                 *td_cmd |= AVF_TX_DESC_CMD_L4T_EOFT_SCTP;
1080                 *td_offset |= (sizeof(struct sctp_hdr) >> 2) <<
1081                               AVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1082                 break;
1083         case PKT_TX_UDP_CKSUM:
1084                 *td_cmd |= AVF_TX_DESC_CMD_L4T_EOFT_UDP;
1085                 *td_offset |= (sizeof(struct udp_hdr) >> 2) <<
1086                               AVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1087                 break;
1088         default:
1089                 break;
1090         }
1091 }
1092
1093 /* set TSO context descriptor
1094  * support IP -> L4 and IP -> IP -> L4
1095  */
1096 static inline uint64_t
1097 avf_set_tso_ctx(struct rte_mbuf *mbuf, union avf_tx_offload tx_offload)
1098 {
1099         uint64_t ctx_desc = 0;
1100         uint32_t cd_cmd, hdr_len, cd_tso_len;
1101
1102         if (!tx_offload.l4_len) {
1103                 PMD_TX_LOG(DEBUG, "L4 length set to 0");
1104                 return ctx_desc;
1105         }
1106
1107         /* in case of non tunneling packet, the outer_l2_len and
1108          * outer_l3_len must be 0.
1109          */
1110         hdr_len = tx_offload.l2_len +
1111                   tx_offload.l3_len +
1112                   tx_offload.l4_len;
1113
1114         cd_cmd = AVF_TX_CTX_DESC_TSO;
1115         cd_tso_len = mbuf->pkt_len - hdr_len;
1116         ctx_desc |= ((uint64_t)cd_cmd << AVF_TXD_CTX_QW1_CMD_SHIFT) |
1117                      ((uint64_t)cd_tso_len << AVF_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1118                      ((uint64_t)mbuf->tso_segsz << AVF_TXD_CTX_QW1_MSS_SHIFT);
1119
1120         return ctx_desc;
1121 }
1122
1123 /* Construct the tx flags */
1124 static inline uint64_t
1125 avf_build_ctob(uint32_t td_cmd, uint32_t td_offset, unsigned int size,
1126                uint32_t td_tag)
1127 {
1128         return rte_cpu_to_le_64(AVF_TX_DESC_DTYPE_DATA |
1129                                 ((uint64_t)td_cmd  << AVF_TXD_QW1_CMD_SHIFT) |
1130                                 ((uint64_t)td_offset <<
1131                                  AVF_TXD_QW1_OFFSET_SHIFT) |
1132                                 ((uint64_t)size  <<
1133                                  AVF_TXD_QW1_TX_BUF_SZ_SHIFT) |
1134                                 ((uint64_t)td_tag  <<
1135                                  AVF_TXD_QW1_L2TAG1_SHIFT));
1136 }
1137
1138 /* TX function */
1139 uint16_t
1140 avf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1141 {
1142         volatile struct avf_tx_desc *txd;
1143         volatile struct avf_tx_desc *txr;
1144         struct avf_tx_queue *txq;
1145         struct avf_tx_entry *sw_ring;
1146         struct avf_tx_entry *txe, *txn;
1147         struct rte_mbuf *tx_pkt;
1148         struct rte_mbuf *m_seg;
1149         uint16_t tx_id;
1150         uint16_t nb_tx;
1151         uint32_t td_cmd;
1152         uint32_t td_offset;
1153         uint32_t td_tag;
1154         uint64_t ol_flags;
1155         uint16_t nb_used;
1156         uint16_t nb_ctx;
1157         uint16_t tx_last;
1158         uint16_t slen;
1159         uint64_t buf_dma_addr;
1160         union avf_tx_offload tx_offload = {0};
1161
1162         txq = tx_queue;
1163         sw_ring = txq->sw_ring;
1164         txr = txq->tx_ring;
1165         tx_id = txq->tx_tail;
1166         txe = &sw_ring[tx_id];
1167
1168         /* Check if the descriptor ring needs to be cleaned. */
1169         if (txq->nb_free < txq->free_thresh)
1170                 avf_xmit_cleanup(txq);
1171
1172         for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1173                 td_cmd = 0;
1174                 td_tag = 0;
1175                 td_offset = 0;
1176
1177                 tx_pkt = *tx_pkts++;
1178                 RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf);
1179
1180                 ol_flags = tx_pkt->ol_flags;
1181                 tx_offload.l2_len = tx_pkt->l2_len;
1182                 tx_offload.l3_len = tx_pkt->l3_len;
1183                 tx_offload.l4_len = tx_pkt->l4_len;
1184                 tx_offload.tso_segsz = tx_pkt->tso_segsz;
1185
1186                 /* Calculate the number of context descriptors needed. */
1187                 nb_ctx = avf_calc_context_desc(ol_flags);
1188
1189                 /* The number of descriptors that must be allocated for
1190                  * a packet equals to the number of the segments of that
1191                  * packet plus 1 context descriptor if needed.
1192                  */
1193                 nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx);
1194                 tx_last = (uint16_t)(tx_id + nb_used - 1);
1195
1196                 /* Circular ring */
1197                 if (tx_last >= txq->nb_tx_desc)
1198                         tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
1199
1200                 PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u"
1201                            " tx_first=%u tx_last=%u",
1202                            txq->port_id, txq->queue_id, tx_id, tx_last);
1203
1204                 if (nb_used > txq->nb_free) {
1205                         if (avf_xmit_cleanup(txq)) {
1206                                 if (nb_tx == 0)
1207                                         return 0;
1208                                 goto end_of_tx;
1209                         }
1210                         if (unlikely(nb_used > txq->rs_thresh)) {
1211                                 while (nb_used > txq->nb_free) {
1212                                         if (avf_xmit_cleanup(txq)) {
1213                                                 if (nb_tx == 0)
1214                                                         return 0;
1215                                                 goto end_of_tx;
1216                                         }
1217                                 }
1218                         }
1219                 }
1220
1221                 /* Descriptor based VLAN insertion */
1222                 if (ol_flags & PKT_TX_VLAN_PKT) {
1223                         td_cmd |= AVF_TX_DESC_CMD_IL2TAG1;
1224                         td_tag = tx_pkt->vlan_tci;
1225                 }
1226
1227                 /* According to datasheet, the bit2 is reserved and must be
1228                  * set to 1.
1229                  */
1230                 td_cmd |= 0x04;
1231
1232                 /* Enable checksum offloading */
1233                 if (ol_flags & AVF_TX_CKSUM_OFFLOAD_MASK)
1234                         avf_txd_enable_checksum(ol_flags, &td_cmd,
1235                                                 &td_offset, tx_offload);
1236
1237                 if (nb_ctx) {
1238                         /* Setup TX context descriptor if required */
1239                         volatile struct avf_tx_context_desc *ctx_txd =
1240                                 (volatile struct avf_tx_context_desc *)
1241                                         &txr[tx_id];
1242                         uint16_t cd_l2tag2 = 0;
1243                         uint64_t cd_type_cmd_tso_mss =
1244                                 AVF_TX_DESC_DTYPE_CONTEXT;
1245
1246                         txn = &sw_ring[txe->next_id];
1247                         RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
1248                         if (txe->mbuf) {
1249                                 rte_pktmbuf_free_seg(txe->mbuf);
1250                                 txe->mbuf = NULL;
1251                         }
1252
1253                         /* TSO enabled */
1254                         if (ol_flags & PKT_TX_TCP_SEG)
1255                                 cd_type_cmd_tso_mss |=
1256                                         avf_set_tso_ctx(tx_pkt, tx_offload);
1257
1258                         AVF_DUMP_TX_DESC(txq, ctx_txd, tx_id);
1259                         txe->last_id = tx_last;
1260                         tx_id = txe->next_id;
1261                         txe = txn;
1262                 }
1263
1264                 m_seg = tx_pkt;
1265                 do {
1266                         txd = &txr[tx_id];
1267                         txn = &sw_ring[txe->next_id];
1268
1269                         if (txe->mbuf)
1270                                 rte_pktmbuf_free_seg(txe->mbuf);
1271                         txe->mbuf = m_seg;
1272
1273                         /* Setup TX Descriptor */
1274                         slen = m_seg->data_len;
1275                         buf_dma_addr = rte_mbuf_data_iova(m_seg);
1276                         txd->buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
1277                         txd->cmd_type_offset_bsz = avf_build_ctob(td_cmd,
1278                                                                   td_offset,
1279                                                                   slen,
1280                                                                   td_tag);
1281
1282                         AVF_DUMP_TX_DESC(txq, txd, tx_id);
1283                         txe->last_id = tx_last;
1284                         tx_id = txe->next_id;
1285                         txe = txn;
1286                         m_seg = m_seg->next;
1287                 } while (m_seg);
1288
1289                 /* The last packet data descriptor needs End Of Packet (EOP) */
1290                 td_cmd |= AVF_TX_DESC_CMD_EOP;
1291                 txq->nb_used = (uint16_t)(txq->nb_used + nb_used);
1292                 txq->nb_free = (uint16_t)(txq->nb_free - nb_used);
1293
1294                 if (txq->nb_used >= txq->rs_thresh) {
1295                         PMD_TX_LOG(DEBUG, "Setting RS bit on TXD id="
1296                                    "%4u (port=%d queue=%d)",
1297                                    tx_last, txq->port_id, txq->queue_id);
1298
1299                         td_cmd |= AVF_TX_DESC_CMD_RS;
1300
1301                         /* Update txq RS bit counters */
1302                         txq->nb_used = 0;
1303                 }
1304
1305                 txd->cmd_type_offset_bsz |=
1306                         rte_cpu_to_le_64(((uint64_t)td_cmd) <<
1307                                          AVF_TXD_QW1_CMD_SHIFT);
1308                 AVF_DUMP_TX_DESC(txq, txd, tx_id);
1309         }
1310
1311 end_of_tx:
1312         rte_wmb();
1313
1314         PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
1315                    txq->port_id, txq->queue_id, tx_id, nb_tx);
1316
1317         AVF_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id);
1318         txq->tx_tail = tx_id;
1319
1320         return nb_tx;
1321 }
1322
1323 /* TX prep functions */
1324 uint16_t
1325 avf_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
1326               uint16_t nb_pkts)
1327 {
1328         int i, ret;
1329         uint64_t ol_flags;
1330         struct rte_mbuf *m;
1331
1332         for (i = 0; i < nb_pkts; i++) {
1333                 m = tx_pkts[i];
1334                 ol_flags = m->ol_flags;
1335
1336                 /* Check condition for nb_segs > AVF_TX_MAX_MTU_SEG. */
1337                 if (!(ol_flags & PKT_TX_TCP_SEG)) {
1338                         if (m->nb_segs > AVF_TX_MAX_MTU_SEG) {
1339                                 rte_errno = -EINVAL;
1340                                 return i;
1341                         }
1342                 } else if ((m->tso_segsz < AVF_MIN_TSO_MSS) ||
1343                            (m->tso_segsz > AVF_MAX_TSO_MSS)) {
1344                         /* MSS outside the range are considered malicious */
1345                         rte_errno = -EINVAL;
1346                         return i;
1347                 }
1348
1349                 if (ol_flags & AVF_TX_OFFLOAD_NOTSUP_MASK) {
1350                         rte_errno = -ENOTSUP;
1351                         return i;
1352                 }
1353
1354 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
1355                 ret = rte_validate_tx_offload(m);
1356                 if (ret != 0) {
1357                         rte_errno = ret;
1358                         return i;
1359                 }
1360 #endif
1361                 ret = rte_net_intel_cksum_prepare(m);
1362                 if (ret != 0) {
1363                         rte_errno = ret;
1364                         return i;
1365                 }
1366         }
1367
1368         return i;
1369 }
1370
1371 /* choose rx function*/
1372 void
1373 avf_set_rx_function(struct rte_eth_dev *dev)
1374 {
1375         if (dev->data->scattered_rx)
1376                 dev->rx_pkt_burst = avf_recv_scattered_pkts;
1377         else
1378                 dev->rx_pkt_burst = avf_recv_pkts;
1379 }
1380
1381 /* choose tx function*/
1382 void
1383 avf_set_tx_function(struct rte_eth_dev *dev)
1384 {
1385         dev->tx_pkt_burst = avf_xmit_pkts;
1386         dev->tx_pkt_prepare = avf_prep_pkts;
1387 }
1388
1389 void
1390 avf_dev_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1391                      struct rte_eth_rxq_info *qinfo)
1392 {
1393         struct avf_rx_queue *rxq;
1394
1395         rxq = dev->data->rx_queues[queue_id];
1396
1397         qinfo->mp = rxq->mp;
1398         qinfo->scattered_rx = dev->data->scattered_rx;
1399         qinfo->nb_desc = rxq->nb_rx_desc;
1400
1401         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
1402         qinfo->conf.rx_drop_en = TRUE;
1403         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
1404 }
1405
1406 void
1407 avf_dev_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1408                      struct rte_eth_txq_info *qinfo)
1409 {
1410         struct avf_tx_queue *txq;
1411
1412         txq = dev->data->tx_queues[queue_id];
1413
1414         qinfo->nb_desc = txq->nb_tx_desc;
1415
1416         qinfo->conf.tx_free_thresh = txq->free_thresh;
1417         qinfo->conf.tx_rs_thresh = txq->rs_thresh;
1418         qinfo->conf.txq_flags = txq->txq_flags;
1419         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1420 }
1421
1422 /* Get the number of used descriptors of a rx queue */
1423 uint32_t
1424 avf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id)
1425 {
1426 #define AVF_RXQ_SCAN_INTERVAL 4
1427         volatile union avf_rx_desc *rxdp;
1428         struct avf_rx_queue *rxq;
1429         uint16_t desc = 0;
1430
1431         rxq = dev->data->rx_queues[queue_id];
1432         rxdp = &rxq->rx_ring[rxq->rx_tail];
1433         while ((desc < rxq->nb_rx_desc) &&
1434                ((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
1435                  AVF_RXD_QW1_STATUS_MASK) >> AVF_RXD_QW1_STATUS_SHIFT) &
1436                (1 << AVF_RX_DESC_STATUS_DD_SHIFT)) {
1437                 /* Check the DD bit of a rx descriptor of each 4 in a group,
1438                  * to avoid checking too frequently and downgrading performance
1439                  * too much.
1440                  */
1441                 desc += AVF_RXQ_SCAN_INTERVAL;
1442                 rxdp += AVF_RXQ_SCAN_INTERVAL;
1443                 if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
1444                         rxdp = &(rxq->rx_ring[rxq->rx_tail +
1445                                         desc - rxq->nb_rx_desc]);
1446         }
1447
1448         return desc;
1449 }
1450
1451 int
1452 avf_dev_rx_desc_status(void *rx_queue, uint16_t offset)
1453 {
1454         struct avf_rx_queue *rxq = rx_queue;
1455         volatile uint64_t *status;
1456         uint64_t mask;
1457         uint32_t desc;
1458
1459         if (unlikely(offset >= rxq->nb_rx_desc))
1460                 return -EINVAL;
1461
1462         if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
1463                 return RTE_ETH_RX_DESC_UNAVAIL;
1464
1465         desc = rxq->rx_tail + offset;
1466         if (desc >= rxq->nb_rx_desc)
1467                 desc -= rxq->nb_rx_desc;
1468
1469         status = &rxq->rx_ring[desc].wb.qword1.status_error_len;
1470         mask = rte_le_to_cpu_64((1ULL << AVF_RX_DESC_STATUS_DD_SHIFT)
1471                 << AVF_RXD_QW1_STATUS_SHIFT);
1472         if (*status & mask)
1473                 return RTE_ETH_RX_DESC_DONE;
1474
1475         return RTE_ETH_RX_DESC_AVAIL;
1476 }
1477
1478 int
1479 avf_dev_tx_desc_status(void *tx_queue, uint16_t offset)
1480 {
1481         struct avf_tx_queue *txq = tx_queue;
1482         volatile uint64_t *status;
1483         uint64_t mask, expect;
1484         uint32_t desc;
1485
1486         if (unlikely(offset >= txq->nb_tx_desc))
1487                 return -EINVAL;
1488
1489         desc = txq->tx_tail + offset;
1490         /* go to next desc that has the RS bit */
1491         desc = ((desc + txq->rs_thresh - 1) / txq->rs_thresh) *
1492                 txq->rs_thresh;
1493         if (desc >= txq->nb_tx_desc) {
1494                 desc -= txq->nb_tx_desc;
1495                 if (desc >= txq->nb_tx_desc)
1496                         desc -= txq->nb_tx_desc;
1497         }
1498
1499         status = &txq->tx_ring[desc].cmd_type_offset_bsz;
1500         mask = rte_le_to_cpu_64(AVF_TXD_QW1_DTYPE_MASK);
1501         expect = rte_cpu_to_le_64(
1502                  AVF_TX_DESC_DTYPE_DESC_DONE << AVF_TXD_QW1_DTYPE_SHIFT);
1503         if ((*status & mask) == expect)
1504                 return RTE_ETH_TX_DESC_DONE;
1505
1506         return RTE_ETH_TX_DESC_FULL;
1507 }