drivers/net/qede/qede_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2016 - 2018 Cavium Inc.
   3  * All rights reserved.
   4  * www.cavium.com
   5  */
   6
   7 #include <rte_net.h>
   8 #include "qede_rxtx.h"
   9
  10 static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq)
  11 {
  12         struct rte_mbuf *new_mb = NULL;
  13         struct eth_rx_bd *rx_bd;
  14         dma_addr_t mapping;
  15         uint16_t idx = rxq->sw_rx_prod & NUM_RX_BDS(rxq);
  16
  17         new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
  18         if (unlikely(!new_mb)) {
  19                 PMD_RX_LOG(ERR, rxq,
  20                            "Failed to allocate rx buffer "
  21                            "sw_rx_prod %u sw_rx_cons %u mp entries %u free %u",
  22                            idx, rxq->sw_rx_cons & NUM_RX_BDS(rxq),
  23                            rte_mempool_avail_count(rxq->mb_pool),
  24                            rte_mempool_in_use_count(rxq->mb_pool));
  25                 return -ENOMEM;
  26         }
  27         rxq->sw_rx_ring[idx] = new_mb;
  28         mapping = rte_mbuf_data_iova_default(new_mb);
  29         /* Advance PROD and get BD pointer */
  30         rx_bd = (struct eth_rx_bd *)ecore_chain_produce(&rxq->rx_bd_ring);
  31         rx_bd->addr.hi = rte_cpu_to_le_32(U64_HI(mapping));
  32         rx_bd->addr.lo = rte_cpu_to_le_32(U64_LO(mapping));
  33         rxq->sw_rx_prod++;
  34         return 0;
  35 }
  36
  37 #define QEDE_MAX_BULK_ALLOC_COUNT 512
  38
  39 static inline int qede_alloc_rx_bulk_mbufs(struct qede_rx_queue *rxq, int count)
  40 {
  41         struct rte_mbuf *mbuf = NULL;
  42         struct eth_rx_bd *rx_bd;
  43         dma_addr_t mapping;
  44         int i, ret = 0;
  45         uint16_t idx;
  46         uint16_t mask = NUM_RX_BDS(rxq);
  47
  48         if (count > QEDE_MAX_BULK_ALLOC_COUNT)
  49                 count = QEDE_MAX_BULK_ALLOC_COUNT;
  50
  51         idx = rxq->sw_rx_prod & NUM_RX_BDS(rxq);
  52
  53         if (count > mask - idx + 1)
  54                 count = mask - idx + 1;
  55
  56         ret = rte_mempool_get_bulk(rxq->mb_pool, (void **)&rxq->sw_rx_ring[idx],
  57                                    count);
  58
  59         if (unlikely(ret)) {
  60                 PMD_RX_LOG(ERR, rxq,
  61                            "Failed to allocate %d rx buffers "
  62                             "sw_rx_prod %u sw_rx_cons %u mp entries %u free %u",
  63                             count,
  64                             rxq->sw_rx_prod & NUM_RX_BDS(rxq),
  65                             rxq->sw_rx_cons & NUM_RX_BDS(rxq),
  66                             rte_mempool_avail_count(rxq->mb_pool),
  67                             rte_mempool_in_use_count(rxq->mb_pool));
  68                 return -ENOMEM;
  69         }
  70
  71         for (i = 0; i < count; i++) {
  72                 rte_prefetch0(rxq->sw_rx_ring[(idx + 1) & NUM_RX_BDS(rxq)]);
  73                 mbuf = rxq->sw_rx_ring[idx & NUM_RX_BDS(rxq)];
  74
  75                 mapping = rte_mbuf_data_iova_default(mbuf);
  76                 rx_bd = (struct eth_rx_bd *)
  77                         ecore_chain_produce(&rxq->rx_bd_ring);
  78                 rx_bd->addr.hi = rte_cpu_to_le_32(U64_HI(mapping));
  79                 rx_bd->addr.lo = rte_cpu_to_le_32(U64_LO(mapping));
  80                 idx++;
  81         }
  82         rxq->sw_rx_prod = idx;
  83
  84         return 0;
  85 }
  86
  87 /* Criterias for calculating Rx buffer size -
  88  * 1) rx_buf_size should not exceed the size of mbuf
  89  * 2) In scattered_rx mode - minimum rx_buf_size should be
  90  *    (MTU + Maximum L2 Header Size + 2) / ETH_RX_MAX_BUFF_PER_PKT
  91  * 3) In regular mode - minimum rx_buf_size should be
  92  *    (MTU + Maximum L2 Header Size + 2)
  93  *    In above cases +2 corrosponds to 2 bytes padding in front of L2
  94  *    header.
  95  * 4) rx_buf_size should be cacheline-size aligned. So considering
  96  *    criteria 1, we need to adjust the size to floor instead of ceil,
  97  *    so that we don't exceed mbuf size while ceiling rx_buf_size.
  98  */
  99 int
 100 qede_calc_rx_buf_size(struct rte_eth_dev *dev, uint16_t mbufsz,
 101                       uint16_t max_frame_size)
 102 {
 103         struct qede_dev *qdev = QEDE_INIT_QDEV(dev);
 104         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 105         int rx_buf_size;
 106
 107         if (dev->data->scattered_rx) {
 108                 /* per HW limitation, only ETH_RX_MAX_BUFF_PER_PKT number of
 109                  * bufferes can be used for single packet. So need to make sure
 110                  * mbuf size is sufficient enough for this.
 111                  */
 112                 if ((mbufsz * ETH_RX_MAX_BUFF_PER_PKT) <
 113                      (max_frame_size + QEDE_ETH_OVERHEAD)) {
 114                         DP_ERR(edev, "mbuf %d size is not enough to hold max fragments (%d) for max rx packet length (%d)\n",
 115                                mbufsz, ETH_RX_MAX_BUFF_PER_PKT, max_frame_size);
 116                         return -EINVAL;
 117                 }
 118
 119                 rx_buf_size = RTE_MAX(mbufsz,
 120                                       (max_frame_size + QEDE_ETH_OVERHEAD) /
 121                                        ETH_RX_MAX_BUFF_PER_PKT);
 122         } else {
 123                 rx_buf_size = max_frame_size + QEDE_ETH_OVERHEAD;
 124         }
 125
 126         /* Align to cache-line size if needed */
 127         return QEDE_FLOOR_TO_CACHE_LINE_SIZE(rx_buf_size);
 128 }
 129
 130 static struct qede_rx_queue *
 131 qede_alloc_rx_queue_mem(struct rte_eth_dev *dev,
 132                         uint16_t queue_idx,
 133                         uint16_t nb_desc,
 134                         unsigned int socket_id,
 135                         struct rte_mempool *mp,
 136                         uint16_t bufsz)
 137 {
 138         struct qede_dev *qdev = QEDE_INIT_QDEV(dev);
 139         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 140         struct qede_rx_queue *rxq;
 141         size_t size;
 142         int rc;
 143
 144         /* First allocate the rx queue data structure */
 145         rxq = rte_zmalloc_socket("qede_rx_queue", sizeof(struct qede_rx_queue),
 146                                  RTE_CACHE_LINE_SIZE, socket_id);
 147
 148         if (!rxq) {
 149                 DP_ERR(edev, "Unable to allocate memory for rxq on socket %u",
 150                           socket_id);
 151                 return NULL;
 152         }
 153
 154         rxq->qdev = qdev;
 155         rxq->mb_pool = mp;
 156         rxq->nb_rx_desc = nb_desc;
 157         rxq->queue_id = queue_idx;
 158         rxq->port_id = dev->data->port_id;
 159
 160
 161         rxq->rx_buf_size = bufsz;
 162
 163         DP_INFO(edev, "mtu %u mbufsz %u bd_max_bytes %u scatter_mode %d\n",
 164                 qdev->mtu, bufsz, rxq->rx_buf_size, dev->data->scattered_rx);
 165
 166         /* Allocate the parallel driver ring for Rx buffers */
 167         size = sizeof(*rxq->sw_rx_ring) * rxq->nb_rx_desc;
 168         rxq->sw_rx_ring = rte_zmalloc_socket("sw_rx_ring", size,
 169                                              RTE_CACHE_LINE_SIZE, socket_id);
 170         if (!rxq->sw_rx_ring) {
 171                 DP_ERR(edev, "Memory allocation fails for sw_rx_ring on"
 172                        " socket %u\n", socket_id);
 173                 rte_free(rxq);
 174                 return NULL;
 175         }
 176
 177         /* Allocate FW Rx ring  */
 178         rc = qdev->ops->common->chain_alloc(edev,
 179                                             ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
 180                                             ECORE_CHAIN_MODE_NEXT_PTR,
 181                                             ECORE_CHAIN_CNT_TYPE_U16,
 182                                             rxq->nb_rx_desc,
 183                                             sizeof(struct eth_rx_bd),
 184                                             &rxq->rx_bd_ring,
 185                                             NULL);
 186
 187         if (rc != ECORE_SUCCESS) {
 188                 DP_ERR(edev, "Memory allocation fails for RX BD ring"
 189                        " on socket %u\n", socket_id);
 190                 rte_free(rxq->sw_rx_ring);
 191                 rte_free(rxq);
 192                 return NULL;
 193         }
 194
 195         /* Allocate FW completion ring */
 196         rc = qdev->ops->common->chain_alloc(edev,
 197                                             ECORE_CHAIN_USE_TO_CONSUME,
 198                                             ECORE_CHAIN_MODE_PBL,
 199                                             ECORE_CHAIN_CNT_TYPE_U16,
 200                                             rxq->nb_rx_desc,
 201                                             sizeof(union eth_rx_cqe),
 202                                             &rxq->rx_comp_ring,
 203                                             NULL);
 204
 205         if (rc != ECORE_SUCCESS) {
 206                 DP_ERR(edev, "Memory allocation fails for RX CQE ring"
 207                        " on socket %u\n", socket_id);
 208                 qdev->ops->common->chain_free(edev, &rxq->rx_bd_ring);
 209                 rte_free(rxq->sw_rx_ring);
 210                 rte_free(rxq);
 211                 return NULL;
 212         }
 213
 214         return rxq;
 215 }
 216
 217 int
 218 qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qid,
 219                     uint16_t nb_desc, unsigned int socket_id,
 220                     __rte_unused const struct rte_eth_rxconf *rx_conf,
 221                     struct rte_mempool *mp)
 222 {
 223         struct qede_dev *qdev = QEDE_INIT_QDEV(dev);
 224         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 225         struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
 226         struct qede_rx_queue *rxq;
 227         uint16_t max_rx_pkt_len;
 228         uint16_t bufsz;
 229         int rc;
 230
 231         PMD_INIT_FUNC_TRACE(edev);
 232
 233         /* Note: Ring size/align is controlled by struct rte_eth_desc_lim */
 234         if (!rte_is_power_of_2(nb_desc)) {
 235                 DP_ERR(edev, "Ring size %u is not power of 2\n",
 236                           nb_desc);
 237                 return -EINVAL;
 238         }
 239
 240         /* Free memory prior to re-allocation if needed... */
 241         if (dev->data->rx_queues[qid] != NULL) {
 242                 qede_rx_queue_release(dev->data->rx_queues[qid]);
 243                 dev->data->rx_queues[qid] = NULL;
 244         }
 245
 246         max_rx_pkt_len = (uint16_t)rxmode->max_rx_pkt_len;
 247
 248         /* Fix up RX buffer size */
 249         bufsz = (uint16_t)rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM;
 250         /* cache align the mbuf size to simplfy rx_buf_size calculation */
 251         bufsz = QEDE_FLOOR_TO_CACHE_LINE_SIZE(bufsz);
 252         if ((rxmode->offloads & DEV_RX_OFFLOAD_SCATTER) ||
 253             (max_rx_pkt_len + QEDE_ETH_OVERHEAD) > bufsz) {
 254                 if (!dev->data->scattered_rx) {
 255                         DP_INFO(edev, "Forcing scatter-gather mode\n");
 256                         dev->data->scattered_rx = 1;
 257                 }
 258         }
 259
 260         rc = qede_calc_rx_buf_size(dev, bufsz, max_rx_pkt_len);
 261         if (rc < 0)
 262                 return rc;
 263
 264         bufsz = rc;
 265
 266         if (ECORE_IS_CMT(edev)) {
 267                 rxq = qede_alloc_rx_queue_mem(dev, qid * 2, nb_desc,
 268                                               socket_id, mp, bufsz);
 269                 if (!rxq)
 270                         return -ENOMEM;
 271
 272                 qdev->fp_array[qid * 2].rxq = rxq;
 273                 rxq = qede_alloc_rx_queue_mem(dev, qid * 2 + 1, nb_desc,
 274                                               socket_id, mp, bufsz);
 275                 if (!rxq)
 276                         return -ENOMEM;
 277
 278                 qdev->fp_array[qid * 2 + 1].rxq = rxq;
 279                 /* provide per engine fp struct as rx queue */
 280                 dev->data->rx_queues[qid] = &qdev->fp_array_cmt[qid];
 281         } else {
 282                 rxq = qede_alloc_rx_queue_mem(dev, qid, nb_desc,
 283                                               socket_id, mp, bufsz);
 284                 if (!rxq)
 285                         return -ENOMEM;
 286
 287                 dev->data->rx_queues[qid] = rxq;
 288                 qdev->fp_array[qid].rxq = rxq;
 289         }
 290
 291         DP_INFO(edev, "rxq %d num_desc %u rx_buf_size=%u socket %u\n",
 292                   qid, nb_desc, rxq->rx_buf_size, socket_id);
 293
 294         return 0;
 295 }
 296
 297 static void
 298 qede_rx_queue_reset(__rte_unused struct qede_dev *qdev,
 299                     struct qede_rx_queue *rxq)
 300 {
 301         DP_INFO(&qdev->edev, "Reset RX queue %u\n", rxq->queue_id);
 302         ecore_chain_reset(&rxq->rx_bd_ring);
 303         ecore_chain_reset(&rxq->rx_comp_ring);
 304         rxq->sw_rx_prod = 0;
 305         rxq->sw_rx_cons = 0;
 306         *rxq->hw_cons_ptr = 0;
 307 }
 308
 309 static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq)
 310 {
 311         uint16_t i;
 312
 313         if (rxq->sw_rx_ring) {
 314                 for (i = 0; i < rxq->nb_rx_desc; i++) {
 315                         if (rxq->sw_rx_ring[i]) {
 316                                 rte_pktmbuf_free(rxq->sw_rx_ring[i]);
 317                                 rxq->sw_rx_ring[i] = NULL;
 318                         }
 319                 }
 320         }
 321 }
 322
 323 static void _qede_rx_queue_release(struct qede_dev *qdev,
 324                                    struct ecore_dev *edev,
 325                                    struct qede_rx_queue *rxq)
 326 {
 327         qede_rx_queue_release_mbufs(rxq);
 328         qdev->ops->common->chain_free(edev, &rxq->rx_bd_ring);
 329         qdev->ops->common->chain_free(edev, &rxq->rx_comp_ring);
 330         rte_free(rxq->sw_rx_ring);
 331         rte_free(rxq);
 332 }
 333
 334 void qede_rx_queue_release(void *rx_queue)
 335 {
 336         struct qede_rx_queue *rxq = rx_queue;
 337         struct qede_fastpath_cmt *fp_cmt;
 338         struct qede_dev *qdev;
 339         struct ecore_dev *edev;
 340
 341         if (rxq) {
 342                 qdev = rxq->qdev;
 343                 edev = QEDE_INIT_EDEV(qdev);
 344                 PMD_INIT_FUNC_TRACE(edev);
 345                 if (ECORE_IS_CMT(edev)) {
 346                         fp_cmt = rx_queue;
 347                         _qede_rx_queue_release(qdev, edev, fp_cmt->fp0->rxq);
 348                         _qede_rx_queue_release(qdev, edev, fp_cmt->fp1->rxq);
 349                 } else {
 350                         _qede_rx_queue_release(qdev, edev, rxq);
 351                 }
 352         }
 353 }
 354
 355 /* Stops a given RX queue in the HW */
 356 static int qede_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 357 {
 358         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
 359         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 360         struct ecore_hwfn *p_hwfn;
 361         struct qede_rx_queue *rxq;
 362         int hwfn_index;
 363         int rc;
 364
 365         if (rx_queue_id < qdev->num_rx_queues) {
 366                 rxq = qdev->fp_array[rx_queue_id].rxq;
 367                 hwfn_index = rx_queue_id % edev->num_hwfns;
 368                 p_hwfn = &edev->hwfns[hwfn_index];
 369                 rc = ecore_eth_rx_queue_stop(p_hwfn, rxq->handle,
 370                                 true, false);
 371                 if (rc != ECORE_SUCCESS) {
 372                         DP_ERR(edev, "RX queue %u stop fails\n", rx_queue_id);
 373                         return -1;
 374                 }
 375                 qede_rx_queue_release_mbufs(rxq);
 376                 qede_rx_queue_reset(qdev, rxq);
 377                 eth_dev->data->rx_queue_state[rx_queue_id] =
 378                         RTE_ETH_QUEUE_STATE_STOPPED;
 379                 DP_INFO(edev, "RX queue %u stopped\n", rx_queue_id);
 380         } else {
 381                 DP_ERR(edev, "RX queue %u is not in range\n", rx_queue_id);
 382                 rc = -EINVAL;
 383         }
 384
 385         return rc;
 386 }
 387
 388 static struct qede_tx_queue *
 389 qede_alloc_tx_queue_mem(struct rte_eth_dev *dev,
 390                         uint16_t queue_idx,
 391                         uint16_t nb_desc,
 392                         unsigned int socket_id,
 393                         const struct rte_eth_txconf *tx_conf)
 394 {
 395         struct qede_dev *qdev = dev->data->dev_private;
 396         struct ecore_dev *edev = &qdev->edev;
 397         struct qede_tx_queue *txq;
 398         int rc;
 399         size_t sw_tx_ring_size;
 400
 401         txq = rte_zmalloc_socket("qede_tx_queue", sizeof(struct qede_tx_queue),
 402                                  RTE_CACHE_LINE_SIZE, socket_id);
 403
 404         if (txq == NULL) {
 405                 DP_ERR(edev,
 406                        "Unable to allocate memory for txq on socket %u",
 407                        socket_id);
 408                 return NULL;
 409         }
 410
 411         txq->nb_tx_desc = nb_desc;
 412         txq->qdev = qdev;
 413         txq->port_id = dev->data->port_id;
 414
 415         rc = qdev->ops->common->chain_alloc(edev,
 416                                             ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
 417                                             ECORE_CHAIN_MODE_PBL,
 418                                             ECORE_CHAIN_CNT_TYPE_U16,
 419                                             txq->nb_tx_desc,
 420                                             sizeof(union eth_tx_bd_types),
 421                                             &txq->tx_pbl,
 422                                             NULL);
 423         if (rc != ECORE_SUCCESS) {
 424                 DP_ERR(edev,
 425                        "Unable to allocate memory for txbd ring on socket %u",
 426                        socket_id);
 427                 qede_tx_queue_release(txq);
 428                 return NULL;
 429         }
 430
 431         /* Allocate software ring */
 432         sw_tx_ring_size = sizeof(txq->sw_tx_ring) * txq->nb_tx_desc;
 433         txq->sw_tx_ring = rte_zmalloc_socket("txq->sw_tx_ring",
 434                                              sw_tx_ring_size,
 435                                              RTE_CACHE_LINE_SIZE, socket_id);
 436
 437         if (!txq->sw_tx_ring) {
 438                 DP_ERR(edev,
 439                        "Unable to allocate memory for txbd ring on socket %u",
 440                        socket_id);
 441                 qdev->ops->common->chain_free(edev, &txq->tx_pbl);
 442                 qede_tx_queue_release(txq);
 443                 return NULL;
 444         }
 445
 446         txq->queue_id = queue_idx;
 447
 448         txq->nb_tx_avail = txq->nb_tx_desc;
 449
 450         txq->tx_free_thresh =
 451             tx_conf->tx_free_thresh ? tx_conf->tx_free_thresh :
 452             (txq->nb_tx_desc - QEDE_DEFAULT_TX_FREE_THRESH);
 453
 454         DP_INFO(edev,
 455                   "txq %u num_desc %u tx_free_thresh %u socket %u\n",
 456                   queue_idx, nb_desc, txq->tx_free_thresh, socket_id);
 457         return txq;
 458 }
 459
 460 int
 461 qede_tx_queue_setup(struct rte_eth_dev *dev,
 462                     uint16_t queue_idx,
 463                     uint16_t nb_desc,
 464                     unsigned int socket_id,
 465                     const struct rte_eth_txconf *tx_conf)
 466 {
 467         struct qede_dev *qdev = dev->data->dev_private;
 468         struct ecore_dev *edev = &qdev->edev;
 469         struct qede_tx_queue *txq;
 470
 471         PMD_INIT_FUNC_TRACE(edev);
 472
 473         if (!rte_is_power_of_2(nb_desc)) {
 474                 DP_ERR(edev, "Ring size %u is not power of 2\n",
 475                        nb_desc);
 476                 return -EINVAL;
 477         }
 478
 479         /* Free memory prior to re-allocation if needed... */
 480         if (dev->data->tx_queues[queue_idx] != NULL) {
 481                 qede_tx_queue_release(dev->data->tx_queues[queue_idx]);
 482                 dev->data->tx_queues[queue_idx] = NULL;
 483         }
 484
 485         if (ECORE_IS_CMT(edev)) {
 486                 txq = qede_alloc_tx_queue_mem(dev, queue_idx * 2, nb_desc,
 487                                               socket_id, tx_conf);
 488                 if (!txq)
 489                         return -ENOMEM;
 490
 491                 qdev->fp_array[queue_idx * 2].txq = txq;
 492                 txq = qede_alloc_tx_queue_mem(dev, (queue_idx * 2) + 1, nb_desc,
 493                                               socket_id, tx_conf);
 494                 if (!txq)
 495                         return -ENOMEM;
 496
 497                 qdev->fp_array[(queue_idx * 2) + 1].txq = txq;
 498                 dev->data->tx_queues[queue_idx] =
 499                                         &qdev->fp_array_cmt[queue_idx];
 500         } else {
 501                 txq = qede_alloc_tx_queue_mem(dev, queue_idx, nb_desc,
 502                                               socket_id, tx_conf);
 503                 if (!txq)
 504                         return -ENOMEM;
 505
 506                 dev->data->tx_queues[queue_idx] = txq;
 507                 qdev->fp_array[queue_idx].txq = txq;
 508         }
 509
 510         return 0;
 511 }
 512
 513 static void
 514 qede_tx_queue_reset(__rte_unused struct qede_dev *qdev,
 515                     struct qede_tx_queue *txq)
 516 {
 517         DP_INFO(&qdev->edev, "Reset TX queue %u\n", txq->queue_id);
 518         ecore_chain_reset(&txq->tx_pbl);
 519         txq->sw_tx_cons = 0;
 520         txq->sw_tx_prod = 0;
 521         *txq->hw_cons_ptr = 0;
 522 }
 523
 524 static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq)
 525 {
 526         uint16_t i;
 527
 528         if (txq->sw_tx_ring) {
 529                 for (i = 0; i < txq->nb_tx_desc; i++) {
 530                         if (txq->sw_tx_ring[i]) {
 531                                 rte_pktmbuf_free(txq->sw_tx_ring[i]);
 532                                 txq->sw_tx_ring[i] = NULL;
 533                         }
 534                 }
 535         }
 536 }
 537
 538 static void _qede_tx_queue_release(struct qede_dev *qdev,
 539                                    struct ecore_dev *edev,
 540                                    struct qede_tx_queue *txq)
 541 {
 542         qede_tx_queue_release_mbufs(txq);
 543         qdev->ops->common->chain_free(edev, &txq->tx_pbl);
 544         rte_free(txq->sw_tx_ring);
 545         rte_free(txq);
 546 }
 547
 548 void qede_tx_queue_release(void *tx_queue)
 549 {
 550         struct qede_tx_queue *txq = tx_queue;
 551         struct qede_fastpath_cmt *fp_cmt;
 552         struct qede_dev *qdev;
 553         struct ecore_dev *edev;
 554
 555         if (txq) {
 556                 qdev = txq->qdev;
 557                 edev = QEDE_INIT_EDEV(qdev);
 558                 PMD_INIT_FUNC_TRACE(edev);
 559
 560                 if (ECORE_IS_CMT(edev)) {
 561                         fp_cmt = tx_queue;
 562                         _qede_tx_queue_release(qdev, edev, fp_cmt->fp0->txq);
 563                         _qede_tx_queue_release(qdev, edev, fp_cmt->fp1->txq);
 564                 } else {
 565                         _qede_tx_queue_release(qdev, edev, txq);
 566                 }
 567         }
 568 }
 569
 570 /* This function allocates fast-path status block memory */
 571 static int
 572 qede_alloc_mem_sb(struct qede_dev *qdev, struct ecore_sb_info *sb_info,
 573                   uint16_t sb_id)
 574 {
 575         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 576         struct status_block *sb_virt;
 577         dma_addr_t sb_phys;
 578         int rc;
 579
 580         sb_virt = OSAL_DMA_ALLOC_COHERENT(edev, &sb_phys,
 581                                           sizeof(struct status_block));
 582         if (!sb_virt) {
 583                 DP_ERR(edev, "Status block allocation failed\n");
 584                 return -ENOMEM;
 585         }
 586         rc = qdev->ops->common->sb_init(edev, sb_info, sb_virt,
 587                                         sb_phys, sb_id);
 588         if (rc) {
 589                 DP_ERR(edev, "Status block initialization failed\n");
 590                 OSAL_DMA_FREE_COHERENT(edev, sb_virt, sb_phys,
 591                                        sizeof(struct status_block));
 592                 return rc;
 593         }
 594
 595         return 0;
 596 }
 597
 598 int qede_alloc_fp_resc(struct qede_dev *qdev)
 599 {
 600         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 601         struct qede_fastpath *fp;
 602         uint32_t num_sbs;
 603         uint16_t sb_idx;
 604         int i;
 605
 606         PMD_INIT_FUNC_TRACE(edev);
 607
 608         if (IS_VF(edev))
 609                 ecore_vf_get_num_sbs(ECORE_LEADING_HWFN(edev), &num_sbs);
 610         else
 611                 num_sbs = ecore_cxt_get_proto_cid_count
 612                           (ECORE_LEADING_HWFN(edev), PROTOCOLID_ETH, NULL);
 613
 614         if (num_sbs == 0) {
 615                 DP_ERR(edev, "No status blocks available\n");
 616                 return -EINVAL;
 617         }
 618
 619         qdev->fp_array = rte_calloc("fp", QEDE_RXTX_MAX(qdev),
 620                                 sizeof(*qdev->fp_array), RTE_CACHE_LINE_SIZE);
 621
 622         if (!qdev->fp_array) {
 623                 DP_ERR(edev, "fp array allocation failed\n");
 624                 return -ENOMEM;
 625         }
 626
 627         memset((void *)qdev->fp_array, 0, QEDE_RXTX_MAX(qdev) *
 628                         sizeof(*qdev->fp_array));
 629
 630         if (ECORE_IS_CMT(edev)) {
 631                 qdev->fp_array_cmt = rte_calloc("fp_cmt",
 632                                                 QEDE_RXTX_MAX(qdev) / 2,
 633                                                 sizeof(*qdev->fp_array_cmt),
 634                                                 RTE_CACHE_LINE_SIZE);
 635
 636                 if (!qdev->fp_array_cmt) {
 637                         DP_ERR(edev, "fp array for CMT allocation failed\n");
 638                         return -ENOMEM;
 639                 }
 640
 641                 memset((void *)qdev->fp_array_cmt, 0,
 642                        (QEDE_RXTX_MAX(qdev) / 2) * sizeof(*qdev->fp_array_cmt));
 643
 644                 /* Establish the mapping of fp_array with fp_array_cmt */
 645                 for (i = 0; i < QEDE_RXTX_MAX(qdev) / 2; i++) {
 646                         qdev->fp_array_cmt[i].qdev = qdev;
 647                         qdev->fp_array_cmt[i].fp0 = &qdev->fp_array[i * 2];
 648                         qdev->fp_array_cmt[i].fp1 = &qdev->fp_array[i * 2 + 1];
 649                 }
 650         }
 651
 652         for (sb_idx = 0; sb_idx < QEDE_RXTX_MAX(qdev); sb_idx++) {
 653                 fp = &qdev->fp_array[sb_idx];
 654                 fp->sb_info = rte_calloc("sb", 1, sizeof(struct ecore_sb_info),
 655                                 RTE_CACHE_LINE_SIZE);
 656                 if (!fp->sb_info) {
 657                         DP_ERR(edev, "FP sb_info allocation fails\n");
 658                         return -1;
 659                 }
 660                 if (qede_alloc_mem_sb(qdev, fp->sb_info, sb_idx)) {
 661                         DP_ERR(edev, "FP status block allocation fails\n");
 662                         return -1;
 663                 }
 664                 DP_INFO(edev, "sb_info idx 0x%x initialized\n",
 665                                 fp->sb_info->igu_sb_id);
 666         }
 667
 668         return 0;
 669 }
 670
 671 void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev)
 672 {
 673         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
 674         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 675         struct qede_fastpath *fp;
 676         uint16_t sb_idx;
 677         uint8_t i;
 678
 679         PMD_INIT_FUNC_TRACE(edev);
 680
 681         for (sb_idx = 0; sb_idx < QEDE_RXTX_MAX(qdev); sb_idx++) {
 682                 fp = &qdev->fp_array[sb_idx];
 683                 if (fp->sb_info) {
 684                         DP_INFO(edev, "Free sb_info index 0x%x\n",
 685                                         fp->sb_info->igu_sb_id);
 686                         OSAL_DMA_FREE_COHERENT(edev, fp->sb_info->sb_virt,
 687                                 fp->sb_info->sb_phys,
 688                                 sizeof(struct status_block));
 689                         rte_free(fp->sb_info);
 690                         fp->sb_info = NULL;
 691                 }
 692         }
 693
 694         /* Free packet buffers and ring memories */
 695         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 696                 if (eth_dev->data->rx_queues[i]) {
 697                         qede_rx_queue_release(eth_dev->data->rx_queues[i]);
 698                         eth_dev->data->rx_queues[i] = NULL;
 699                 }
 700         }
 701
 702         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
 703                 if (eth_dev->data->tx_queues[i]) {
 704                         qede_tx_queue_release(eth_dev->data->tx_queues[i]);
 705                         eth_dev->data->tx_queues[i] = NULL;
 706                 }
 707         }
 708
 709         if (qdev->fp_array)
 710                 rte_free(qdev->fp_array);
 711         qdev->fp_array = NULL;
 712
 713         if (qdev->fp_array_cmt)
 714                 rte_free(qdev->fp_array_cmt);
 715         qdev->fp_array_cmt = NULL;
 716 }
 717
 718 static inline void
 719 qede_update_rx_prod(__rte_unused struct qede_dev *edev,
 720                     struct qede_rx_queue *rxq)
 721 {
 722         uint16_t bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring);
 723         uint16_t cqe_prod = ecore_chain_get_prod_idx(&rxq->rx_comp_ring);
 724         struct eth_rx_prod_data rx_prods;
 725
 726         /* Update producers */
 727         memset(&rx_prods, 0, sizeof(rx_prods));
 728         rx_prods.bd_prod = rte_cpu_to_le_16(bd_prod);
 729         rx_prods.cqe_prod = rte_cpu_to_le_16(cqe_prod);
 730
 731         /* Make sure that the BD and SGE data is updated before updating the
 732          * producers since FW might read the BD/SGE right after the producer
 733          * is updated.
 734          */
 735         rte_wmb();
 736
 737         internal_ram_wr(rxq->hw_rxq_prod_addr, sizeof(rx_prods),
 738                         (uint32_t *)&rx_prods);
 739
 740         /* mmiowb is needed to synchronize doorbell writes from more than one
 741          * processor. It guarantees that the write arrives to the device before
 742          * the napi lock is released and another qede_poll is called (possibly
 743          * on another CPU). Without this barrier, the next doorbell can bypass
 744          * this doorbell. This is applicable to IA64/Altix systems.
 745          */
 746         rte_wmb();
 747
 748         PMD_RX_LOG(DEBUG, rxq, "bd_prod %u  cqe_prod %u", bd_prod, cqe_prod);
 749 }
 750
 751 /* Starts a given RX queue in HW */
 752 static int
 753 qede_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 754 {
 755         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
 756         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 757         struct ecore_queue_start_common_params params;
 758         struct ecore_rxq_start_ret_params ret_params;
 759         struct qede_rx_queue *rxq;
 760         struct qede_fastpath *fp;
 761         struct ecore_hwfn *p_hwfn;
 762         dma_addr_t p_phys_table;
 763         uint16_t page_cnt;
 764         uint16_t j;
 765         int hwfn_index;
 766         int rc;
 767
 768         if (rx_queue_id < qdev->num_rx_queues) {
 769                 fp = &qdev->fp_array[rx_queue_id];
 770                 rxq = fp->rxq;
 771                 /* Allocate buffers for the Rx ring */
 772                 for (j = 0; j < rxq->nb_rx_desc; j++) {
 773                         rc = qede_alloc_rx_buffer(rxq);
 774                         if (rc) {
 775                                 DP_ERR(edev, "RX buffer allocation failed"
 776                                                 " for rxq = %u\n", rx_queue_id);
 777                                 return -ENOMEM;
 778                         }
 779                 }
 780                 /* disable interrupts */
 781                 ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
 782                 /* Prepare ramrod */
 783                 memset(&params, 0, sizeof(params));
 784                 params.queue_id = rx_queue_id / edev->num_hwfns;
 785                 params.vport_id = 0;
 786                 params.stats_id = params.vport_id;
 787                 params.p_sb = fp->sb_info;
 788                 DP_INFO(edev, "rxq %u igu_sb_id 0x%x\n",
 789                                 fp->rxq->queue_id, fp->sb_info->igu_sb_id);
 790                 params.sb_idx = RX_PI;
 791                 hwfn_index = rx_queue_id % edev->num_hwfns;
 792                 p_hwfn = &edev->hwfns[hwfn_index];
 793                 p_phys_table = ecore_chain_get_pbl_phys(&fp->rxq->rx_comp_ring);
 794                 page_cnt = ecore_chain_get_page_cnt(&fp->rxq->rx_comp_ring);
 795                 memset(&ret_params, 0, sizeof(ret_params));
 796                 rc = ecore_eth_rx_queue_start(p_hwfn,
 797                                 p_hwfn->hw_info.opaque_fid,
 798                                 &params, fp->rxq->rx_buf_size,
 799                                 fp->rxq->rx_bd_ring.p_phys_addr,
 800                                 p_phys_table, page_cnt,
 801                                 &ret_params);
 802                 if (rc) {
 803                         DP_ERR(edev, "RX queue %u could not be started, rc = %d\n",
 804                                         rx_queue_id, rc);
 805                         return -1;
 806                 }
 807                 /* Update with the returned parameters */
 808                 fp->rxq->hw_rxq_prod_addr = ret_params.p_prod;
 809                 fp->rxq->handle = ret_params.p_handle;
 810
 811                 fp->rxq->hw_cons_ptr = &fp->sb_info->sb_pi_array[RX_PI];
 812                 qede_update_rx_prod(qdev, fp->rxq);
 813                 eth_dev->data->rx_queue_state[rx_queue_id] =
 814                         RTE_ETH_QUEUE_STATE_STARTED;
 815                 DP_INFO(edev, "RX queue %u started\n", rx_queue_id);
 816         } else {
 817                 DP_ERR(edev, "RX queue %u is not in range\n", rx_queue_id);
 818                 rc = -EINVAL;
 819         }
 820
 821         return rc;
 822 }
 823
 824 static int
 825 qede_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 826 {
 827         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
 828         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 829         struct ecore_queue_start_common_params params;
 830         struct ecore_txq_start_ret_params ret_params;
 831         struct ecore_hwfn *p_hwfn;
 832         dma_addr_t p_phys_table;
 833         struct qede_tx_queue *txq;
 834         struct qede_fastpath *fp;
 835         uint16_t page_cnt;
 836         int hwfn_index;
 837         int rc;
 838
 839         if (tx_queue_id < qdev->num_tx_queues) {
 840                 fp = &qdev->fp_array[tx_queue_id];
 841                 txq = fp->txq;
 842                 memset(&params, 0, sizeof(params));
 843                 params.queue_id = tx_queue_id / edev->num_hwfns;
 844                 params.vport_id = 0;
 845                 params.stats_id = params.vport_id;
 846                 params.p_sb = fp->sb_info;
 847                 DP_INFO(edev, "txq %u igu_sb_id 0x%x\n",
 848                                 fp->txq->queue_id, fp->sb_info->igu_sb_id);
 849                 params.sb_idx = TX_PI(0); /* tc = 0 */
 850                 p_phys_table = ecore_chain_get_pbl_phys(&txq->tx_pbl);
 851                 page_cnt = ecore_chain_get_page_cnt(&txq->tx_pbl);
 852                 hwfn_index = tx_queue_id % edev->num_hwfns;
 853                 p_hwfn = &edev->hwfns[hwfn_index];
 854                 if (qdev->dev_info.is_legacy)
 855                         fp->txq->is_legacy = true;
 856                 rc = ecore_eth_tx_queue_start(p_hwfn,
 857                                 p_hwfn->hw_info.opaque_fid,
 858                                 &params, 0 /* tc */,
 859                                 p_phys_table, page_cnt,
 860                                 &ret_params);
 861                 if (rc != ECORE_SUCCESS) {
 862                         DP_ERR(edev, "TX queue %u couldn't be started, rc=%d\n",
 863                                         tx_queue_id, rc);
 864                         return -1;
 865                 }
 866                 txq->doorbell_addr = ret_params.p_doorbell;
 867                 txq->handle = ret_params.p_handle;
 868
 869                 txq->hw_cons_ptr = &fp->sb_info->sb_pi_array[TX_PI(0)];
 870                 SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_DEST,
 871                                 DB_DEST_XCM);
 872                 SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD,
 873                                 DB_AGG_CMD_SET);
 874                 SET_FIELD(txq->tx_db.data.params,
 875                                 ETH_DB_DATA_AGG_VAL_SEL,
 876                                 DQ_XCM_ETH_TX_BD_PROD_CMD);
 877                 txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
 878                 eth_dev->data->tx_queue_state[tx_queue_id] =
 879                         RTE_ETH_QUEUE_STATE_STARTED;
 880                 DP_INFO(edev, "TX queue %u started\n", tx_queue_id);
 881         } else {
 882                 DP_ERR(edev, "TX queue %u is not in range\n", tx_queue_id);
 883                 rc = -EINVAL;
 884         }
 885
 886         return rc;
 887 }
 888
 889 static inline void
 890 qede_process_tx_compl(__rte_unused struct ecore_dev *edev,
 891                       struct qede_tx_queue *txq)
 892 {
 893         uint16_t hw_bd_cons;
 894         uint16_t sw_tx_cons;
 895         uint16_t remaining;
 896         uint16_t mask;
 897         struct rte_mbuf *mbuf;
 898         uint16_t nb_segs;
 899         uint16_t idx;
 900         uint16_t first_idx;
 901
 902         rte_compiler_barrier();
 903         rte_prefetch0(txq->hw_cons_ptr);
 904         sw_tx_cons = ecore_chain_get_cons_idx(&txq->tx_pbl);
 905         hw_bd_cons = rte_le_to_cpu_16(*txq->hw_cons_ptr);
 906 #ifdef RTE_LIBRTE_QEDE_DEBUG_TX
 907         PMD_TX_LOG(DEBUG, txq, "Tx Completions = %u\n",
 908                    abs(hw_bd_cons - sw_tx_cons));
 909 #endif
 910
 911         mask = NUM_TX_BDS(txq);
 912         idx = txq->sw_tx_cons & mask;
 913
 914         remaining = hw_bd_cons - sw_tx_cons;
 915         txq->nb_tx_avail += remaining;
 916         first_idx = idx;
 917
 918         while (remaining) {
 919                 mbuf = txq->sw_tx_ring[idx];
 920                 RTE_ASSERT(mbuf);
 921                 nb_segs = mbuf->nb_segs;
 922                 remaining -= nb_segs;
 923
 924                 /* Prefetch the next mbuf. Note that at least the last 4 mbufs
 925                  * that are prefetched will not be used in the current call.
 926                  */
 927                 rte_mbuf_prefetch_part1(txq->sw_tx_ring[(idx + 4) & mask]);
 928                 rte_mbuf_prefetch_part2(txq->sw_tx_ring[(idx + 4) & mask]);
 929
 930                 PMD_TX_LOG(DEBUG, txq, "nb_segs to free %u\n", nb_segs);
 931
 932                 while (nb_segs) {
 933                         ecore_chain_consume(&txq->tx_pbl);
 934                         nb_segs--;
 935                 }
 936
 937                 idx = (idx + 1) & mask;
 938                 PMD_TX_LOG(DEBUG, txq, "Freed tx packet\n");
 939         }
 940         txq->sw_tx_cons = idx;
 941
 942         if (first_idx > idx) {
 943                 rte_pktmbuf_free_bulk(&txq->sw_tx_ring[first_idx],
 944                                                           mask - first_idx + 1);
 945                 rte_pktmbuf_free_bulk(&txq->sw_tx_ring[0], idx);
 946         } else {
 947                 rte_pktmbuf_free_bulk(&txq->sw_tx_ring[first_idx],
 948                                                           idx - first_idx);
 949         }
 950 }
 951
 952 static int qede_drain_txq(struct qede_dev *qdev,
 953                           struct qede_tx_queue *txq, bool allow_drain)
 954 {
 955         struct ecore_dev *edev = &qdev->edev;
 956         int rc, cnt = 1000;
 957
 958         while (txq->sw_tx_cons != txq->sw_tx_prod) {
 959                 qede_process_tx_compl(edev, txq);
 960                 if (!cnt) {
 961                         if (allow_drain) {
 962                                 DP_ERR(edev, "Tx queue[%u] is stuck,"
 963                                           "requesting MCP to drain\n",
 964                                           txq->queue_id);
 965                                 rc = qdev->ops->common->drain(edev);
 966                                 if (rc)
 967                                         return rc;
 968                                 return qede_drain_txq(qdev, txq, false);
 969                         }
 970                         DP_ERR(edev, "Timeout waiting for tx queue[%d]:"
 971                                   "PROD=%d, CONS=%d\n",
 972                                   txq->queue_id, txq->sw_tx_prod,
 973                                   txq->sw_tx_cons);
 974                         return -1;
 975                 }
 976                 cnt--;
 977                 DELAY(1000);
 978                 rte_compiler_barrier();
 979         }
 980
 981         /* FW finished processing, wait for HW to transmit all tx packets */
 982         DELAY(2000);
 983
 984         return 0;
 985 }
 986
 987 /* Stops a given TX queue in the HW */
 988 static int qede_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 989 {
 990         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
 991         struct ecore_dev *edev = QEDE_INIT_EDEV(qdev);
 992         struct ecore_hwfn *p_hwfn;
 993         struct qede_tx_queue *txq;
 994         int hwfn_index;
 995         int rc;
 996
 997         if (tx_queue_id < qdev->num_tx_queues) {
 998                 txq = qdev->fp_array[tx_queue_id].txq;
 999                 /* Drain txq */
1000                 if (qede_drain_txq(qdev, txq, true))
1001                         return -1; /* For the lack of retcodes */
1002                 /* Stop txq */
1003                 hwfn_index = tx_queue_id % edev->num_hwfns;
1004                 p_hwfn = &edev->hwfns[hwfn_index];
1005                 rc = ecore_eth_tx_queue_stop(p_hwfn, txq->handle);
1006                 if (rc != ECORE_SUCCESS) {
1007                         DP_ERR(edev, "TX queue %u stop fails\n", tx_queue_id);
1008                         return -1;
1009                 }
1010                 qede_tx_queue_release_mbufs(txq);
1011                 qede_tx_queue_reset(qdev, txq);
1012                 eth_dev->data->tx_queue_state[tx_queue_id] =
1013                         RTE_ETH_QUEUE_STATE_STOPPED;
1014                 DP_INFO(edev, "TX queue %u stopped\n", tx_queue_id);
1015         } else {
1016                 DP_ERR(edev, "TX queue %u is not in range\n", tx_queue_id);
1017                 rc = -EINVAL;
1018         }
1019
1020         return rc;
1021 }
1022
1023 int qede_start_queues(struct rte_eth_dev *eth_dev)
1024 {
1025         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
1026         uint8_t id;
1027         int rc = -1;
1028
1029         for (id = 0; id < qdev->num_rx_queues; id++) {
1030                 rc = qede_rx_queue_start(eth_dev, id);
1031                 if (rc != ECORE_SUCCESS)
1032                         return -1;
1033         }
1034
1035         for (id = 0; id < qdev->num_tx_queues; id++) {
1036                 rc = qede_tx_queue_start(eth_dev, id);
1037                 if (rc != ECORE_SUCCESS)
1038                         return -1;
1039         }
1040
1041         return rc;
1042 }
1043
1044 void qede_stop_queues(struct rte_eth_dev *eth_dev)
1045 {
1046         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
1047         uint8_t id;
1048
1049         /* Stopping RX/TX queues */
1050         for (id = 0; id < qdev->num_tx_queues; id++)
1051                 qede_tx_queue_stop(eth_dev, id);
1052
1053         for (id = 0; id < qdev->num_rx_queues; id++)
1054                 qede_rx_queue_stop(eth_dev, id);
1055 }
1056
1057 static inline bool qede_tunn_exist(uint16_t flag)
1058 {
1059         return !!((PARSING_AND_ERR_FLAGS_TUNNELEXIST_MASK <<
1060                     PARSING_AND_ERR_FLAGS_TUNNELEXIST_SHIFT) & flag);
1061 }
1062
1063 static inline uint8_t qede_check_tunn_csum_l3(uint16_t flag)
1064 {
1065         return !!((PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_MASK <<
1066                 PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_SHIFT) & flag);
1067 }
1068
1069 /*
1070  * qede_check_tunn_csum_l4:
1071  * Returns:
1072  * 1 : If L4 csum is enabled AND if the validation has failed.
1073  * 0 : Otherwise
1074  */
1075 static inline uint8_t qede_check_tunn_csum_l4(uint16_t flag)
1076 {
1077         if ((PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_MASK <<
1078              PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_SHIFT) & flag)
1079                 return !!((PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_MASK <<
1080                         PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT) & flag);
1081
1082         return 0;
1083 }
1084
1085 static inline uint8_t qede_check_notunn_csum_l4(uint16_t flag)
1086 {
1087         if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK <<
1088              PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag)
1089                 return !!((PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK <<
1090                            PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT) & flag);
1091
1092         return 0;
1093 }
1094
1095 /* Returns outer L2, L3 and L4 packet_type for tunneled packets */
1096 static inline uint32_t qede_rx_cqe_to_pkt_type_outer(struct rte_mbuf *m)
1097 {
1098         uint32_t packet_type = RTE_PTYPE_UNKNOWN;
1099         struct rte_ether_hdr *eth_hdr;
1100         struct rte_ipv4_hdr *ipv4_hdr;
1101         struct rte_ipv6_hdr *ipv6_hdr;
1102         struct rte_vlan_hdr *vlan_hdr;
1103         uint16_t ethertype;
1104         bool vlan_tagged = 0;
1105         uint16_t len;
1106
1107         eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
1108         len = sizeof(struct rte_ether_hdr);
1109         ethertype = rte_cpu_to_be_16(eth_hdr->ether_type);
1110
1111          /* Note: Valid only if VLAN stripping is disabled */
1112         if (ethertype == RTE_ETHER_TYPE_VLAN) {
1113                 vlan_tagged = 1;
1114                 vlan_hdr = (struct rte_vlan_hdr *)(eth_hdr + 1);
1115                 len += sizeof(struct rte_vlan_hdr);
1116                 ethertype = rte_cpu_to_be_16(vlan_hdr->eth_proto);
1117         }
1118
1119         if (ethertype == RTE_ETHER_TYPE_IPV4) {
1120                 packet_type |= RTE_PTYPE_L3_IPV4;
1121                 ipv4_hdr = rte_pktmbuf_mtod_offset(m,
1122                                         struct rte_ipv4_hdr *, len);
1123                 if (ipv4_hdr->next_proto_id == IPPROTO_TCP)
1124                         packet_type |= RTE_PTYPE_L4_TCP;
1125                 else if (ipv4_hdr->next_proto_id == IPPROTO_UDP)
1126                         packet_type |= RTE_PTYPE_L4_UDP;
1127         } else if (ethertype == RTE_ETHER_TYPE_IPV6) {
1128                 packet_type |= RTE_PTYPE_L3_IPV6;
1129                 ipv6_hdr = rte_pktmbuf_mtod_offset(m,
1130                                                 struct rte_ipv6_hdr *, len);
1131                 if (ipv6_hdr->proto == IPPROTO_TCP)
1132                         packet_type |= RTE_PTYPE_L4_TCP;
1133                 else if (ipv6_hdr->proto == IPPROTO_UDP)
1134                         packet_type |= RTE_PTYPE_L4_UDP;
1135         }
1136
1137         if (vlan_tagged)
1138                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
1139         else
1140                 packet_type |= RTE_PTYPE_L2_ETHER;
1141
1142         return packet_type;
1143 }
1144
1145 static inline uint32_t qede_rx_cqe_to_pkt_type_inner(uint16_t flags)
1146 {
1147         uint16_t val;
1148
1149         /* Lookup table */
1150         static const uint32_t
1151         ptype_lkup_tbl[QEDE_PKT_TYPE_MAX] __rte_cache_aligned = {
1152                 [QEDE_PKT_TYPE_IPV4] = RTE_PTYPE_INNER_L3_IPV4          |
1153                                        RTE_PTYPE_INNER_L2_ETHER,
1154                 [QEDE_PKT_TYPE_IPV6] = RTE_PTYPE_INNER_L3_IPV6          |
1155                                        RTE_PTYPE_INNER_L2_ETHER,
1156                 [QEDE_PKT_TYPE_IPV4_TCP] = RTE_PTYPE_INNER_L3_IPV4      |
1157                                            RTE_PTYPE_INNER_L4_TCP       |
1158                                            RTE_PTYPE_INNER_L2_ETHER,
1159                 [QEDE_PKT_TYPE_IPV6_TCP] = RTE_PTYPE_INNER_L3_IPV6      |
1160                                            RTE_PTYPE_INNER_L4_TCP       |
1161                                            RTE_PTYPE_INNER_L2_ETHER,
1162                 [QEDE_PKT_TYPE_IPV4_UDP] = RTE_PTYPE_INNER_L3_IPV4      |
1163                                            RTE_PTYPE_INNER_L4_UDP       |
1164                                            RTE_PTYPE_INNER_L2_ETHER,
1165                 [QEDE_PKT_TYPE_IPV6_UDP] = RTE_PTYPE_INNER_L3_IPV6      |
1166                                            RTE_PTYPE_INNER_L4_UDP       |
1167                                            RTE_PTYPE_INNER_L2_ETHER,
1168                 /* Frags with no VLAN */
1169                 [QEDE_PKT_TYPE_IPV4_FRAG] = RTE_PTYPE_INNER_L3_IPV4     |
1170                                             RTE_PTYPE_INNER_L4_FRAG     |
1171                                             RTE_PTYPE_INNER_L2_ETHER,
1172                 [QEDE_PKT_TYPE_IPV6_FRAG] = RTE_PTYPE_INNER_L3_IPV6     |
1173                                             RTE_PTYPE_INNER_L4_FRAG     |
1174                                             RTE_PTYPE_INNER_L2_ETHER,
1175                 /* VLANs */
1176                 [QEDE_PKT_TYPE_IPV4_VLAN] = RTE_PTYPE_INNER_L3_IPV4     |
1177                                             RTE_PTYPE_INNER_L2_ETHER_VLAN,
1178                 [QEDE_PKT_TYPE_IPV6_VLAN] = RTE_PTYPE_INNER_L3_IPV6     |
1179                                             RTE_PTYPE_INNER_L2_ETHER_VLAN,
1180                 [QEDE_PKT_TYPE_IPV4_TCP_VLAN] = RTE_PTYPE_INNER_L3_IPV4 |
1181                                                 RTE_PTYPE_INNER_L4_TCP  |
1182                                                 RTE_PTYPE_INNER_L2_ETHER_VLAN,
1183                 [QEDE_PKT_TYPE_IPV6_TCP_VLAN] = RTE_PTYPE_INNER_L3_IPV6 |
1184                                                 RTE_PTYPE_INNER_L4_TCP  |
1185                                                 RTE_PTYPE_INNER_L2_ETHER_VLAN,
1186                 [QEDE_PKT_TYPE_IPV4_UDP_VLAN] = RTE_PTYPE_INNER_L3_IPV4 |
1187                                                 RTE_PTYPE_INNER_L4_UDP  |
1188                                                 RTE_PTYPE_INNER_L2_ETHER_VLAN,
1189                 [QEDE_PKT_TYPE_IPV6_UDP_VLAN] = RTE_PTYPE_INNER_L3_IPV6 |
1190                                                 RTE_PTYPE_INNER_L4_UDP  |
1191                                                 RTE_PTYPE_INNER_L2_ETHER_VLAN,
1192                 /* Frags with VLAN */
1193                 [QEDE_PKT_TYPE_IPV4_VLAN_FRAG] = RTE_PTYPE_INNER_L3_IPV4 |
1194                                                  RTE_PTYPE_INNER_L4_FRAG |
1195                                                  RTE_PTYPE_INNER_L2_ETHER_VLAN,
1196                 [QEDE_PKT_TYPE_IPV6_VLAN_FRAG] = RTE_PTYPE_INNER_L3_IPV6 |
1197                                                  RTE_PTYPE_INNER_L4_FRAG |
1198                                                  RTE_PTYPE_INNER_L2_ETHER_VLAN,
1199         };
1200
1201         /* Bits (0..3) provides L3/L4 protocol type */
1202         /* Bits (4,5) provides frag and VLAN info */
1203         val = ((PARSING_AND_ERR_FLAGS_L3TYPE_MASK <<
1204                PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT) |
1205                (PARSING_AND_ERR_FLAGS_L4PROTOCOL_MASK <<
1206                 PARSING_AND_ERR_FLAGS_L4PROTOCOL_SHIFT) |
1207                (PARSING_AND_ERR_FLAGS_IPV4FRAG_MASK <<
1208                 PARSING_AND_ERR_FLAGS_IPV4FRAG_SHIFT) |
1209                 (PARSING_AND_ERR_FLAGS_TAG8021QEXIST_MASK <<
1210                  PARSING_AND_ERR_FLAGS_TAG8021QEXIST_SHIFT)) & flags;
1211
1212         if (val < QEDE_PKT_TYPE_MAX)
1213                 return ptype_lkup_tbl[val];
1214
1215         return RTE_PTYPE_UNKNOWN;
1216 }
1217
1218 static inline uint32_t qede_rx_cqe_to_pkt_type(uint16_t flags)
1219 {
1220         uint16_t val;
1221
1222         /* Lookup table */
1223         static const uint32_t
1224         ptype_lkup_tbl[QEDE_PKT_TYPE_MAX] __rte_cache_aligned = {
1225                 [QEDE_PKT_TYPE_IPV4] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER,
1226                 [QEDE_PKT_TYPE_IPV6] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L2_ETHER,
1227                 [QEDE_PKT_TYPE_IPV4_TCP] = RTE_PTYPE_L3_IPV4    |
1228                                            RTE_PTYPE_L4_TCP     |
1229                                            RTE_PTYPE_L2_ETHER,
1230                 [QEDE_PKT_TYPE_IPV6_TCP] = RTE_PTYPE_L3_IPV6    |
1231                                            RTE_PTYPE_L4_TCP     |
1232                                            RTE_PTYPE_L2_ETHER,
1233                 [QEDE_PKT_TYPE_IPV4_UDP] = RTE_PTYPE_L3_IPV4    |
1234                                            RTE_PTYPE_L4_UDP     |
1235                                            RTE_PTYPE_L2_ETHER,
1236                 [QEDE_PKT_TYPE_IPV6_UDP] = RTE_PTYPE_L3_IPV6    |
1237                                            RTE_PTYPE_L4_UDP     |
1238                                            RTE_PTYPE_L2_ETHER,
1239                 /* Frags with no VLAN */
1240                 [QEDE_PKT_TYPE_IPV4_FRAG] = RTE_PTYPE_L3_IPV4   |
1241                                             RTE_PTYPE_L4_FRAG   |
1242                                             RTE_PTYPE_L2_ETHER,
1243                 [QEDE_PKT_TYPE_IPV6_FRAG] = RTE_PTYPE_L3_IPV6   |
1244                                             RTE_PTYPE_L4_FRAG   |
1245                                             RTE_PTYPE_L2_ETHER,
1246                 /* VLANs */
1247                 [QEDE_PKT_TYPE_IPV4_VLAN] = RTE_PTYPE_L3_IPV4           |
1248                                             RTE_PTYPE_L2_ETHER_VLAN,
1249                 [QEDE_PKT_TYPE_IPV6_VLAN] = RTE_PTYPE_L3_IPV6           |
1250                                             RTE_PTYPE_L2_ETHER_VLAN,
1251                 [QEDE_PKT_TYPE_IPV4_TCP_VLAN] = RTE_PTYPE_L3_IPV4       |
1252                                                 RTE_PTYPE_L4_TCP        |
1253                                                 RTE_PTYPE_L2_ETHER_VLAN,
1254                 [QEDE_PKT_TYPE_IPV6_TCP_VLAN] = RTE_PTYPE_L3_IPV6       |
1255                                                 RTE_PTYPE_L4_TCP        |
1256                                                 RTE_PTYPE_L2_ETHER_VLAN,
1257                 [QEDE_PKT_TYPE_IPV4_UDP_VLAN] = RTE_PTYPE_L3_IPV4       |
1258                                                 RTE_PTYPE_L4_UDP        |
1259                                                 RTE_PTYPE_L2_ETHER_VLAN,
1260                 [QEDE_PKT_TYPE_IPV6_UDP_VLAN] = RTE_PTYPE_L3_IPV6       |
1261                                                 RTE_PTYPE_L4_UDP        |
1262                                                 RTE_PTYPE_L2_ETHER_VLAN,
1263                 /* Frags with VLAN */
1264                 [QEDE_PKT_TYPE_IPV4_VLAN_FRAG] = RTE_PTYPE_L3_IPV4      |
1265                                                  RTE_PTYPE_L4_FRAG      |
1266                                                  RTE_PTYPE_L2_ETHER_VLAN,
1267                 [QEDE_PKT_TYPE_IPV6_VLAN_FRAG] = RTE_PTYPE_L3_IPV6      |
1268                                                  RTE_PTYPE_L4_FRAG      |
1269                                                  RTE_PTYPE_L2_ETHER_VLAN,
1270         };
1271
1272         /* Bits (0..3) provides L3/L4 protocol type */
1273         /* Bits (4,5) provides frag and VLAN info */
1274         val = ((PARSING_AND_ERR_FLAGS_L3TYPE_MASK <<
1275                PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT) |
1276                (PARSING_AND_ERR_FLAGS_L4PROTOCOL_MASK <<
1277                 PARSING_AND_ERR_FLAGS_L4PROTOCOL_SHIFT) |
1278                (PARSING_AND_ERR_FLAGS_IPV4FRAG_MASK <<
1279                 PARSING_AND_ERR_FLAGS_IPV4FRAG_SHIFT) |
1280                 (PARSING_AND_ERR_FLAGS_TAG8021QEXIST_MASK <<
1281                  PARSING_AND_ERR_FLAGS_TAG8021QEXIST_SHIFT)) & flags;
1282
1283         if (val < QEDE_PKT_TYPE_MAX)
1284                 return ptype_lkup_tbl[val];
1285
1286         return RTE_PTYPE_UNKNOWN;
1287 }
1288
1289 static inline uint8_t
1290 qede_check_notunn_csum_l3(struct rte_mbuf *m, uint16_t flag)
1291 {
1292         struct rte_ipv4_hdr *ip;
1293         uint16_t pkt_csum;
1294         uint16_t calc_csum;
1295         uint16_t val;
1296
1297         val = ((PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK <<
1298                 PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT) & flag);
1299
1300         if (unlikely(val)) {
1301                 m->packet_type = qede_rx_cqe_to_pkt_type(flag);
1302                 if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) {
1303                         ip = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *,
1304                                            sizeof(struct rte_ether_hdr));
1305                         pkt_csum = ip->hdr_checksum;
1306                         ip->hdr_checksum = 0;
1307                         calc_csum = rte_ipv4_cksum(ip);
1308                         ip->hdr_checksum = pkt_csum;
1309                         return (calc_csum != pkt_csum);
1310                 } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) {
1311                         return 1;
1312                 }
1313         }
1314         return 0;
1315 }
1316
1317 static inline void qede_rx_bd_ring_consume(struct qede_rx_queue *rxq)
1318 {
1319         ecore_chain_consume(&rxq->rx_bd_ring);
1320         rxq->sw_rx_cons++;
1321 }
1322
1323 static inline void
1324 qede_reuse_page(__rte_unused struct qede_dev *qdev,
1325                 struct qede_rx_queue *rxq, struct rte_mbuf *curr_cons)
1326 {
1327         struct eth_rx_bd *rx_bd_prod = ecore_chain_produce(&rxq->rx_bd_ring);
1328         uint16_t idx = rxq->sw_rx_prod & NUM_RX_BDS(rxq);
1329         dma_addr_t new_mapping;
1330
1331         rxq->sw_rx_ring[idx] = curr_cons;
1332
1333         new_mapping = rte_mbuf_data_iova_default(curr_cons);
1334
1335         rx_bd_prod->addr.hi = rte_cpu_to_le_32(U64_HI(new_mapping));
1336         rx_bd_prod->addr.lo = rte_cpu_to_le_32(U64_LO(new_mapping));
1337
1338         rxq->sw_rx_prod++;
1339 }
1340
1341 static inline void
1342 qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq,
1343                         struct qede_dev *qdev, uint8_t count)
1344 {
1345         struct rte_mbuf *curr_cons;
1346
1347         for (; count > 0; count--) {
1348                 curr_cons = rxq->sw_rx_ring[rxq->sw_rx_cons & NUM_RX_BDS(rxq)];
1349                 qede_reuse_page(qdev, rxq, curr_cons);
1350                 qede_rx_bd_ring_consume(rxq);
1351         }
1352 }
1353
1354 static inline void
1355 qede_rx_process_tpa_cmn_cont_end_cqe(__rte_unused struct qede_dev *qdev,
1356                                      struct qede_rx_queue *rxq,
1357                                      uint8_t agg_index, uint16_t len)
1358 {
1359         struct qede_agg_info *tpa_info;
1360         struct rte_mbuf *curr_frag; /* Pointer to currently filled TPA seg */
1361         uint16_t cons_idx;
1362
1363         /* Under certain conditions it is possible that FW may not consume
1364          * additional or new BD. So decision to consume the BD must be made
1365          * based on len_list[0].
1366          */
1367         if (rte_le_to_cpu_16(len)) {
1368                 tpa_info = &rxq->tpa_info[agg_index];
1369                 cons_idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
1370                 curr_frag = rxq->sw_rx_ring[cons_idx];
1371                 assert(curr_frag);
1372                 curr_frag->nb_segs = 1;
1373                 curr_frag->pkt_len = rte_le_to_cpu_16(len);
1374                 curr_frag->data_len = curr_frag->pkt_len;
1375                 tpa_info->tpa_tail->next = curr_frag;
1376                 tpa_info->tpa_tail = curr_frag;
1377                 qede_rx_bd_ring_consume(rxq);
1378                 if (unlikely(qede_alloc_rx_buffer(rxq) != 0)) {
1379                         PMD_RX_LOG(ERR, rxq, "mbuf allocation fails\n");
1380                         rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;
1381                         rxq->rx_alloc_errors++;
1382                 }
1383         }
1384 }
1385
1386 static inline void
1387 qede_rx_process_tpa_cont_cqe(struct qede_dev *qdev,
1388                              struct qede_rx_queue *rxq,
1389                              struct eth_fast_path_rx_tpa_cont_cqe *cqe)
1390 {
1391         PMD_RX_LOG(INFO, rxq, "TPA cont[%d] - len [%d]\n",
1392                    cqe->tpa_agg_index, rte_le_to_cpu_16(cqe->len_list[0]));
1393         /* only len_list[0] will have value */
1394         qede_rx_process_tpa_cmn_cont_end_cqe(qdev, rxq, cqe->tpa_agg_index,
1395                                              cqe->len_list[0]);
1396 }
1397
1398 static inline void
1399 qede_rx_process_tpa_end_cqe(struct qede_dev *qdev,
1400                             struct qede_rx_queue *rxq,
1401                             struct eth_fast_path_rx_tpa_end_cqe *cqe)
1402 {
1403         struct rte_mbuf *rx_mb; /* Pointer to head of the chained agg */
1404
1405         qede_rx_process_tpa_cmn_cont_end_cqe(qdev, rxq, cqe->tpa_agg_index,
1406                                              cqe->len_list[0]);
1407         /* Update total length and frags based on end TPA */
1408         rx_mb = rxq->tpa_info[cqe->tpa_agg_index].tpa_head;
1409         /* TODO:  Add Sanity Checks */
1410         rx_mb->nb_segs = cqe->num_of_bds;
1411         rx_mb->pkt_len = cqe->total_packet_len;
1412
1413         PMD_RX_LOG(INFO, rxq, "TPA End[%d] reason %d cqe_len %d nb_segs %d"
1414                    " pkt_len %d\n", cqe->tpa_agg_index, cqe->end_reason,
1415                    rte_le_to_cpu_16(cqe->len_list[0]), rx_mb->nb_segs,
1416                    rx_mb->pkt_len);
1417 }
1418
1419 static inline uint32_t qede_rx_cqe_to_tunn_pkt_type(uint16_t flags)
1420 {
1421         uint32_t val;
1422
1423         /* Lookup table */
1424         static const uint32_t
1425         ptype_tunn_lkup_tbl[QEDE_PKT_TYPE_TUNN_MAX_TYPE] __rte_cache_aligned = {
1426                 [QEDE_PKT_TYPE_UNKNOWN] = RTE_PTYPE_UNKNOWN,
1427                 [QEDE_PKT_TYPE_TUNN_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE,
1428                 [QEDE_PKT_TYPE_TUNN_GRE] = RTE_PTYPE_TUNNEL_GRE,
1429                 [QEDE_PKT_TYPE_TUNN_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
1430                 [QEDE_PKT_TYPE_TUNN_L2_TENID_NOEXIST_GENEVE] =
1431                                 RTE_PTYPE_TUNNEL_GENEVE,
1432                 [QEDE_PKT_TYPE_TUNN_L2_TENID_NOEXIST_GRE] =
1433                                 RTE_PTYPE_TUNNEL_GRE,
1434                 [QEDE_PKT_TYPE_TUNN_L2_TENID_NOEXIST_VXLAN] =
1435                                 RTE_PTYPE_TUNNEL_VXLAN,
1436                 [QEDE_PKT_TYPE_TUNN_L2_TENID_EXIST_GENEVE] =
1437                                 RTE_PTYPE_TUNNEL_GENEVE,
1438                 [QEDE_PKT_TYPE_TUNN_L2_TENID_EXIST_GRE] =
1439                                 RTE_PTYPE_TUNNEL_GRE,
1440                 [QEDE_PKT_TYPE_TUNN_L2_TENID_EXIST_VXLAN] =
1441                                 RTE_PTYPE_TUNNEL_VXLAN,
1442                 [QEDE_PKT_TYPE_TUNN_IPV4_TENID_NOEXIST_GENEVE] =
1443                                 RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L3_IPV4,
1444                 [QEDE_PKT_TYPE_TUNN_IPV4_TENID_NOEXIST_GRE] =
1445                                 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_L3_IPV4,
1446                 [QEDE_PKT_TYPE_TUNN_IPV4_TENID_NOEXIST_VXLAN] =
1447                                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L3_IPV4,
1448                 [QEDE_PKT_TYPE_TUNN_IPV4_TENID_EXIST_GENEVE] =
1449                                 RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L3_IPV4,
1450                 [QEDE_PKT_TYPE_TUNN_IPV4_TENID_EXIST_GRE] =
1451                                 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_L3_IPV4,
1452                 [QEDE_PKT_TYPE_TUNN_IPV4_TENID_EXIST_VXLAN] =
1453                                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L3_IPV4,
1454                 [QEDE_PKT_TYPE_TUNN_IPV6_TENID_NOEXIST_GENEVE] =
1455                                 RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L3_IPV6,
1456                 [QEDE_PKT_TYPE_TUNN_IPV6_TENID_NOEXIST_GRE] =
1457                                 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_L3_IPV6,
1458                 [QEDE_PKT_TYPE_TUNN_IPV6_TENID_NOEXIST_VXLAN] =
1459                                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L3_IPV6,
1460                 [QEDE_PKT_TYPE_TUNN_IPV6_TENID_EXIST_GENEVE] =
1461                                 RTE_PTYPE_TUNNEL_GENEVE | RTE_PTYPE_L3_IPV6,
1462                 [QEDE_PKT_TYPE_TUNN_IPV6_TENID_EXIST_GRE] =
1463                                 RTE_PTYPE_TUNNEL_GRE | RTE_PTYPE_L3_IPV6,
1464                 [QEDE_PKT_TYPE_TUNN_IPV6_TENID_EXIST_VXLAN] =
1465                                 RTE_PTYPE_TUNNEL_VXLAN | RTE_PTYPE_L3_IPV6,
1466         };
1467
1468         /* Cover bits[4-0] to include tunn_type and next protocol */
1469         val = ((ETH_TUNNEL_PARSING_FLAGS_TYPE_MASK <<
1470                 ETH_TUNNEL_PARSING_FLAGS_TYPE_SHIFT) |
1471                 (ETH_TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_MASK <<
1472                 ETH_TUNNEL_PARSING_FLAGS_NEXT_PROTOCOL_SHIFT)) & flags;
1473
1474         if (val < QEDE_PKT_TYPE_TUNN_MAX_TYPE)
1475                 return ptype_tunn_lkup_tbl[val];
1476         else
1477                 return RTE_PTYPE_UNKNOWN;
1478 }
1479
1480 static inline int
1481 qede_process_sg_pkts(void *p_rxq,  struct rte_mbuf *rx_mb,
1482                      uint8_t num_segs, uint16_t pkt_len)
1483 {
1484         struct qede_rx_queue *rxq = p_rxq;
1485         struct qede_dev *qdev = rxq->qdev;
1486         register struct rte_mbuf *seg1 = NULL;
1487         register struct rte_mbuf *seg2 = NULL;
1488         uint16_t sw_rx_index;
1489         uint16_t cur_size;
1490
1491         seg1 = rx_mb;
1492         while (num_segs) {
1493                 cur_size = pkt_len > rxq->rx_buf_size ? rxq->rx_buf_size :
1494                                                         pkt_len;
1495                 if (unlikely(!cur_size)) {
1496                         PMD_RX_LOG(ERR, rxq, "Length is 0 while %u BDs"
1497                                    " left for mapping jumbo\n", num_segs);
1498                         qede_recycle_rx_bd_ring(rxq, qdev, num_segs);
1499                         return -EINVAL;
1500                 }
1501                 sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
1502                 seg2 = rxq->sw_rx_ring[sw_rx_index];
1503                 qede_rx_bd_ring_consume(rxq);
1504                 pkt_len -= cur_size;
1505                 seg2->data_len = cur_size;
1506                 seg1->next = seg2;
1507                 seg1 = seg1->next;
1508                 num_segs--;
1509                 rxq->rx_segs++;
1510         }
1511
1512         return 0;
1513 }
1514
1515 #ifdef RTE_LIBRTE_QEDE_DEBUG_RX
1516 static inline void
1517 print_rx_bd_info(struct rte_mbuf *m, struct qede_rx_queue *rxq,
1518                  uint8_t bitfield)
1519 {
1520         PMD_RX_LOG(INFO, rxq,
1521                 "len 0x%04x bf 0x%04x hash_val 0x%x"
1522                 " ol_flags 0x%04lx l2=%s l3=%s l4=%s tunn=%s"
1523                 " inner_l2=%s inner_l3=%s inner_l4=%s\n",
1524                 m->data_len, bitfield, m->hash.rss,
1525                 (unsigned long)m->ol_flags,
1526                 rte_get_ptype_l2_name(m->packet_type),
1527                 rte_get_ptype_l3_name(m->packet_type),
1528                 rte_get_ptype_l4_name(m->packet_type),
1529                 rte_get_ptype_tunnel_name(m->packet_type),
1530                 rte_get_ptype_inner_l2_name(m->packet_type),
1531                 rte_get_ptype_inner_l3_name(m->packet_type),
1532                 rte_get_ptype_inner_l4_name(m->packet_type));
1533 }
1534 #endif
1535
1536 uint16_t
1537 qede_recv_pkts_regular(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1538 {
1539         struct eth_fast_path_rx_reg_cqe *fp_cqe = NULL;
1540         register struct rte_mbuf *rx_mb = NULL;
1541         struct qede_rx_queue *rxq = p_rxq;
1542         struct qede_dev *qdev = rxq->qdev;
1543         struct ecore_dev *edev = &qdev->edev;
1544         union eth_rx_cqe *cqe;
1545         uint64_t ol_flags;
1546         enum eth_rx_cqe_type cqe_type;
1547         int rss_enable = qdev->rss_enable;
1548         int rx_alloc_count = 0;
1549         uint32_t packet_type;
1550         uint32_t rss_hash;
1551         uint16_t vlan_tci, port_id;
1552         uint16_t hw_comp_cons, sw_comp_cons, sw_rx_index, num_rx_bds;
1553         uint16_t rx_pkt = 0;
1554         uint16_t pkt_len = 0;
1555         uint16_t len; /* Length of first BD */
1556         uint16_t preload_idx;
1557         uint16_t parse_flag;
1558 #ifdef RTE_LIBRTE_QEDE_DEBUG_RX
1559         uint8_t bitfield_val;
1560 #endif
1561         uint8_t offset, flags, bd_num;
1562
1563
1564         /* Allocate buffers that we used in previous loop */
1565         if (rxq->rx_alloc_count) {
1566                 if (unlikely(qede_alloc_rx_bulk_mbufs(rxq,
1567                              rxq->rx_alloc_count))) {
1568                         struct rte_eth_dev *dev;
1569
1570                         PMD_RX_LOG(ERR, rxq,
1571                                    "New buffer allocation failed,"
1572                                    "dropping incoming packetn");
1573                         dev = &rte_eth_devices[rxq->port_id];
1574                         dev->data->rx_mbuf_alloc_failed +=
1575                                                         rxq->rx_alloc_count;
1576                         rxq->rx_alloc_errors += rxq->rx_alloc_count;
1577                         return 0;
1578                 }
1579                 qede_update_rx_prod(qdev, rxq);
1580                 rxq->rx_alloc_count = 0;
1581         }
1582
1583         hw_comp_cons = rte_le_to_cpu_16(*rxq->hw_cons_ptr);
1584         sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
1585
1586         rte_rmb();
1587
1588         if (hw_comp_cons == sw_comp_cons)
1589                 return 0;
1590
1591         num_rx_bds =  NUM_RX_BDS(rxq);
1592         port_id = rxq->port_id;
1593
1594         while (sw_comp_cons != hw_comp_cons) {
1595                 ol_flags = 0;
1596                 packet_type = RTE_PTYPE_UNKNOWN;
1597                 vlan_tci = 0;
1598                 rss_hash = 0;
1599
1600                 /* Get the CQE from the completion ring */
1601                 cqe =
1602                     (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring);
1603                 cqe_type = cqe->fast_path_regular.type;
1604                 PMD_RX_LOG(INFO, rxq, "Rx CQE type %d\n", cqe_type);
1605
1606                 if (likely(cqe_type == ETH_RX_CQE_TYPE_REGULAR)) {
1607                         fp_cqe = &cqe->fast_path_regular;
1608                 } else {
1609                         if (cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH) {
1610                                 PMD_RX_LOG(INFO, rxq, "Got unexpected slowpath CQE\n");
1611                                 ecore_eth_cqe_completion
1612                                         (&edev->hwfns[rxq->queue_id %
1613                                                       edev->num_hwfns],
1614                                          (struct eth_slow_path_rx_cqe *)cqe);
1615                         }
1616                         goto next_cqe;
1617                 }
1618
1619                 /* Get the data from the SW ring */
1620                 sw_rx_index = rxq->sw_rx_cons & num_rx_bds;
1621                 rx_mb = rxq->sw_rx_ring[sw_rx_index];
1622                 assert(rx_mb != NULL);
1623
1624                 parse_flag = rte_le_to_cpu_16(fp_cqe->pars_flags.flags);
1625                 offset = fp_cqe->placement_offset;
1626                 len = rte_le_to_cpu_16(fp_cqe->len_on_first_bd);
1627                 pkt_len = rte_le_to_cpu_16(fp_cqe->pkt_len);
1628                 vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag);
1629                 rss_hash = rte_le_to_cpu_32(fp_cqe->rss_hash);
1630                 bd_num = fp_cqe->bd_num;
1631 #ifdef RTE_LIBRTE_QEDE_DEBUG_RX
1632                 bitfield_val = fp_cqe->bitfields;
1633 #endif
1634
1635                 if (unlikely(qede_tunn_exist(parse_flag))) {
1636                         PMD_RX_LOG(INFO, rxq, "Rx tunneled packet\n");
1637                         if (unlikely(qede_check_tunn_csum_l4(parse_flag))) {
1638                                 PMD_RX_LOG(ERR, rxq,
1639                                             "L4 csum failed, flags = 0x%x\n",
1640                                             parse_flag);
1641                                 rxq->rx_hw_errors++;
1642                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
1643                         } else {
1644                                 ol_flags |= PKT_RX_L4_CKSUM_GOOD;
1645                         }
1646
1647                         if (unlikely(qede_check_tunn_csum_l3(parse_flag))) {
1648                                 PMD_RX_LOG(ERR, rxq,
1649                                         "Outer L3 csum failed, flags = 0x%x\n",
1650                                         parse_flag);
1651                                 rxq->rx_hw_errors++;
1652                                 ol_flags |= PKT_RX_OUTER_IP_CKSUM_BAD;
1653                         } else {
1654                                 ol_flags |= PKT_RX_IP_CKSUM_GOOD;
1655                         }
1656
1657                         flags = fp_cqe->tunnel_pars_flags.flags;
1658
1659                         /* Tunnel_type */
1660                         packet_type =
1661                                 qede_rx_cqe_to_tunn_pkt_type(flags);
1662
1663                         /* Inner header */
1664                         packet_type |=
1665                               qede_rx_cqe_to_pkt_type_inner(parse_flag);
1666
1667                         /* Outer L3/L4 types is not available in CQE */
1668                         packet_type |= qede_rx_cqe_to_pkt_type_outer(rx_mb);
1669
1670                         /* Outer L3/L4 types is not available in CQE.
1671                          * Need to add offset to parse correctly,
1672                          */
1673                         rx_mb->data_off = offset + RTE_PKTMBUF_HEADROOM;
1674                         packet_type |= qede_rx_cqe_to_pkt_type_outer(rx_mb);
1675                 } else {
1676                         packet_type |= qede_rx_cqe_to_pkt_type(parse_flag);
1677                 }
1678
1679                 /* Common handling for non-tunnel packets and for inner
1680                  * headers in the case of tunnel.
1681                  */
1682                 if (unlikely(qede_check_notunn_csum_l4(parse_flag))) {
1683                         PMD_RX_LOG(ERR, rxq,
1684                                     "L4 csum failed, flags = 0x%x\n",
1685                                     parse_flag);
1686                         rxq->rx_hw_errors++;
1687                         ol_flags |= PKT_RX_L4_CKSUM_BAD;
1688                 } else {
1689                         ol_flags |= PKT_RX_L4_CKSUM_GOOD;
1690                 }
1691                 if (unlikely(qede_check_notunn_csum_l3(rx_mb, parse_flag))) {
1692                         PMD_RX_LOG(ERR, rxq, "IP csum failed, flags = 0x%x\n",
1693                                    parse_flag);
1694                         rxq->rx_hw_errors++;
1695                         ol_flags |= PKT_RX_IP_CKSUM_BAD;
1696                 } else {
1697                         ol_flags |= PKT_RX_IP_CKSUM_GOOD;
1698                 }
1699
1700                 if (unlikely(CQE_HAS_VLAN(parse_flag) ||
1701                              CQE_HAS_OUTER_VLAN(parse_flag))) {
1702                         /* Note: FW doesn't indicate Q-in-Q packet */
1703                         ol_flags |= PKT_RX_VLAN;
1704                         if (qdev->vlan_strip_flg) {
1705                                 ol_flags |= PKT_RX_VLAN_STRIPPED;
1706                                 rx_mb->vlan_tci = vlan_tci;
1707                         }
1708                 }
1709
1710                 if (rss_enable) {
1711                         ol_flags |= PKT_RX_RSS_HASH;
1712                         rx_mb->hash.rss = rss_hash;
1713                 }
1714
1715                 rx_alloc_count++;
1716                 qede_rx_bd_ring_consume(rxq);
1717
1718                 /* Prefetch next mbuf while processing current one. */
1719                 preload_idx = rxq->sw_rx_cons & num_rx_bds;
1720                 rte_prefetch0(rxq->sw_rx_ring[preload_idx]);
1721
1722                 /* Update rest of the MBUF fields */
1723                 rx_mb->data_off = offset + RTE_PKTMBUF_HEADROOM;
1724                 rx_mb->port = port_id;
1725                 rx_mb->ol_flags = ol_flags;
1726                 rx_mb->data_len = len;
1727                 rx_mb->packet_type = packet_type;
1728 #ifdef RTE_LIBRTE_QEDE_DEBUG_RX
1729                 print_rx_bd_info(rx_mb, rxq, bitfield_val);
1730 #endif
1731                 rx_mb->nb_segs = bd_num;
1732                 rx_mb->pkt_len = pkt_len;
1733
1734                 rx_pkts[rx_pkt] = rx_mb;
1735                 rx_pkt++;
1736
1737 next_cqe:
1738                 ecore_chain_recycle_consumed(&rxq->rx_comp_ring);
1739                 sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
1740                 if (rx_pkt == nb_pkts) {
1741                         PMD_RX_LOG(DEBUG, rxq,
1742                                    "Budget reached nb_pkts=%u received=%u",
1743                                    rx_pkt, nb_pkts);
1744                         break;
1745                 }
1746         }
1747
1748         /* Request number of bufferes to be allocated in next loop */
1749         rxq->rx_alloc_count = rx_alloc_count;
1750
1751         rxq->rcv_pkts += rx_pkt;
1752         rxq->rx_segs += rx_pkt;
1753         PMD_RX_LOG(DEBUG, rxq, "rx_pkts=%u core=%d", rx_pkt, rte_lcore_id());
1754
1755         return rx_pkt;
1756 }
1757
1758 uint16_t
1759 qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1760 {
1761         struct qede_rx_queue *rxq = p_rxq;
1762         struct qede_dev *qdev = rxq->qdev;
1763         struct ecore_dev *edev = &qdev->edev;
1764         uint16_t hw_comp_cons, sw_comp_cons, sw_rx_index;
1765         uint16_t rx_pkt = 0;
1766         union eth_rx_cqe *cqe;
1767         struct eth_fast_path_rx_reg_cqe *fp_cqe = NULL;
1768         register struct rte_mbuf *rx_mb = NULL;
1769         register struct rte_mbuf *seg1 = NULL;
1770         enum eth_rx_cqe_type cqe_type;
1771         uint16_t pkt_len = 0; /* Sum of all BD segments */
1772         uint16_t len; /* Length of first BD */
1773         uint8_t num_segs = 1;
1774         uint16_t preload_idx;
1775         uint16_t parse_flag;
1776 #ifdef RTE_LIBRTE_QEDE_DEBUG_RX
1777         uint8_t bitfield_val;
1778 #endif
1779         uint8_t tunn_parse_flag;
1780         struct eth_fast_path_rx_tpa_start_cqe *cqe_start_tpa;
1781         uint64_t ol_flags;
1782         uint32_t packet_type;
1783         uint16_t vlan_tci;
1784         bool tpa_start_flg;
1785         uint8_t offset, tpa_agg_idx, flags;
1786         struct qede_agg_info *tpa_info = NULL;
1787         uint32_t rss_hash;
1788         int rx_alloc_count = 0;
1789
1790
1791         /* Allocate buffers that we used in previous loop */
1792         if (rxq->rx_alloc_count) {
1793                 if (unlikely(qede_alloc_rx_bulk_mbufs(rxq,
1794                              rxq->rx_alloc_count))) {
1795                         struct rte_eth_dev *dev;
1796
1797                         PMD_RX_LOG(ERR, rxq,
1798                                    "New buffer allocation failed,"
1799                                    "dropping incoming packetn");
1800                         dev = &rte_eth_devices[rxq->port_id];
1801                         dev->data->rx_mbuf_alloc_failed +=
1802                                                         rxq->rx_alloc_count;
1803                         rxq->rx_alloc_errors += rxq->rx_alloc_count;
1804                         return 0;
1805                 }
1806                 qede_update_rx_prod(qdev, rxq);
1807                 rxq->rx_alloc_count = 0;
1808         }
1809
1810         hw_comp_cons = rte_le_to_cpu_16(*rxq->hw_cons_ptr);
1811         sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
1812
1813         rte_rmb();
1814
1815         if (hw_comp_cons == sw_comp_cons)
1816                 return 0;
1817
1818         while (sw_comp_cons != hw_comp_cons) {
1819                 ol_flags = 0;
1820                 packet_type = RTE_PTYPE_UNKNOWN;
1821                 vlan_tci = 0;
1822                 tpa_start_flg = false;
1823                 rss_hash = 0;
1824
1825                 /* Get the CQE from the completion ring */
1826                 cqe =
1827                     (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring);
1828                 cqe_type = cqe->fast_path_regular.type;
1829                 PMD_RX_LOG(INFO, rxq, "Rx CQE type %d\n", cqe_type);
1830
1831                 switch (cqe_type) {
1832                 case ETH_RX_CQE_TYPE_REGULAR:
1833                         fp_cqe = &cqe->fast_path_regular;
1834                 break;
1835                 case ETH_RX_CQE_TYPE_TPA_START:
1836                         cqe_start_tpa = &cqe->fast_path_tpa_start;
1837                         tpa_info = &rxq->tpa_info[cqe_start_tpa->tpa_agg_index];
1838                         tpa_start_flg = true;
1839                         /* Mark it as LRO packet */
1840                         ol_flags |= PKT_RX_LRO;
1841                         /* In split mode,  seg_len is same as len_on_first_bd
1842                          * and bw_ext_bd_len_list will be empty since there are
1843                          * no additional buffers
1844                          */
1845                         PMD_RX_LOG(INFO, rxq,
1846                          "TPA start[%d] - len_on_first_bd %d header %d"
1847                          " [bd_list[0] %d], [seg_len %d]\n",
1848                          cqe_start_tpa->tpa_agg_index,
1849                          rte_le_to_cpu_16(cqe_start_tpa->len_on_first_bd),
1850                          cqe_start_tpa->header_len,
1851                          rte_le_to_cpu_16(cqe_start_tpa->bw_ext_bd_len_list[0]),
1852                          rte_le_to_cpu_16(cqe_start_tpa->seg_len));
1853
1854                 break;
1855                 case ETH_RX_CQE_TYPE_TPA_CONT:
1856                         qede_rx_process_tpa_cont_cqe(qdev, rxq,
1857                                                      &cqe->fast_path_tpa_cont);
1858                         goto next_cqe;
1859                 case ETH_RX_CQE_TYPE_TPA_END:
1860                         qede_rx_process_tpa_end_cqe(qdev, rxq,
1861                                                     &cqe->fast_path_tpa_end);
1862                         tpa_agg_idx = cqe->fast_path_tpa_end.tpa_agg_index;
1863                         tpa_info = &rxq->tpa_info[tpa_agg_idx];
1864                         rx_mb = rxq->tpa_info[tpa_agg_idx].tpa_head;
1865                         goto tpa_end;
1866                 case ETH_RX_CQE_TYPE_SLOW_PATH:
1867                         PMD_RX_LOG(INFO, rxq, "Got unexpected slowpath CQE\n");
1868                         ecore_eth_cqe_completion(
1869                                 &edev->hwfns[rxq->queue_id % edev->num_hwfns],
1870                                 (struct eth_slow_path_rx_cqe *)cqe);
1871                         /* fall-thru */
1872                 default:
1873                         goto next_cqe;
1874                 }
1875
1876                 /* Get the data from the SW ring */
1877                 sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
1878                 rx_mb = rxq->sw_rx_ring[sw_rx_index];
1879                 assert(rx_mb != NULL);
1880
1881                 /* Handle regular CQE or TPA start CQE */
1882                 if (!tpa_start_flg) {
1883                         parse_flag = rte_le_to_cpu_16(fp_cqe->pars_flags.flags);
1884                         offset = fp_cqe->placement_offset;
1885                         len = rte_le_to_cpu_16(fp_cqe->len_on_first_bd);
1886                         pkt_len = rte_le_to_cpu_16(fp_cqe->pkt_len);
1887                         vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag);
1888                         rss_hash = rte_le_to_cpu_32(fp_cqe->rss_hash);
1889 #ifdef RTE_LIBRTE_QEDE_DEBUG_RX
1890                         bitfield_val = fp_cqe->bitfields;
1891 #endif
1892                 } else {
1893                         parse_flag =
1894                             rte_le_to_cpu_16(cqe_start_tpa->pars_flags.flags);
1895                         offset = cqe_start_tpa->placement_offset;
1896                         /* seg_len = len_on_first_bd */
1897                         len = rte_le_to_cpu_16(cqe_start_tpa->len_on_first_bd);
1898                         vlan_tci = rte_le_to_cpu_16(cqe_start_tpa->vlan_tag);
1899 #ifdef RTE_LIBRTE_QEDE_DEBUG_RX
1900                         bitfield_val = cqe_start_tpa->bitfields;
1901 #endif
1902                         rss_hash = rte_le_to_cpu_32(cqe_start_tpa->rss_hash);
1903                 }
1904                 if (qede_tunn_exist(parse_flag)) {
1905                         PMD_RX_LOG(INFO, rxq, "Rx tunneled packet\n");
1906                         if (unlikely(qede_check_tunn_csum_l4(parse_flag))) {
1907                                 PMD_RX_LOG(ERR, rxq,
1908                                             "L4 csum failed, flags = 0x%x\n",
1909                                             parse_flag);
1910                                 rxq->rx_hw_errors++;
1911                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
1912                         } else {
1913                                 ol_flags |= PKT_RX_L4_CKSUM_GOOD;
1914                         }
1915
1916                         if (unlikely(qede_check_tunn_csum_l3(parse_flag))) {
1917                                 PMD_RX_LOG(ERR, rxq,
1918                                         "Outer L3 csum failed, flags = 0x%x\n",
1919                                         parse_flag);
1920                                   rxq->rx_hw_errors++;
1921                                   ol_flags |= PKT_RX_OUTER_IP_CKSUM_BAD;
1922                         } else {
1923                                   ol_flags |= PKT_RX_IP_CKSUM_GOOD;
1924                         }
1925
1926                         if (tpa_start_flg)
1927                                 flags = cqe_start_tpa->tunnel_pars_flags.flags;
1928                         else
1929                                 flags = fp_cqe->tunnel_pars_flags.flags;
1930                         tunn_parse_flag = flags;
1931
1932                         /* Tunnel_type */
1933                         packet_type =
1934                                 qede_rx_cqe_to_tunn_pkt_type(tunn_parse_flag);
1935
1936                         /* Inner header */
1937                         packet_type |=
1938                               qede_rx_cqe_to_pkt_type_inner(parse_flag);
1939
1940                         /* Outer L3/L4 types is not available in CQE */
1941                         packet_type |= qede_rx_cqe_to_pkt_type_outer(rx_mb);
1942
1943                         /* Outer L3/L4 types is not available in CQE.
1944                          * Need to add offset to parse correctly,
1945                          */
1946                         rx_mb->data_off = offset + RTE_PKTMBUF_HEADROOM;
1947                         packet_type |= qede_rx_cqe_to_pkt_type_outer(rx_mb);
1948                 } else {
1949                         packet_type |= qede_rx_cqe_to_pkt_type(parse_flag);
1950                 }
1951
1952                 /* Common handling for non-tunnel packets and for inner
1953                  * headers in the case of tunnel.
1954                  */
1955                 if (unlikely(qede_check_notunn_csum_l4(parse_flag))) {
1956                         PMD_RX_LOG(ERR, rxq,
1957                                     "L4 csum failed, flags = 0x%x\n",
1958                                     parse_flag);
1959                         rxq->rx_hw_errors++;
1960                         ol_flags |= PKT_RX_L4_CKSUM_BAD;
1961                 } else {
1962                         ol_flags |= PKT_RX_L4_CKSUM_GOOD;
1963                 }
1964                 if (unlikely(qede_check_notunn_csum_l3(rx_mb, parse_flag))) {
1965                         PMD_RX_LOG(ERR, rxq, "IP csum failed, flags = 0x%x\n",
1966                                    parse_flag);
1967                         rxq->rx_hw_errors++;
1968                         ol_flags |= PKT_RX_IP_CKSUM_BAD;
1969                 } else {
1970                         ol_flags |= PKT_RX_IP_CKSUM_GOOD;
1971                 }
1972
1973                 if (CQE_HAS_VLAN(parse_flag) ||
1974                     CQE_HAS_OUTER_VLAN(parse_flag)) {
1975                         /* Note: FW doesn't indicate Q-in-Q packet */
1976                         ol_flags |= PKT_RX_VLAN;
1977                         if (qdev->vlan_strip_flg) {
1978                                 ol_flags |= PKT_RX_VLAN_STRIPPED;
1979                                 rx_mb->vlan_tci = vlan_tci;
1980                         }
1981                 }
1982
1983                 /* RSS Hash */
1984                 if (qdev->rss_enable) {
1985                         ol_flags |= PKT_RX_RSS_HASH;
1986                         rx_mb->hash.rss = rss_hash;
1987                 }
1988
1989                 rx_alloc_count++;
1990                 qede_rx_bd_ring_consume(rxq);
1991
1992                 if (!tpa_start_flg && fp_cqe->bd_num > 1) {
1993                         PMD_RX_LOG(DEBUG, rxq, "Jumbo-over-BD packet: %02x BDs"
1994                                    " len on first: %04x Total Len: %04x",
1995                                    fp_cqe->bd_num, len, pkt_len);
1996                         num_segs = fp_cqe->bd_num - 1;
1997                         seg1 = rx_mb;
1998                         if (qede_process_sg_pkts(p_rxq, seg1, num_segs,
1999                                                  pkt_len - len))
2000                                 goto next_cqe;
2001
2002                         rx_alloc_count += num_segs;
2003                         rxq->rx_segs += num_segs;
2004                 }
2005                 rxq->rx_segs++; /* for the first segment */
2006
2007                 /* Prefetch next mbuf while processing current one. */
2008                 preload_idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
2009                 rte_prefetch0(rxq->sw_rx_ring[preload_idx]);
2010
2011                 /* Update rest of the MBUF fields */
2012                 rx_mb->data_off = offset + RTE_PKTMBUF_HEADROOM;
2013                 rx_mb->port = rxq->port_id;
2014                 rx_mb->ol_flags = ol_flags;
2015                 rx_mb->data_len = len;
2016                 rx_mb->packet_type = packet_type;
2017 #ifdef RTE_LIBRTE_QEDE_DEBUG_RX
2018                 print_rx_bd_info(rx_mb, rxq, bitfield_val);
2019 #endif
2020                 if (!tpa_start_flg) {
2021                         rx_mb->nb_segs = fp_cqe->bd_num;
2022                         rx_mb->pkt_len = pkt_len;
2023                 } else {
2024                         /* store ref to the updated mbuf */
2025                         tpa_info->tpa_head = rx_mb;
2026                         tpa_info->tpa_tail = tpa_info->tpa_head;
2027                 }
2028                 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
2029 tpa_end:
2030                 if (!tpa_start_flg) {
2031                         rx_pkts[rx_pkt] = rx_mb;
2032                         rx_pkt++;
2033                 }
2034 next_cqe:
2035                 ecore_chain_recycle_consumed(&rxq->rx_comp_ring);
2036                 sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
2037                 if (rx_pkt == nb_pkts) {
2038                         PMD_RX_LOG(DEBUG, rxq,
2039                                    "Budget reached nb_pkts=%u received=%u",
2040                                    rx_pkt, nb_pkts);
2041                         break;
2042                 }
2043         }
2044
2045         /* Request number of bufferes to be allocated in next loop */
2046         rxq->rx_alloc_count = rx_alloc_count;
2047
2048         rxq->rcv_pkts += rx_pkt;
2049
2050         PMD_RX_LOG(DEBUG, rxq, "rx_pkts=%u core=%d", rx_pkt, rte_lcore_id());
2051
2052         return rx_pkt;
2053 }
2054
2055 uint16_t
2056 qede_recv_pkts_cmt(void *p_fp_cmt, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2057 {
2058         struct qede_fastpath_cmt *fp_cmt = p_fp_cmt;
2059         uint16_t eng0_pkts, eng1_pkts;
2060
2061         eng0_pkts = nb_pkts / 2;
2062
2063         eng0_pkts = qede_recv_pkts(fp_cmt->fp0->rxq, rx_pkts, eng0_pkts);
2064
2065         eng1_pkts = nb_pkts - eng0_pkts;
2066
2067         eng1_pkts = qede_recv_pkts(fp_cmt->fp1->rxq, rx_pkts + eng0_pkts,
2068                                    eng1_pkts);
2069
2070         return eng0_pkts + eng1_pkts;
2071 }
2072
2073 /* Populate scatter gather buffer descriptor fields */
2074 static inline uint16_t
2075 qede_encode_sg_bd(struct qede_tx_queue *p_txq, struct rte_mbuf *m_seg,
2076                   struct eth_tx_2nd_bd **bd2, struct eth_tx_3rd_bd **bd3,
2077                   uint16_t start_seg)
2078 {
2079         struct qede_tx_queue *txq = p_txq;
2080         struct eth_tx_bd *tx_bd = NULL;
2081         dma_addr_t mapping;
2082         uint16_t nb_segs = 0;
2083
2084         /* Check for scattered buffers */
2085         while (m_seg) {
2086                 if (start_seg == 0) {
2087                         if (!*bd2) {
2088                                 *bd2 = (struct eth_tx_2nd_bd *)
2089                                         ecore_chain_produce(&txq->tx_pbl);
2090                                 memset(*bd2, 0, sizeof(struct eth_tx_2nd_bd));
2091                                 nb_segs++;
2092                         }
2093                         mapping = rte_mbuf_data_iova(m_seg);
2094                         QEDE_BD_SET_ADDR_LEN(*bd2, mapping, m_seg->data_len);
2095                         PMD_TX_LOG(DEBUG, txq, "BD2 len %04x", m_seg->data_len);
2096                 } else if (start_seg == 1) {
2097                         if (!*bd3) {
2098                                 *bd3 = (struct eth_tx_3rd_bd *)
2099                                         ecore_chain_produce(&txq->tx_pbl);
2100                                 memset(*bd3, 0, sizeof(struct eth_tx_3rd_bd));
2101                                 nb_segs++;
2102                         }
2103                         mapping = rte_mbuf_data_iova(m_seg);
2104                         QEDE_BD_SET_ADDR_LEN(*bd3, mapping, m_seg->data_len);
2105                         PMD_TX_LOG(DEBUG, txq, "BD3 len %04x", m_seg->data_len);
2106                 } else {
2107                         tx_bd = (struct eth_tx_bd *)
2108                                 ecore_chain_produce(&txq->tx_pbl);
2109                         memset(tx_bd, 0, sizeof(*tx_bd));
2110                         nb_segs++;
2111                         mapping = rte_mbuf_data_iova(m_seg);
2112                         QEDE_BD_SET_ADDR_LEN(tx_bd, mapping, m_seg->data_len);
2113                         PMD_TX_LOG(DEBUG, txq, "BD len %04x", m_seg->data_len);
2114                 }
2115                 start_seg++;
2116                 m_seg = m_seg->next;
2117         }
2118
2119         /* Return total scattered buffers */
2120         return nb_segs;
2121 }
2122
2123 #ifdef RTE_LIBRTE_QEDE_DEBUG_TX
2124 static inline void
2125 print_tx_bd_info(struct qede_tx_queue *txq,
2126                  struct eth_tx_1st_bd *bd1,
2127                  struct eth_tx_2nd_bd *bd2,
2128                  struct eth_tx_3rd_bd *bd3,
2129                  uint64_t tx_ol_flags)
2130 {
2131         char ol_buf[256] = { 0 }; /* for verbose prints */
2132
2133         if (bd1)
2134                 PMD_TX_LOG(INFO, txq,
2135                    "BD1: nbytes=0x%04x nbds=0x%04x bd_flags=0x%04x bf=0x%04x",
2136                    rte_cpu_to_le_16(bd1->nbytes), bd1->data.nbds,
2137                    bd1->data.bd_flags.bitfields,
2138                    rte_cpu_to_le_16(bd1->data.bitfields));
2139         if (bd2)
2140                 PMD_TX_LOG(INFO, txq,
2141                    "BD2: nbytes=0x%04x bf1=0x%04x bf2=0x%04x tunn_ip=0x%04x\n",
2142                    rte_cpu_to_le_16(bd2->nbytes), bd2->data.bitfields1,
2143                    bd2->data.bitfields2, bd2->data.tunn_ip_size);
2144         if (bd3)
2145                 PMD_TX_LOG(INFO, txq,
2146                    "BD3: nbytes=0x%04x bf=0x%04x MSS=0x%04x "
2147                    "tunn_l4_hdr_start_offset_w=0x%04x tunn_hdr_size=0x%04x\n",
2148                    rte_cpu_to_le_16(bd3->nbytes),
2149                    rte_cpu_to_le_16(bd3->data.bitfields),
2150                    rte_cpu_to_le_16(bd3->data.lso_mss),
2151                    bd3->data.tunn_l4_hdr_start_offset_w,
2152                    bd3->data.tunn_hdr_size_w);
2153
2154         rte_get_tx_ol_flag_list(tx_ol_flags, ol_buf, sizeof(ol_buf));
2155         PMD_TX_LOG(INFO, txq, "TX offloads = %s\n", ol_buf);
2156 }
2157 #endif
2158
2159 /* TX prepare to check packets meets TX conditions */
2160 uint16_t
2161 #ifdef RTE_LIBRTE_QEDE_DEBUG_TX
2162 qede_xmit_prep_pkts(void *p_txq, struct rte_mbuf **tx_pkts,
2163                     uint16_t nb_pkts)
2164 {
2165         struct qede_tx_queue *txq = p_txq;
2166 #else
2167 qede_xmit_prep_pkts(__rte_unused void *p_txq, struct rte_mbuf **tx_pkts,
2168                     uint16_t nb_pkts)
2169 {
2170 #endif
2171         uint64_t ol_flags;
2172         struct rte_mbuf *m;
2173         uint16_t i;
2174 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
2175         int ret;
2176 #endif
2177
2178         for (i = 0; i < nb_pkts; i++) {
2179                 m = tx_pkts[i];
2180                 ol_flags = m->ol_flags;
2181                 if (ol_flags & PKT_TX_TCP_SEG) {
2182                         if (m->nb_segs >= ETH_TX_MAX_BDS_PER_LSO_PACKET) {
2183                                 rte_errno = EINVAL;
2184                                 break;
2185                         }
2186                         /* TBD: confirm its ~9700B for both ? */
2187                         if (m->tso_segsz > ETH_TX_MAX_NON_LSO_PKT_LEN) {
2188                                 rte_errno = EINVAL;
2189                                 break;
2190                         }
2191                 } else {
2192                         if (m->nb_segs >= ETH_TX_MAX_BDS_PER_NON_LSO_PACKET) {
2193                                 rte_errno = EINVAL;
2194                                 break;
2195                         }
2196                 }
2197                 if (ol_flags & QEDE_TX_OFFLOAD_NOTSUP_MASK) {
2198                         /* We support only limited tunnel protocols */
2199                         if (ol_flags & PKT_TX_TUNNEL_MASK) {
2200                                 uint64_t temp;
2201
2202                                 temp = ol_flags & PKT_TX_TUNNEL_MASK;
2203                                 if (temp == PKT_TX_TUNNEL_VXLAN ||
2204                                     temp == PKT_TX_TUNNEL_GENEVE ||
2205                                     temp == PKT_TX_TUNNEL_MPLSINUDP ||
2206                                     temp == PKT_TX_TUNNEL_GRE)
2207                                         continue;
2208                         }
2209
2210                         rte_errno = ENOTSUP;
2211                         break;
2212                 }
2213
2214 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
2215                 ret = rte_validate_tx_offload(m);
2216                 if (ret != 0) {
2217                         rte_errno = -ret;
2218                         break;
2219                 }
2220 #endif
2221         }
2222
2223 #ifdef RTE_LIBRTE_QEDE_DEBUG_TX
2224         if (unlikely(i != nb_pkts))
2225                 PMD_TX_LOG(ERR, txq, "TX prepare failed for %u\n",
2226                            nb_pkts - i);
2227 #endif
2228         return i;
2229 }
2230
2231 #define MPLSINUDP_HDR_SIZE                      (12)
2232
2233 #ifdef RTE_LIBRTE_QEDE_DEBUG_TX
2234 static inline void
2235 qede_mpls_tunn_tx_sanity_check(struct rte_mbuf *mbuf,
2236                                struct qede_tx_queue *txq)
2237 {
2238         if (((mbuf->outer_l2_len + mbuf->outer_l3_len) / 2) > 0xff)
2239                 PMD_TX_LOG(ERR, txq, "tunn_l4_hdr_start_offset overflow\n");
2240         if (((mbuf->outer_l2_len + mbuf->outer_l3_len +
2241                 MPLSINUDP_HDR_SIZE) / 2) > 0xff)
2242                 PMD_TX_LOG(ERR, txq, "tunn_hdr_size overflow\n");
2243         if (((mbuf->l2_len - MPLSINUDP_HDR_SIZE) / 2) >
2244                 ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_MASK)
2245                 PMD_TX_LOG(ERR, txq, "inner_l2_hdr_size overflow\n");
2246         if (((mbuf->l2_len - MPLSINUDP_HDR_SIZE + mbuf->l3_len) / 2) >
2247                 ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_MASK)
2248                 PMD_TX_LOG(ERR, txq, "inner_l2_hdr_size overflow\n");
2249 }
2250 #endif
2251
2252 uint16_t
2253 qede_xmit_pkts_regular(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2254 {
2255         struct qede_tx_queue *txq = p_txq;
2256         struct qede_dev *qdev = txq->qdev;
2257         struct ecore_dev *edev = &qdev->edev;
2258         struct eth_tx_1st_bd *bd1;
2259         struct eth_tx_2nd_bd *bd2;
2260         struct eth_tx_3rd_bd *bd3;
2261         struct rte_mbuf *m_seg = NULL;
2262         struct rte_mbuf *mbuf;
2263         struct rte_mbuf **sw_tx_ring;
2264         uint16_t nb_tx_pkts;
2265         uint16_t bd_prod;
2266         uint16_t idx;
2267         uint16_t nb_frags = 0;
2268         uint16_t nb_pkt_sent = 0;
2269         uint8_t nbds;
2270         uint64_t tx_ol_flags;
2271         /* BD1 */
2272         uint16_t bd1_bf;
2273         uint8_t bd1_bd_flags_bf;
2274
2275         if (unlikely(txq->nb_tx_avail < txq->tx_free_thresh)) {
2276                 PMD_TX_LOG(DEBUG, txq, "send=%u avail=%u free_thresh=%u",
2277                            nb_pkts, txq->nb_tx_avail, txq->tx_free_thresh);
2278                 qede_process_tx_compl(edev, txq);
2279         }
2280
2281         nb_tx_pkts  = nb_pkts;
2282         bd_prod = rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl));
2283         sw_tx_ring = txq->sw_tx_ring;
2284
2285         while (nb_tx_pkts--) {
2286                 /* Init flags/values */
2287                 nbds = 0;
2288                 bd1 = NULL;
2289                 bd2 = NULL;
2290                 bd3 = NULL;
2291                 bd1_bf = 0;
2292                 bd1_bd_flags_bf = 0;
2293                 nb_frags = 0;
2294
2295                 mbuf = *tx_pkts++;
2296                 assert(mbuf);
2297
2298
2299                 /* Check minimum TX BDS availability against available BDs */
2300                 if (unlikely(txq->nb_tx_avail < mbuf->nb_segs))
2301                         break;
2302
2303                 tx_ol_flags = mbuf->ol_flags;
2304                 bd1_bd_flags_bf |= 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT;
2305
2306                 if (unlikely(txq->nb_tx_avail <
2307                                 ETH_TX_MIN_BDS_PER_NON_LSO_PKT))
2308                         break;
2309                 bd1_bf |=
2310                        (mbuf->pkt_len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK)
2311                         << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT;
2312
2313                 /* Offload the IP checksum in the hardware */
2314                 if (tx_ol_flags & PKT_TX_IP_CKSUM)
2315                         bd1_bd_flags_bf |=
2316                                 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT;
2317
2318                 /* L4 checksum offload (tcp or udp) */
2319                 if ((tx_ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)) &&
2320                     (tx_ol_flags & (PKT_TX_UDP_CKSUM | PKT_TX_TCP_CKSUM)))
2321                         bd1_bd_flags_bf |=
2322                                 1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT;
2323
2324                 /* Fill the entry in the SW ring and the BDs in the FW ring */
2325                 idx = TX_PROD(txq);
2326                 sw_tx_ring[idx] = mbuf;
2327
2328                 /* BD1 */
2329                 bd1 = (struct eth_tx_1st_bd *)ecore_chain_produce(&txq->tx_pbl);
2330                 memset(bd1, 0, sizeof(struct eth_tx_1st_bd));
2331                 nbds++;
2332
2333                 /* Map MBUF linear data for DMA and set in the BD1 */
2334                 QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_iova(mbuf),
2335                                      mbuf->data_len);
2336                 bd1->data.bitfields = rte_cpu_to_le_16(bd1_bf);
2337                 bd1->data.bd_flags.bitfields = bd1_bd_flags_bf;
2338
2339                 /* Handle fragmented MBUF */
2340                 if (unlikely(mbuf->nb_segs > 1)) {
2341                         m_seg = mbuf->next;
2342
2343                         /* Encode scatter gather buffer descriptors */
2344                         nb_frags = qede_encode_sg_bd(txq, m_seg, &bd2, &bd3,
2345                                                      nbds - 1);
2346                 }
2347
2348                 bd1->data.nbds = nbds + nb_frags;
2349
2350                 txq->nb_tx_avail -= bd1->data.nbds;
2351                 txq->sw_tx_prod++;
2352                 bd_prod =
2353                     rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl));
2354 #ifdef RTE_LIBRTE_QEDE_DEBUG_TX
2355                 print_tx_bd_info(txq, bd1, bd2, bd3, tx_ol_flags);
2356 #endif
2357                 nb_pkt_sent++;
2358                 txq->xmit_pkts++;
2359         }
2360
2361         /* Write value of prod idx into bd_prod */
2362         txq->tx_db.data.bd_prod = bd_prod;
2363         rte_wmb();
2364         rte_compiler_barrier();
2365         DIRECT_REG_WR_RELAXED(edev, txq->doorbell_addr, txq->tx_db.raw);
2366         rte_wmb();
2367
2368         /* Check again for Tx completions */
2369         qede_process_tx_compl(edev, txq);
2370
2371         PMD_TX_LOG(DEBUG, txq, "to_send=%u sent=%u bd_prod=%u core=%d",
2372                    nb_pkts, nb_pkt_sent, TX_PROD(txq), rte_lcore_id());
2373
2374         return nb_pkt_sent;
2375 }
2376
2377 uint16_t
2378 qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2379 {
2380         struct qede_tx_queue *txq = p_txq;
2381         struct qede_dev *qdev = txq->qdev;
2382         struct ecore_dev *edev = &qdev->edev;
2383         struct rte_mbuf *mbuf;
2384         struct rte_mbuf *m_seg = NULL;
2385         uint16_t nb_tx_pkts;
2386         uint16_t bd_prod;
2387         uint16_t idx;
2388         uint16_t nb_frags;
2389         uint16_t nb_pkt_sent = 0;
2390         uint8_t nbds;
2391         bool lso_flg;
2392         bool mplsoudp_flg;
2393         __rte_unused bool tunn_flg;
2394         bool tunn_ipv6_ext_flg;
2395         struct eth_tx_1st_bd *bd1;
2396         struct eth_tx_2nd_bd *bd2;
2397         struct eth_tx_3rd_bd *bd3;
2398         uint64_t tx_ol_flags;
2399         uint16_t hdr_size;
2400         /* BD1 */
2401         uint16_t bd1_bf;
2402         uint8_t bd1_bd_flags_bf;
2403         uint16_t vlan;
2404         /* BD2 */
2405         uint16_t bd2_bf1;
2406         uint16_t bd2_bf2;
2407         /* BD3 */
2408         uint16_t mss;
2409         uint16_t bd3_bf;
2410
2411         uint8_t tunn_l4_hdr_start_offset;
2412         uint8_t tunn_hdr_size;
2413         uint8_t inner_l2_hdr_size;
2414         uint16_t inner_l4_hdr_offset;
2415
2416         if (unlikely(txq->nb_tx_avail < txq->tx_free_thresh)) {
2417                 PMD_TX_LOG(DEBUG, txq, "send=%u avail=%u free_thresh=%u",
2418                            nb_pkts, txq->nb_tx_avail, txq->tx_free_thresh);
2419                 qede_process_tx_compl(edev, txq);
2420         }
2421
2422         nb_tx_pkts  = nb_pkts;
2423         bd_prod = rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl));
2424         while (nb_tx_pkts--) {
2425                 /* Init flags/values */
2426                 tunn_flg = false;
2427                 lso_flg = false;
2428                 nbds = 0;
2429                 vlan = 0;
2430                 bd1 = NULL;
2431                 bd2 = NULL;
2432                 bd3 = NULL;
2433                 hdr_size = 0;
2434                 bd1_bf = 0;
2435                 bd1_bd_flags_bf = 0;
2436                 bd2_bf1 = 0;
2437                 bd2_bf2 = 0;
2438                 mss = 0;
2439                 bd3_bf = 0;
2440                 mplsoudp_flg = false;
2441                 tunn_ipv6_ext_flg = false;
2442                 tunn_hdr_size = 0;
2443                 tunn_l4_hdr_start_offset = 0;
2444
2445                 mbuf = *tx_pkts++;
2446                 assert(mbuf);
2447
2448                 /* Check minimum TX BDS availability against available BDs */
2449                 if (unlikely(txq->nb_tx_avail < mbuf->nb_segs))
2450                         break;
2451
2452                 tx_ol_flags = mbuf->ol_flags;
2453                 bd1_bd_flags_bf |= 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT;
2454
2455                 /* TX prepare would have already checked supported tunnel Tx
2456                  * offloads. Don't rely on pkt_type marked by Rx, instead use
2457                  * tx_ol_flags to decide.
2458                  */
2459                 tunn_flg = !!(tx_ol_flags & PKT_TX_TUNNEL_MASK);
2460
2461                 if (tunn_flg) {
2462                         /* Check against max which is Tunnel IPv6 + ext */
2463                         if (unlikely(txq->nb_tx_avail <
2464                                 ETH_TX_MIN_BDS_PER_TUNN_IPV6_WITH_EXT_PKT))
2465                                         break;
2466
2467                         /* First indicate its a tunnel pkt */
2468                         bd1_bf |= ETH_TX_DATA_1ST_BD_TUNN_FLAG_MASK <<
2469                                   ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT;
2470                         /* Legacy FW had flipped behavior in regard to this bit
2471                          * i.e. it needed to set to prevent FW from touching
2472                          * encapsulated packets when it didn't need to.
2473                          */
2474                         if (unlikely(txq->is_legacy)) {
2475                                 bd1_bf ^= 1 <<
2476                                         ETH_TX_DATA_1ST_BD_TUNN_FLAG_SHIFT;
2477                         }
2478
2479                         /* Outer IP checksum offload */
2480                         if (tx_ol_flags & (PKT_TX_OUTER_IP_CKSUM |
2481                                            PKT_TX_OUTER_IPV4)) {
2482                                 bd1_bd_flags_bf |=
2483                                         ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_MASK <<
2484                                         ETH_TX_1ST_BD_FLAGS_TUNN_IP_CSUM_SHIFT;
2485                         }
2486
2487                         /**
2488                          * Currently, only inner checksum offload in MPLS-in-UDP
2489                          * tunnel with one MPLS label is supported. Both outer
2490                          * and inner layers  lengths need to be provided in
2491                          * mbuf.
2492                          */
2493                         if ((tx_ol_flags & PKT_TX_TUNNEL_MASK) ==
2494                                                 PKT_TX_TUNNEL_MPLSINUDP) {
2495                                 mplsoudp_flg = true;
2496 #ifdef RTE_LIBRTE_QEDE_DEBUG_TX
2497                                 qede_mpls_tunn_tx_sanity_check(mbuf, txq);
2498 #endif
2499                                 /* Outer L4 offset in two byte words */
2500                                 tunn_l4_hdr_start_offset =
2501                                   (mbuf->outer_l2_len + mbuf->outer_l3_len) / 2;
2502                                 /* Tunnel header size in two byte words */
2503                                 tunn_hdr_size = (mbuf->outer_l2_len +
2504                                                 mbuf->outer_l3_len +
2505                                                 MPLSINUDP_HDR_SIZE) / 2;
2506                                 /* Inner L2 header size in two byte words */
2507                                 inner_l2_hdr_size = (mbuf->l2_len -
2508                                                 MPLSINUDP_HDR_SIZE) / 2;
2509                                 /* Inner L4 header offset from the beggining
2510                                  * of inner packet in two byte words
2511                                  */
2512                                 inner_l4_hdr_offset = (mbuf->l2_len -
2513                                         MPLSINUDP_HDR_SIZE + mbuf->l3_len) / 2;
2514
2515                                 /* Inner L2 size and address type */
2516                                 bd2_bf1 |= (inner_l2_hdr_size &
2517                                         ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_MASK) <<
2518                                         ETH_TX_DATA_2ND_BD_TUNN_INNER_L2_HDR_SIZE_W_SHIFT;
2519                                 bd2_bf1 |= (UNICAST_ADDRESS &
2520                                         ETH_TX_DATA_2ND_BD_TUNN_INNER_ETH_TYPE_MASK) <<
2521                                         ETH_TX_DATA_2ND_BD_TUNN_INNER_ETH_TYPE_SHIFT;
2522                                 /* Treated as IPv6+Ext */
2523                                 bd2_bf1 |=
2524                                     1 << ETH_TX_DATA_2ND_BD_TUNN_IPV6_EXT_SHIFT;
2525
2526                                 /* Mark inner IPv6 if present */
2527                                 if (tx_ol_flags & PKT_TX_IPV6)
2528                                         bd2_bf1 |=
2529                                                 1 << ETH_TX_DATA_2ND_BD_TUNN_INNER_IPV6_SHIFT;
2530
2531                                 /* Inner L4 offsets */
2532                                 if ((tx_ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)) &&
2533                                      (tx_ol_flags & (PKT_TX_UDP_CKSUM |
2534                                                         PKT_TX_TCP_CKSUM))) {
2535                                         /* Determines if BD3 is needed */
2536                                         tunn_ipv6_ext_flg = true;
2537                                         if ((tx_ol_flags & PKT_TX_L4_MASK) ==
2538                                                         PKT_TX_UDP_CKSUM) {
2539                                                 bd2_bf1 |=
2540                                                         1 << ETH_TX_DATA_2ND_BD_L4_UDP_SHIFT;
2541                                         }
2542
2543                                         /* TODO other pseudo checksum modes are
2544                                          * not supported
2545                                          */
2546                                         bd2_bf1 |=
2547                                         ETH_L4_PSEUDO_CSUM_CORRECT_LENGTH <<
2548                                         ETH_TX_DATA_2ND_BD_L4_PSEUDO_CSUM_MODE_SHIFT;
2549                                         bd2_bf2 |= (inner_l4_hdr_offset &
2550                                                 ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_MASK) <<
2551                                                 ETH_TX_DATA_2ND_BD_L4_HDR_START_OFFSET_W_SHIFT;
2552                                 }
2553                         } /* End MPLSoUDP */
2554                 } /* End Tunnel handling */
2555
2556                 if (tx_ol_flags & PKT_TX_TCP_SEG) {
2557                         lso_flg = true;
2558                         if (unlikely(txq->nb_tx_avail <
2559                                                 ETH_TX_MIN_BDS_PER_LSO_PKT))
2560                                 break;
2561                         /* For LSO, packet header and payload must reside on
2562                          * buffers pointed by different BDs. Using BD1 for HDR
2563                          * and BD2 onwards for data.
2564                          */
2565                         hdr_size = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len;
2566                         if (tunn_flg)
2567                                 hdr_size += mbuf->outer_l2_len +
2568                                             mbuf->outer_l3_len;
2569
2570                         bd1_bd_flags_bf |= 1 << ETH_TX_1ST_BD_FLAGS_LSO_SHIFT;
2571                         bd1_bd_flags_bf |=
2572                                         1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT;
2573                         /* PKT_TX_TCP_SEG implies PKT_TX_TCP_CKSUM */
2574                         bd1_bd_flags_bf |=
2575                                         1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT;
2576                         mss = rte_cpu_to_le_16(mbuf->tso_segsz);
2577                         /* Using one header BD */
2578                         bd3_bf |= rte_cpu_to_le_16(1 <<
2579                                         ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT);
2580                 } else {
2581                         if (unlikely(txq->nb_tx_avail <
2582                                         ETH_TX_MIN_BDS_PER_NON_LSO_PKT))
2583                                 break;
2584                         bd1_bf |=
2585                                (mbuf->pkt_len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK)
2586                                 << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT;
2587                 }
2588
2589                 /* Descriptor based VLAN insertion */
2590                 if (tx_ol_flags & PKT_TX_VLAN_PKT) {
2591                         vlan = rte_cpu_to_le_16(mbuf->vlan_tci);
2592                         bd1_bd_flags_bf |=
2593                             1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT;
2594                 }
2595
2596                 /* Offload the IP checksum in the hardware */
2597                 if (tx_ol_flags & PKT_TX_IP_CKSUM) {
2598                         bd1_bd_flags_bf |=
2599                                 1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT;
2600                         /* There's no DPDK flag to request outer-L4 csum
2601                          * offload. But in the case of tunnel if inner L3 or L4
2602                          * csum offload is requested then we need to force
2603                          * recalculation of L4 tunnel header csum also.
2604                          */
2605                         if (tunn_flg && ((tx_ol_flags & PKT_TX_TUNNEL_MASK) !=
2606                                                         PKT_TX_TUNNEL_GRE)) {
2607                                 bd1_bd_flags_bf |=
2608                                         ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK <<
2609                                         ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_SHIFT;
2610                         }
2611                 }
2612
2613                 /* L4 checksum offload (tcp or udp) */
2614                 if ((tx_ol_flags & (PKT_TX_IPV4 | PKT_TX_IPV6)) &&
2615                     (tx_ol_flags & (PKT_TX_UDP_CKSUM | PKT_TX_TCP_CKSUM))) {
2616                         bd1_bd_flags_bf |=
2617                                 1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT;
2618                         /* There's no DPDK flag to request outer-L4 csum
2619                          * offload. But in the case of tunnel if inner L3 or L4
2620                          * csum offload is requested then we need to force
2621                          * recalculation of L4 tunnel header csum also.
2622                          */
2623                         if (tunn_flg) {
2624                                 bd1_bd_flags_bf |=
2625                                         ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_MASK <<
2626                                         ETH_TX_1ST_BD_FLAGS_TUNN_L4_CSUM_SHIFT;
2627                         }
2628                 }
2629
2630                 /* Fill the entry in the SW ring and the BDs in the FW ring */
2631                 idx = TX_PROD(txq);
2632                 txq->sw_tx_ring[idx] = mbuf;
2633
2634                 /* BD1 */
2635                 bd1 = (struct eth_tx_1st_bd *)ecore_chain_produce(&txq->tx_pbl);
2636                 memset(bd1, 0, sizeof(struct eth_tx_1st_bd));
2637                 nbds++;
2638
2639                 /* Map MBUF linear data for DMA and set in the BD1 */
2640                 QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_iova(mbuf),
2641                                      mbuf->data_len);
2642                 bd1->data.bitfields = rte_cpu_to_le_16(bd1_bf);
2643                 bd1->data.bd_flags.bitfields = bd1_bd_flags_bf;
2644                 bd1->data.vlan = vlan;
2645
2646                 if (lso_flg || mplsoudp_flg) {
2647                         bd2 = (struct eth_tx_2nd_bd *)ecore_chain_produce
2648                                                         (&txq->tx_pbl);
2649                         memset(bd2, 0, sizeof(struct eth_tx_2nd_bd));
2650                         nbds++;
2651
2652                         /* BD1 */
2653                         QEDE_BD_SET_ADDR_LEN(bd1, rte_mbuf_data_iova(mbuf),
2654                                              hdr_size);
2655                         /* BD2 */
2656                         QEDE_BD_SET_ADDR_LEN(bd2, (hdr_size +
2657                                              rte_mbuf_data_iova(mbuf)),
2658                                              mbuf->data_len - hdr_size);
2659                         bd2->data.bitfields1 = rte_cpu_to_le_16(bd2_bf1);
2660                         if (mplsoudp_flg) {
2661                                 bd2->data.bitfields2 =
2662                                         rte_cpu_to_le_16(bd2_bf2);
2663                                 /* Outer L3 size */
2664                                 bd2->data.tunn_ip_size =
2665                                         rte_cpu_to_le_16(mbuf->outer_l3_len);
2666                         }
2667                         /* BD3 */
2668                         if (lso_flg || (mplsoudp_flg && tunn_ipv6_ext_flg)) {
2669                                 bd3 = (struct eth_tx_3rd_bd *)
2670                                         ecore_chain_produce(&txq->tx_pbl);
2671                                 memset(bd3, 0, sizeof(struct eth_tx_3rd_bd));
2672                                 nbds++;
2673                                 bd3->data.bitfields = rte_cpu_to_le_16(bd3_bf);
2674                                 if (lso_flg)
2675                                         bd3->data.lso_mss = mss;
2676                                 if (mplsoudp_flg) {
2677                                         bd3->data.tunn_l4_hdr_start_offset_w =
2678                                                 tunn_l4_hdr_start_offset;
2679                                         bd3->data.tunn_hdr_size_w =
2680                                                 tunn_hdr_size;
2681                                 }
2682                         }
2683                 }
2684
2685                 /* Handle fragmented MBUF */
2686                 m_seg = mbuf->next;
2687
2688                 /* Encode scatter gather buffer descriptors if required */
2689                 nb_frags = qede_encode_sg_bd(txq, m_seg, &bd2, &bd3, nbds - 1);
2690                 bd1->data.nbds = nbds + nb_frags;
2691
2692                 txq->nb_tx_avail -= bd1->data.nbds;
2693                 txq->sw_tx_prod++;
2694                 bd_prod =
2695                     rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl));
2696 #ifdef RTE_LIBRTE_QEDE_DEBUG_TX
2697                 print_tx_bd_info(txq, bd1, bd2, bd3, tx_ol_flags);
2698 #endif
2699                 nb_pkt_sent++;
2700                 txq->xmit_pkts++;
2701         }
2702
2703         /* Write value of prod idx into bd_prod */
2704         txq->tx_db.data.bd_prod = bd_prod;
2705         rte_wmb();
2706         rte_compiler_barrier();
2707         DIRECT_REG_WR_RELAXED(edev, txq->doorbell_addr, txq->tx_db.raw);
2708         rte_wmb();
2709
2710         /* Check again for Tx completions */
2711         qede_process_tx_compl(edev, txq);
2712
2713         PMD_TX_LOG(DEBUG, txq, "to_send=%u sent=%u bd_prod=%u core=%d",
2714                    nb_pkts, nb_pkt_sent, TX_PROD(txq), rte_lcore_id());
2715
2716         return nb_pkt_sent;
2717 }
2718
2719 uint16_t
2720 qede_xmit_pkts_cmt(void *p_fp_cmt, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2721 {
2722         struct qede_fastpath_cmt *fp_cmt = p_fp_cmt;
2723         uint16_t eng0_pkts, eng1_pkts;
2724
2725         eng0_pkts = nb_pkts / 2;
2726
2727         eng0_pkts = qede_xmit_pkts(fp_cmt->fp0->txq, tx_pkts, eng0_pkts);
2728
2729         eng1_pkts = nb_pkts - eng0_pkts;
2730
2731         eng1_pkts = qede_xmit_pkts(fp_cmt->fp1->txq, tx_pkts + eng0_pkts,
2732                                    eng1_pkts);
2733
2734         return eng0_pkts + eng1_pkts;
2735 }
2736
2737 uint16_t
2738 qede_rxtx_pkts_dummy(__rte_unused void *p_rxq,
2739                      __rte_unused struct rte_mbuf **pkts,
2740                      __rte_unused uint16_t nb_pkts)
2741 {
2742         return 0;
2743 }
2744
2745
2746 /* this function does a fake walk through over completion queue
2747  * to calculate number of BDs used by HW.
2748  * At the end, it restores the state of completion queue.
2749  */
2750 static uint16_t
2751 qede_parse_fp_cqe(struct qede_rx_queue *rxq)
2752 {
2753         uint16_t hw_comp_cons, sw_comp_cons, bd_count = 0;
2754         union eth_rx_cqe *cqe, *orig_cqe = NULL;
2755
2756         hw_comp_cons = rte_le_to_cpu_16(*rxq->hw_cons_ptr);
2757         sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
2758
2759         if (hw_comp_cons == sw_comp_cons)
2760                 return 0;
2761
2762         /* Get the CQE from the completion ring */
2763         cqe = (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring);
2764         orig_cqe = cqe;
2765
2766         while (sw_comp_cons != hw_comp_cons) {
2767                 switch (cqe->fast_path_regular.type) {
2768                 case ETH_RX_CQE_TYPE_REGULAR:
2769                         bd_count += cqe->fast_path_regular.bd_num;
2770                         break;
2771                 case ETH_RX_CQE_TYPE_TPA_END:
2772                         bd_count += cqe->fast_path_tpa_end.num_of_bds;
2773                         break;
2774                 default:
2775                         break;
2776                 }
2777
2778                 cqe =
2779                 (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring);
2780                 sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
2781         }
2782
2783         /* revert comp_ring to original state */
2784         ecore_chain_set_cons(&rxq->rx_comp_ring, sw_comp_cons, orig_cqe);
2785
2786         return bd_count;
2787 }
2788
2789 int
2790 qede_rx_descriptor_status(void *p_rxq, uint16_t offset)
2791 {
2792         uint16_t hw_bd_cons, sw_bd_cons, sw_bd_prod;
2793         uint16_t produced, consumed;
2794         struct qede_rx_queue *rxq = p_rxq;
2795
2796         if (offset > rxq->nb_rx_desc)
2797                 return -EINVAL;
2798
2799         sw_bd_cons = ecore_chain_get_cons_idx(&rxq->rx_bd_ring);
2800         sw_bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring);
2801
2802         /* find BDs used by HW from completion queue elements */
2803         hw_bd_cons = sw_bd_cons + qede_parse_fp_cqe(rxq);
2804
2805         if (hw_bd_cons < sw_bd_cons)
2806                 /* wraparound case */
2807                 consumed = (0xffff - sw_bd_cons) + hw_bd_cons;
2808         else
2809                 consumed = hw_bd_cons - sw_bd_cons;
2810
2811         if (offset <= consumed)
2812                 return RTE_ETH_RX_DESC_DONE;
2813
2814         if (sw_bd_prod < sw_bd_cons)
2815                 /* wraparound case */
2816                 produced = (0xffff - sw_bd_cons) + sw_bd_prod;
2817         else
2818                 produced = sw_bd_prod - sw_bd_cons;
2819
2820         if (offset <= produced)
2821                 return RTE_ETH_RX_DESC_AVAIL;
2822
2823         return RTE_ETH_RX_DESC_UNAVAIL;
2824 }