drivers/net/qede/qede_rxtx.c

   1 /*
   2  * Copyright (c) 2016 QLogic Corporation.
   3  * All rights reserved.
   4  * www.qlogic.com
   5  *
   6  * See LICENSE.qede_pmd for copyright and licensing details.
   7  */
   8
   9 #include "qede_rxtx.h"
  10
  11 static bool gro_disable = 1;    /* mod_param */
  12
  13 #define QEDE_FASTPATH_TX        (1 << 0)
  14 #define QEDE_FASTPATH_RX        (1 << 1)
  15
  16 static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq)
  17 {
  18         struct rte_mbuf *new_mb = NULL;
  19         struct eth_rx_bd *rx_bd;
  20         dma_addr_t mapping;
  21         uint16_t idx = rxq->sw_rx_prod & NUM_RX_BDS(rxq);
  22
  23         new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
  24         if (unlikely(!new_mb)) {
  25                 PMD_RX_LOG(ERR, rxq,
  26                            "Failed to allocate rx buffer "
  27                            "sw_rx_prod %u sw_rx_cons %u mp entries %u free %u",
  28                            idx, rxq->sw_rx_cons & NUM_RX_BDS(rxq),
  29                            rte_mempool_avail_count(rxq->mb_pool),
  30                            rte_mempool_in_use_count(rxq->mb_pool));
  31                 return -ENOMEM;
  32         }
  33         rxq->sw_rx_ring[idx].mbuf = new_mb;
  34         rxq->sw_rx_ring[idx].page_offset = 0;
  35         mapping = rte_mbuf_data_dma_addr_default(new_mb);
  36         /* Advance PROD and get BD pointer */
  37         rx_bd = (struct eth_rx_bd *)ecore_chain_produce(&rxq->rx_bd_ring);
  38         rx_bd->addr.hi = rte_cpu_to_le_32(U64_HI(mapping));
  39         rx_bd->addr.lo = rte_cpu_to_le_32(U64_LO(mapping));
  40         rxq->sw_rx_prod++;
  41         return 0;
  42 }
  43
  44 static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq)
  45 {
  46         uint16_t i;
  47
  48         if (rxq->sw_rx_ring != NULL) {
  49                 for (i = 0; i < rxq->nb_rx_desc; i++) {
  50                         if (rxq->sw_rx_ring[i].mbuf != NULL) {
  51                                 rte_pktmbuf_free(rxq->sw_rx_ring[i].mbuf);
  52                                 rxq->sw_rx_ring[i].mbuf = NULL;
  53                         }
  54                 }
  55         }
  56 }
  57
  58 void qede_rx_queue_release(void *rx_queue)
  59 {
  60         struct qede_rx_queue *rxq = rx_queue;
  61
  62         if (rxq != NULL) {
  63                 qede_rx_queue_release_mbufs(rxq);
  64                 rte_free(rxq->sw_rx_ring);
  65                 rxq->sw_rx_ring = NULL;
  66                 rte_free(rxq);
  67                 rx_queue = NULL;
  68         }
  69 }
  70
  71 static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq)
  72 {
  73         unsigned int i;
  74
  75         PMD_TX_LOG(DEBUG, txq, "releasing %u mbufs\n", txq->nb_tx_desc);
  76
  77         if (txq->sw_tx_ring) {
  78                 for (i = 0; i < txq->nb_tx_desc; i++) {
  79                         if (txq->sw_tx_ring[i].mbuf) {
  80                                 rte_pktmbuf_free(txq->sw_tx_ring[i].mbuf);
  81                                 txq->sw_tx_ring[i].mbuf = NULL;
  82                         }
  83                 }
  84         }
  85 }
  86
  87 int
  88 qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
  89                     uint16_t nb_desc, unsigned int socket_id,
  90                     const struct rte_eth_rxconf *rx_conf,
  91                     struct rte_mempool *mp)
  92 {
  93         struct qede_dev *qdev = dev->data->dev_private;
  94         struct ecore_dev *edev = &qdev->edev;
  95         struct rte_eth_dev_data *eth_data = dev->data;
  96         struct qede_rx_queue *rxq;
  97         uint16_t pkt_len = (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len;
  98         size_t size;
  99         uint16_t data_size;
 100         int rc;
 101         int i;
 102
 103         PMD_INIT_FUNC_TRACE(edev);
 104
 105         /* Note: Ring size/align is controlled by struct rte_eth_desc_lim */
 106         if (!rte_is_power_of_2(nb_desc)) {
 107                 DP_ERR(edev, "Ring size %u is not power of 2\n",
 108                           nb_desc);
 109                 return -EINVAL;
 110         }
 111
 112         /* Free memory prior to re-allocation if needed... */
 113         if (dev->data->rx_queues[queue_idx] != NULL) {
 114                 qede_rx_queue_release(dev->data->rx_queues[queue_idx]);
 115                 dev->data->rx_queues[queue_idx] = NULL;
 116         }
 117
 118         /* First allocate the rx queue data structure */
 119         rxq = rte_zmalloc_socket("qede_rx_queue", sizeof(struct qede_rx_queue),
 120                                  RTE_CACHE_LINE_SIZE, socket_id);
 121
 122         if (!rxq) {
 123                 DP_ERR(edev, "Unable to allocate memory for rxq on socket %u",
 124                           socket_id);
 125                 return -ENOMEM;
 126         }
 127
 128         rxq->qdev = qdev;
 129         rxq->mb_pool = mp;
 130         rxq->nb_rx_desc = nb_desc;
 131         rxq->queue_id = queue_idx;
 132         rxq->port_id = dev->data->port_id;
 133
 134         /* Sanity check */
 135         data_size = (uint16_t)rte_pktmbuf_data_room_size(mp) -
 136                                 RTE_PKTMBUF_HEADROOM;
 137
 138         qdev->mtu = pkt_len;
 139         rxq->rx_buf_size = data_size;
 140
 141         DP_INFO(edev, "MTU = %u ; RX buffer = %u\n",
 142                 qdev->mtu, rxq->rx_buf_size);
 143
 144         if (pkt_len > ETHER_MAX_LEN) {
 145                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
 146                 DP_NOTICE(edev, false, "jumbo frame enabled\n");
 147         } else {
 148                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
 149         }
 150
 151         /* Allocate the parallel driver ring for Rx buffers */
 152         size = sizeof(*rxq->sw_rx_ring) * rxq->nb_rx_desc;
 153         rxq->sw_rx_ring = rte_zmalloc_socket("sw_rx_ring", size,
 154                                              RTE_CACHE_LINE_SIZE, socket_id);
 155         if (!rxq->sw_rx_ring) {
 156                 DP_NOTICE(edev, false,
 157                           "Unable to alloc memory for sw_rx_ring on socket %u\n",
 158                           socket_id);
 159                 rte_free(rxq);
 160                 rxq = NULL;
 161                 return -ENOMEM;
 162         }
 163
 164         /* Allocate FW Rx ring  */
 165         rc = qdev->ops->common->chain_alloc(edev,
 166                                             ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
 167                                             ECORE_CHAIN_MODE_NEXT_PTR,
 168                                             ECORE_CHAIN_CNT_TYPE_U16,
 169                                             rxq->nb_rx_desc,
 170                                             sizeof(struct eth_rx_bd),
 171                                             &rxq->rx_bd_ring);
 172
 173         if (rc != ECORE_SUCCESS) {
 174                 DP_NOTICE(edev, false,
 175                           "Unable to alloc memory for rxbd ring on socket %u\n",
 176                           socket_id);
 177                 rte_free(rxq->sw_rx_ring);
 178                 rxq->sw_rx_ring = NULL;
 179                 rte_free(rxq);
 180                 rxq = NULL;
 181                 return -ENOMEM;
 182         }
 183
 184         /* Allocate FW completion ring */
 185         rc = qdev->ops->common->chain_alloc(edev,
 186                                             ECORE_CHAIN_USE_TO_CONSUME,
 187                                             ECORE_CHAIN_MODE_PBL,
 188                                             ECORE_CHAIN_CNT_TYPE_U16,
 189                                             rxq->nb_rx_desc,
 190                                             sizeof(union eth_rx_cqe),
 191                                             &rxq->rx_comp_ring);
 192
 193         if (rc != ECORE_SUCCESS) {
 194                 DP_NOTICE(edev, false,
 195                           "Unable to alloc memory for cqe ring on socket %u\n",
 196                           socket_id);
 197                 /* TBD: Freeing RX BD ring */
 198                 rte_free(rxq->sw_rx_ring);
 199                 rxq->sw_rx_ring = NULL;
 200                 rte_free(rxq);
 201                 return -ENOMEM;
 202         }
 203
 204         /* Allocate buffers for the Rx ring */
 205         for (i = 0; i < rxq->nb_rx_desc; i++) {
 206                 rc = qede_alloc_rx_buffer(rxq);
 207                 if (rc) {
 208                         DP_NOTICE(edev, false,
 209                                   "RX buffer allocation failed at idx=%d\n", i);
 210                         goto err4;
 211                 }
 212         }
 213
 214         dev->data->rx_queues[queue_idx] = rxq;
 215
 216         DP_INFO(edev, "rxq %d num_desc %u rx_buf_size=%u socket %u\n",
 217                   queue_idx, nb_desc, qdev->mtu, socket_id);
 218
 219         return 0;
 220 err4:
 221         qede_rx_queue_release(rxq);
 222         return -ENOMEM;
 223 }
 224
 225 void qede_tx_queue_release(void *tx_queue)
 226 {
 227         struct qede_tx_queue *txq = tx_queue;
 228
 229         if (txq != NULL) {
 230                 qede_tx_queue_release_mbufs(txq);
 231                 if (txq->sw_tx_ring) {
 232                         rte_free(txq->sw_tx_ring);
 233                         txq->sw_tx_ring = NULL;
 234                 }
 235                 rte_free(txq);
 236         }
 237         tx_queue = NULL;
 238 }
 239
 240 int
 241 qede_tx_queue_setup(struct rte_eth_dev *dev,
 242                     uint16_t queue_idx,
 243                     uint16_t nb_desc,
 244                     unsigned int socket_id,
 245                     const struct rte_eth_txconf *tx_conf)
 246 {
 247         struct qede_dev *qdev = dev->data->dev_private;
 248         struct ecore_dev *edev = &qdev->edev;
 249         struct qede_tx_queue *txq;
 250         int rc;
 251
 252         PMD_INIT_FUNC_TRACE(edev);
 253
 254         if (!rte_is_power_of_2(nb_desc)) {
 255                 DP_ERR(edev, "Ring size %u is not power of 2\n",
 256                        nb_desc);
 257                 return -EINVAL;
 258         }
 259
 260         /* Free memory prior to re-allocation if needed... */
 261         if (dev->data->tx_queues[queue_idx] != NULL) {
 262                 qede_tx_queue_release(dev->data->tx_queues[queue_idx]);
 263                 dev->data->tx_queues[queue_idx] = NULL;
 264         }
 265
 266         txq = rte_zmalloc_socket("qede_tx_queue", sizeof(struct qede_tx_queue),
 267                                  RTE_CACHE_LINE_SIZE, socket_id);
 268
 269         if (txq == NULL) {
 270                 DP_ERR(edev,
 271                        "Unable to allocate memory for txq on socket %u",
 272                        socket_id);
 273                 return -ENOMEM;
 274         }
 275
 276         txq->nb_tx_desc = nb_desc;
 277         txq->qdev = qdev;
 278         txq->port_id = dev->data->port_id;
 279
 280         rc = qdev->ops->common->chain_alloc(edev,
 281                                             ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
 282                                             ECORE_CHAIN_MODE_PBL,
 283                                             ECORE_CHAIN_CNT_TYPE_U16,
 284                                             txq->nb_tx_desc,
 285                                             sizeof(union eth_tx_bd_types),
 286                                             &txq->tx_pbl);
 287         if (rc != ECORE_SUCCESS) {
 288                 DP_ERR(edev,
 289                        "Unable to allocate memory for txbd ring on socket %u",
 290                        socket_id);
 291                 qede_tx_queue_release(txq);
 292                 return -ENOMEM;
 293         }
 294
 295         /* Allocate software ring */
 296         txq->sw_tx_ring = rte_zmalloc_socket("txq->sw_tx_ring",
 297                                              (sizeof(struct qede_tx_entry) *
 298                                               txq->nb_tx_desc),
 299                                              RTE_CACHE_LINE_SIZE, socket_id);
 300
 301         if (!txq->sw_tx_ring) {
 302                 DP_ERR(edev,
 303                        "Unable to allocate memory for txbd ring on socket %u",
 304                        socket_id);
 305                 qede_tx_queue_release(txq);
 306                 return -ENOMEM;
 307         }
 308
 309         txq->queue_id = queue_idx;
 310
 311         txq->nb_tx_avail = txq->nb_tx_desc;
 312
 313         txq->tx_free_thresh =
 314             tx_conf->tx_free_thresh ? tx_conf->tx_free_thresh :
 315             (txq->nb_tx_desc - QEDE_DEFAULT_TX_FREE_THRESH);
 316
 317         dev->data->tx_queues[queue_idx] = txq;
 318
 319         DP_INFO(edev,
 320                   "txq %u num_desc %u tx_free_thresh %u socket %u\n",
 321                   queue_idx, nb_desc, txq->tx_free_thresh, socket_id);
 322
 323         return 0;
 324 }
 325
 326 /* This function inits fp content and resets the SB, RXQ and TXQ arrays */
 327 static void qede_init_fp(struct qede_dev *qdev)
 328 {
 329         struct qede_fastpath *fp;
 330         uint8_t i, rss_id, tc;
 331         int fp_rx = qdev->fp_num_rx, rxq = 0, txq = 0;
 332
 333         memset((void *)qdev->fp_array, 0, (QEDE_QUEUE_CNT(qdev) *
 334                                            sizeof(*qdev->fp_array)));
 335         memset((void *)qdev->sb_array, 0, (QEDE_QUEUE_CNT(qdev) *
 336                                            sizeof(*qdev->sb_array)));
 337         for_each_queue(i) {
 338                 fp = &qdev->fp_array[i];
 339                 if (fp_rx) {
 340                         fp->type = QEDE_FASTPATH_RX;
 341                         fp_rx--;
 342                 } else{
 343                         fp->type = QEDE_FASTPATH_TX;
 344                 }
 345                 fp->qdev = qdev;
 346                 fp->id = i;
 347                 fp->sb_info = &qdev->sb_array[i];
 348                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", "qdev", i);
 349         }
 350
 351         qdev->gro_disable = gro_disable;
 352 }
 353
 354 void qede_free_fp_arrays(struct qede_dev *qdev)
 355 {
 356         /* It asseumes qede_free_mem_load() is called before */
 357         if (qdev->fp_array != NULL) {
 358                 rte_free(qdev->fp_array);
 359                 qdev->fp_array = NULL;
 360         }
 361
 362         if (qdev->sb_array != NULL) {
 363                 rte_free(qdev->sb_array);
 364                 qdev->sb_array = NULL;
 365         }
 366 }
 367
 368 int qede_alloc_fp_array(struct qede_dev *qdev)
 369 {
 370         struct qede_fastpath *fp;
 371         struct ecore_dev *edev = &qdev->edev;
 372         int i;
 373
 374         qdev->fp_array = rte_calloc("fp", QEDE_QUEUE_CNT(qdev),
 375                                     sizeof(*qdev->fp_array),
 376                                     RTE_CACHE_LINE_SIZE);
 377
 378         if (!qdev->fp_array) {
 379                 DP_ERR(edev, "fp array allocation failed\n");
 380                 return -ENOMEM;
 381         }
 382
 383         qdev->sb_array = rte_calloc("sb", QEDE_QUEUE_CNT(qdev),
 384                                     sizeof(*qdev->sb_array),
 385                                     RTE_CACHE_LINE_SIZE);
 386
 387         if (!qdev->sb_array) {
 388                 DP_ERR(edev, "sb array allocation failed\n");
 389                 rte_free(qdev->fp_array);
 390                 return -ENOMEM;
 391         }
 392
 393         return 0;
 394 }
 395
 396 /* This function allocates fast-path status block memory */
 397 static int
 398 qede_alloc_mem_sb(struct qede_dev *qdev, struct ecore_sb_info *sb_info,
 399                   uint16_t sb_id)
 400 {
 401         struct ecore_dev *edev = &qdev->edev;
 402         struct status_block *sb_virt;
 403         dma_addr_t sb_phys;
 404         int rc;
 405
 406         sb_virt = OSAL_DMA_ALLOC_COHERENT(edev, &sb_phys, sizeof(*sb_virt));
 407
 408         if (!sb_virt) {
 409                 DP_ERR(edev, "Status block allocation failed\n");
 410                 return -ENOMEM;
 411         }
 412
 413         rc = qdev->ops->common->sb_init(edev, sb_info,
 414                                         sb_virt, sb_phys, sb_id,
 415                                         QED_SB_TYPE_L2_QUEUE);
 416         if (rc) {
 417                 DP_ERR(edev, "Status block initialization failed\n");
 418                 /* TBD: No dma_free_coherent possible */
 419                 return rc;
 420         }
 421
 422         return 0;
 423 }
 424
 425 int qede_alloc_fp_resc(struct qede_dev *qdev)
 426 {
 427         struct qede_fastpath *fp;
 428         int rc, i;
 429
 430         if (qdev->fp_array)
 431                 qede_free_fp_arrays(qdev);
 432
 433         rc = qede_alloc_fp_array(qdev);
 434         if (rc != 0)
 435                 return rc;
 436
 437         qede_init_fp(qdev);
 438
 439         for (i = 0; i < QEDE_QUEUE_CNT(qdev); i++) {
 440                 fp = &qdev->fp_array[i];
 441                 if (qede_alloc_mem_sb(qdev, fp->sb_info, i)) {
 442                         qede_free_fp_arrays(qdev);
 443                         return -ENOMEM;
 444                 }
 445         }
 446
 447         return 0;
 448 }
 449
 450 void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev)
 451 {
 452         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
 453
 454         qede_free_mem_load(eth_dev);
 455         qede_free_fp_arrays(qdev);
 456 }
 457
 458 static inline void
 459 qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq)
 460 {
 461         uint16_t bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring);
 462         uint16_t cqe_prod = ecore_chain_get_prod_idx(&rxq->rx_comp_ring);
 463         struct eth_rx_prod_data rx_prods = { 0 };
 464
 465         /* Update producers */
 466         rx_prods.bd_prod = rte_cpu_to_le_16(bd_prod);
 467         rx_prods.cqe_prod = rte_cpu_to_le_16(cqe_prod);
 468
 469         /* Make sure that the BD and SGE data is updated before updating the
 470          * producers since FW might read the BD/SGE right after the producer
 471          * is updated.
 472          */
 473         rte_wmb();
 474
 475         internal_ram_wr(rxq->hw_rxq_prod_addr, sizeof(rx_prods),
 476                         (uint32_t *)&rx_prods);
 477
 478         /* mmiowb is needed to synchronize doorbell writes from more than one
 479          * processor. It guarantees that the write arrives to the device before
 480          * the napi lock is released and another qede_poll is called (possibly
 481          * on another CPU). Without this barrier, the next doorbell can bypass
 482          * this doorbell. This is applicable to IA64/Altix systems.
 483          */
 484         rte_wmb();
 485
 486         PMD_RX_LOG(DEBUG, rxq, "bd_prod %u  cqe_prod %u\n", bd_prod, cqe_prod);
 487 }
 488
 489 static inline uint32_t
 490 qede_rxfh_indir_default(uint32_t index, uint32_t n_rx_rings)
 491 {
 492         return index % n_rx_rings;
 493 }
 494
 495 static void qede_prandom_bytes(uint32_t *buff, size_t bytes)
 496 {
 497         unsigned int i;
 498
 499         srand((unsigned int)time(NULL));
 500
 501         for (i = 0; i < ECORE_RSS_KEY_SIZE; i++)
 502                 buff[i] = rand();
 503 }
 504
 505 static int
 506 qede_config_rss(struct rte_eth_dev *eth_dev,
 507                 struct qed_update_vport_rss_params *rss_params)
 508 {
 509         struct rte_eth_rss_conf rss_conf;
 510         enum rte_eth_rx_mq_mode mode = eth_dev->data->dev_conf.rxmode.mq_mode;
 511         struct qede_dev *qdev = eth_dev->data->dev_private;
 512         struct ecore_dev *edev = &qdev->edev;
 513         uint8_t rss_caps;
 514         unsigned int i;
 515         uint64_t hf;
 516         uint32_t *key;
 517
 518         rss_conf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 519         key = (uint32_t *)rss_conf.rss_key;
 520         hf = rss_conf.rss_hf;
 521         PMD_INIT_FUNC_TRACE(edev);
 522
 523         /* Check if RSS conditions are met.
 524          * Note: Even though its meaningless to enable RSS with one queue, it
 525          * could be used to produce RSS Hash, so skipping that check.
 526          */
 527
 528         if (!(mode & ETH_MQ_RX_RSS)) {
 529                 DP_INFO(edev, "RSS flag is not set\n");
 530                 return -EINVAL;
 531         }
 532
 533         DP_INFO(edev, "RSS flag is set\n");
 534
 535         if (rss_conf.rss_hf == 0)
 536                 DP_NOTICE(edev, false, "RSS hash function = 0, disables RSS\n");
 537
 538         if (rss_conf.rss_key != NULL)
 539                 memcpy(qdev->rss_params.rss_key, rss_conf.rss_key,
 540                        rss_conf.rss_key_len);
 541
 542         memset(rss_params, 0, sizeof(*rss_params));
 543
 544         for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++)
 545                 rss_params->rss_ind_table[i] = qede_rxfh_indir_default(i,
 546                                                         QEDE_RSS_CNT(qdev));
 547
 548         /* key and protocols */
 549         if (rss_conf.rss_key == NULL)
 550                 qede_prandom_bytes(rss_params->rss_key,
 551                                    sizeof(rss_params->rss_key));
 552         else
 553                 memcpy(rss_params->rss_key, rss_conf.rss_key,
 554                        rss_conf.rss_key_len);
 555
 556         rss_caps = 0;
 557         rss_caps |= (hf & ETH_RSS_IPV4)              ? ECORE_RSS_IPV4 : 0;
 558         rss_caps |= (hf & ETH_RSS_IPV6)              ? ECORE_RSS_IPV6 : 0;
 559         rss_caps |= (hf & ETH_RSS_IPV6_EX)           ? ECORE_RSS_IPV6 : 0;
 560         rss_caps |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? ECORE_RSS_IPV4_TCP : 0;
 561         rss_caps |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? ECORE_RSS_IPV6_TCP : 0;
 562         rss_caps |= (hf & ETH_RSS_IPV6_TCP_EX)       ? ECORE_RSS_IPV6_TCP : 0;
 563
 564         rss_params->rss_caps = rss_caps;
 565
 566         DP_INFO(edev, "RSS check passes\n");
 567
 568         return 0;
 569 }
 570
 571 static int qede_start_queues(struct rte_eth_dev *eth_dev, bool clear_stats)
 572 {
 573         struct qede_dev *qdev = eth_dev->data->dev_private;
 574         struct ecore_dev *edev = &qdev->edev;
 575         struct qed_update_vport_rss_params *rss_params = &qdev->rss_params;
 576         struct qed_dev_info *qed_info = &qdev->dev_info.common;
 577         struct qed_update_vport_params vport_update_params;
 578         struct qede_tx_queue *txq;
 579         struct qede_fastpath *fp;
 580         dma_addr_t p_phys_table;
 581         int txq_index;
 582         uint16_t page_cnt;
 583         int vlan_removal_en = 1;
 584         int rc, tc, i;
 585
 586         for_each_queue(i) {
 587                 fp = &qdev->fp_array[i];
 588                 if (fp->type & QEDE_FASTPATH_RX) {
 589                         p_phys_table = ecore_chain_get_pbl_phys(&fp->rxq->
 590                                                                 rx_comp_ring);
 591                         page_cnt = ecore_chain_get_page_cnt(&fp->rxq->
 592                                                                 rx_comp_ring);
 593
 594                         ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
 595
 596                         rc = qdev->ops->q_rx_start(edev, i, fp->rxq->queue_id,
 597                                            0,
 598                                            fp->sb_info->igu_sb_id,
 599                                            RX_PI,
 600                                            fp->rxq->rx_buf_size,
 601                                            fp->rxq->rx_bd_ring.p_phys_addr,
 602                                            p_phys_table,
 603                                            page_cnt,
 604                                            &fp->rxq->hw_rxq_prod_addr);
 605                         if (rc) {
 606                                 DP_ERR(edev, "Start rxq #%d failed %d\n",
 607                                        fp->rxq->queue_id, rc);
 608                                 return rc;
 609                         }
 610
 611                         fp->rxq->hw_cons_ptr =
 612                                         &fp->sb_info->sb_virt->pi_array[RX_PI];
 613
 614                         qede_update_rx_prod(qdev, fp->rxq);
 615                 }
 616
 617                 if (!(fp->type & QEDE_FASTPATH_TX))
 618                         continue;
 619                 for (tc = 0; tc < qdev->num_tc; tc++) {
 620                         txq = fp->txqs[tc];
 621                         txq_index = tc * QEDE_RSS_CNT(qdev) + i;
 622
 623                         p_phys_table = ecore_chain_get_pbl_phys(&txq->tx_pbl);
 624                         page_cnt = ecore_chain_get_page_cnt(&txq->tx_pbl);
 625                         rc = qdev->ops->q_tx_start(edev, i, txq->queue_id,
 626                                                    0,
 627                                                    fp->sb_info->igu_sb_id,
 628                                                    TX_PI(tc),
 629                                                    p_phys_table, page_cnt,
 630                                                    &txq->doorbell_addr);
 631                         if (rc) {
 632                                 DP_ERR(edev, "Start txq %u failed %d\n",
 633                                        txq_index, rc);
 634                                 return rc;
 635                         }
 636
 637                         txq->hw_cons_ptr =
 638                             &fp->sb_info->sb_virt->pi_array[TX_PI(tc)];
 639                         SET_FIELD(txq->tx_db.data.params,
 640                                   ETH_DB_DATA_DEST, DB_DEST_XCM);
 641                         SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD,
 642                                   DB_AGG_CMD_SET);
 643                         SET_FIELD(txq->tx_db.data.params,
 644                                   ETH_DB_DATA_AGG_VAL_SEL,
 645                                   DQ_XCM_ETH_TX_BD_PROD_CMD);
 646
 647                         txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
 648                 }
 649         }
 650
 651         /* Prepare and send the vport enable */
 652         memset(&vport_update_params, 0, sizeof(vport_update_params));
 653         /* Update MTU via vport update */
 654         vport_update_params.mtu = qdev->mtu;
 655         vport_update_params.vport_id = 0;
 656         vport_update_params.update_vport_active_flg = 1;
 657         vport_update_params.vport_active_flg = 1;
 658
 659         /* @DPDK */
 660         if (qed_info->mf_mode == MF_NPAR && qed_info->tx_switching) {
 661                 /* TBD: Check SRIOV enabled for VF */
 662                 vport_update_params.update_tx_switching_flg = 1;
 663                 vport_update_params.tx_switching_flg = 1;
 664         }
 665
 666         if (!qede_config_rss(eth_dev, rss_params)) {
 667                 vport_update_params.update_rss_flg = 1;
 668
 669                 qdev->rss_enabled = 1;
 670                 DP_INFO(edev, "Updating RSS flag\n");
 671         } else {
 672                 qdev->rss_enabled = 0;
 673                 DP_INFO(edev, "Not Updating RSS flag\n");
 674         }
 675
 676         rte_memcpy(&vport_update_params.rss_params, rss_params,
 677                sizeof(*rss_params));
 678
 679         rc = qdev->ops->vport_update(edev, &vport_update_params);
 680         if (rc) {
 681                 DP_ERR(edev, "Update V-PORT failed %d\n", rc);
 682                 return rc;
 683         }
 684
 685         return 0;
 686 }
 687
 688 #ifdef ENC_SUPPORTED
 689 static bool qede_tunn_exist(uint16_t flag)
 690 {
 691         return !!((PARSING_AND_ERR_FLAGS_TUNNELEXIST_MASK <<
 692                     PARSING_AND_ERR_FLAGS_TUNNELEXIST_SHIFT) & flag);
 693 }
 694
 695 static inline uint8_t qede_check_tunn_csum(uint16_t flag)
 696 {
 697         uint8_t tcsum = 0;
 698         uint16_t csum_flag = 0;
 699
 700         if ((PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_MASK <<
 701              PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_SHIFT) & flag)
 702                 csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_MASK <<
 703                     PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT;
 704
 705         if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK <<
 706              PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) {
 707                 csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK <<
 708                     PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT;
 709                 tcsum = QEDE_TUNN_CSUM_UNNECESSARY;
 710         }
 711
 712         csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_MASK <<
 713             PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_SHIFT |
 714             PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK <<
 715             PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT;
 716
 717         if (csum_flag & flag)
 718                 return QEDE_CSUM_ERROR;
 719
 720         return QEDE_CSUM_UNNECESSARY | tcsum;
 721 }
 722 #else
 723 static inline uint8_t qede_tunn_exist(uint16_t flag)
 724 {
 725         return 0;
 726 }
 727
 728 static inline uint8_t qede_check_tunn_csum(uint16_t flag)
 729 {
 730         return 0;
 731 }
 732 #endif
 733
 734 static inline uint8_t qede_check_notunn_csum(uint16_t flag)
 735 {
 736         uint8_t csum = 0;
 737         uint16_t csum_flag = 0;
 738
 739         if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK <<
 740              PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) {
 741                 csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK <<
 742                     PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT;
 743                 csum = QEDE_CSUM_UNNECESSARY;
 744         }
 745
 746         csum_flag |= PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK <<
 747             PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT;
 748
 749         if (csum_flag & flag)
 750                 return QEDE_CSUM_ERROR;
 751
 752         return csum;
 753 }
 754
 755 static inline uint8_t qede_check_csum(uint16_t flag)
 756 {
 757         if (likely(!qede_tunn_exist(flag)))
 758                 return qede_check_notunn_csum(flag);
 759         else
 760                 return qede_check_tunn_csum(flag);
 761 }
 762
 763 static inline void qede_rx_bd_ring_consume(struct qede_rx_queue *rxq)
 764 {
 765         ecore_chain_consume(&rxq->rx_bd_ring);
 766         rxq->sw_rx_cons++;
 767 }
 768
 769 static inline void
 770 qede_reuse_page(struct qede_dev *qdev,
 771                 struct qede_rx_queue *rxq, struct qede_rx_entry *curr_cons)
 772 {
 773         struct eth_rx_bd *rx_bd_prod = ecore_chain_produce(&rxq->rx_bd_ring);
 774         uint16_t idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
 775         struct qede_rx_entry *curr_prod;
 776         dma_addr_t new_mapping;
 777
 778         curr_prod = &rxq->sw_rx_ring[idx];
 779         *curr_prod = *curr_cons;
 780
 781         new_mapping = rte_mbuf_data_dma_addr_default(curr_prod->mbuf) +
 782                       curr_prod->page_offset;
 783
 784         rx_bd_prod->addr.hi = rte_cpu_to_le_32(U64_HI(new_mapping));
 785         rx_bd_prod->addr.lo = rte_cpu_to_le_32(U64_LO(new_mapping));
 786
 787         rxq->sw_rx_prod++;
 788 }
 789
 790 static inline void
 791 qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq,
 792                         struct qede_dev *qdev, uint8_t count)
 793 {
 794         struct qede_rx_entry *curr_cons;
 795
 796         for (; count > 0; count--) {
 797                 curr_cons = &rxq->sw_rx_ring[rxq->sw_rx_cons & NUM_RX_BDS(rxq)];
 798                 qede_reuse_page(qdev, rxq, curr_cons);
 799                 qede_rx_bd_ring_consume(rxq);
 800         }
 801 }
 802
 803 static inline uint32_t qede_rx_cqe_to_pkt_type(uint16_t flags)
 804 {
 805         uint32_t p_type;
 806         /* TBD - L4 indications needed ? */
 807         uint16_t protocol = ((PARSING_AND_ERR_FLAGS_L3TYPE_MASK <<
 808                               PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT) & flags);
 809
 810         /* protocol = 3 means LLC/SNAP over Ethernet */
 811         if (unlikely(protocol == 0 || protocol == 3))
 812                 p_type = RTE_PTYPE_UNKNOWN;
 813         else if (protocol == 1)
 814                 p_type = RTE_PTYPE_L3_IPV4;
 815         else if (protocol == 2)
 816                 p_type = RTE_PTYPE_L3_IPV6;
 817
 818         return RTE_PTYPE_L2_ETHER | p_type;
 819 }
 820
 821 uint16_t
 822 qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 823 {
 824         struct qede_rx_queue *rxq = p_rxq;
 825         struct qede_dev *qdev = rxq->qdev;
 826         struct ecore_dev *edev = &qdev->edev;
 827         struct qede_fastpath *fp = &qdev->fp_array[rxq->queue_id];
 828         uint16_t hw_comp_cons, sw_comp_cons, sw_rx_index;
 829         uint16_t rx_pkt = 0;
 830         union eth_rx_cqe *cqe;
 831         struct eth_fast_path_rx_reg_cqe *fp_cqe;
 832         register struct rte_mbuf *rx_mb = NULL;
 833         enum eth_rx_cqe_type cqe_type;
 834         uint16_t len, pad;
 835         uint16_t preload_idx;
 836         uint8_t csum_flag;
 837         uint16_t parse_flag;
 838         enum rss_hash_type htype;
 839
 840         hw_comp_cons = rte_le_to_cpu_16(*rxq->hw_cons_ptr);
 841         sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
 842
 843         rte_rmb();
 844
 845         if (hw_comp_cons == sw_comp_cons)
 846                 return 0;
 847
 848         while (sw_comp_cons != hw_comp_cons) {
 849                 /* Get the CQE from the completion ring */
 850                 cqe =
 851                     (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring);
 852                 cqe_type = cqe->fast_path_regular.type;
 853
 854                 if (unlikely(cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH)) {
 855                         PMD_RX_LOG(DEBUG, rxq, "Got a slowath CQE\n");
 856
 857                         qdev->ops->eth_cqe_completion(edev, fp->id,
 858                                 (struct eth_slow_path_rx_cqe *)cqe);
 859                         goto next_cqe;
 860                 }
 861
 862                 /* Get the data from the SW ring */
 863                 sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
 864                 rx_mb = rxq->sw_rx_ring[sw_rx_index].mbuf;
 865                 assert(rx_mb != NULL);
 866
 867                 /* non GRO */
 868                 fp_cqe = &cqe->fast_path_regular;
 869
 870                 len = rte_le_to_cpu_16(fp_cqe->len_on_first_bd);
 871                 pad = fp_cqe->placement_offset;
 872                 assert((len + pad) <= rx_mb->buf_len);
 873
 874                 PMD_RX_LOG(DEBUG, rxq,
 875                            "CQE type = 0x%x, flags = 0x%x, vlan = 0x%x"
 876                            " len = %u, parsing_flags = %d\n",
 877                            cqe_type, fp_cqe->bitfields,
 878                            rte_le_to_cpu_16(fp_cqe->vlan_tag),
 879                            len, rte_le_to_cpu_16(fp_cqe->pars_flags.flags));
 880
 881                 /* If this is an error packet then drop it */
 882                 parse_flag =
 883                     rte_le_to_cpu_16(cqe->fast_path_regular.pars_flags.flags);
 884                 csum_flag = qede_check_csum(parse_flag);
 885                 if (unlikely(csum_flag == QEDE_CSUM_ERROR)) {
 886                         PMD_RX_LOG(ERR, rxq,
 887                                    "CQE in CONS = %u has error, flags = 0x%x "
 888                                    "dropping incoming packet\n",
 889                                    sw_comp_cons, parse_flag);
 890                         rxq->rx_hw_errors++;
 891                         qede_recycle_rx_bd_ring(rxq, qdev, fp_cqe->bd_num);
 892                         goto next_cqe;
 893                 }
 894
 895                 if (unlikely(qede_alloc_rx_buffer(rxq) != 0)) {
 896                         PMD_RX_LOG(ERR, rxq,
 897                                    "New buffer allocation failed,"
 898                                    "dropping incoming packet\n");
 899                         qede_recycle_rx_bd_ring(rxq, qdev, fp_cqe->bd_num);
 900                         rte_eth_devices[rxq->port_id].
 901                             data->rx_mbuf_alloc_failed++;
 902                         rxq->rx_alloc_errors++;
 903                         break;
 904                 }
 905
 906                 qede_rx_bd_ring_consume(rxq);
 907
 908                 /* Prefetch next mbuf while processing current one. */
 909                 preload_idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
 910                 rte_prefetch0(rxq->sw_rx_ring[preload_idx].mbuf);
 911
 912                 if (fp_cqe->bd_num != 1)
 913                         PMD_RX_LOG(DEBUG, rxq,
 914                                    "Jumbo-over-BD packet not supported\n");
 915
 916                 /* Update MBUF fields */
 917                 rx_mb->ol_flags = 0;
 918                 rx_mb->data_off = pad + RTE_PKTMBUF_HEADROOM;
 919                 rx_mb->nb_segs = 1;
 920                 rx_mb->data_len = len;
 921                 rx_mb->pkt_len = len;
 922                 rx_mb->port = rxq->port_id;
 923                 rx_mb->packet_type = qede_rx_cqe_to_pkt_type(parse_flag);
 924
 925                 htype = (uint8_t)GET_FIELD(fp_cqe->bitfields,
 926                                 ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE);
 927                 if (qdev->rss_enabled && htype) {
 928                         rx_mb->ol_flags |= PKT_RX_RSS_HASH;
 929                         rx_mb->hash.rss = rte_le_to_cpu_32(fp_cqe->rss_hash);
 930                         PMD_RX_LOG(DEBUG, rxq, "Hash result 0x%x\n",
 931                                    rx_mb->hash.rss);
 932                 }
 933
 934                 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
 935
 936                 if (CQE_HAS_VLAN(parse_flag)) {
 937                         rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag);
 938                         rx_mb->ol_flags |= PKT_RX_VLAN_PKT;
 939                 }
 940
 941                 if (CQE_HAS_OUTER_VLAN(parse_flag)) {
 942                         /* FW does not provide indication of Outer VLAN tag,
 943                          * which is always stripped, so vlan_tci_outer is set
 944                          * to 0. Here vlan_tag represents inner VLAN tag.
 945                          */
 946                         rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag);
 947                         rx_mb->ol_flags |= PKT_RX_QINQ_PKT;
 948                         rx_mb->vlan_tci_outer = 0;
 949                 }
 950
 951                 rx_pkts[rx_pkt] = rx_mb;
 952                 rx_pkt++;
 953 next_cqe:
 954                 ecore_chain_recycle_consumed(&rxq->rx_comp_ring);
 955                 sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
 956                 if (rx_pkt == nb_pkts) {
 957                         PMD_RX_LOG(DEBUG, rxq,
 958                                    "Budget reached nb_pkts=%u received=%u\n",
 959                                    rx_pkt, nb_pkts);
 960                         break;
 961                 }
 962         }
 963
 964         qede_update_rx_prod(qdev, rxq);
 965
 966         PMD_RX_LOG(DEBUG, rxq, "rx_pkts=%u core=%d\n", rx_pkt, rte_lcore_id());
 967
 968         return rx_pkt;
 969 }
 970
 971 static inline int
 972 qede_free_tx_pkt(struct ecore_dev *edev, struct qede_tx_queue *txq)
 973 {
 974         uint16_t idx = TX_CONS(txq);
 975         struct eth_tx_bd *tx_data_bd;
 976         struct rte_mbuf *mbuf = txq->sw_tx_ring[idx].mbuf;
 977
 978         if (unlikely(!mbuf)) {
 979                 PMD_TX_LOG(ERR, txq,
 980                            "null mbuf nb_tx_desc %u nb_tx_avail %u "
 981                            "sw_tx_cons %u sw_tx_prod %u\n",
 982                            txq->nb_tx_desc, txq->nb_tx_avail, idx,
 983                            TX_PROD(txq));
 984                 return -1;
 985         }
 986
 987         /* Free now */
 988         rte_pktmbuf_free_seg(mbuf);
 989         txq->sw_tx_ring[idx].mbuf = NULL;
 990         ecore_chain_consume(&txq->tx_pbl);
 991         txq->nb_tx_avail++;
 992
 993         return 0;
 994 }
 995
 996 static inline uint16_t
 997 qede_process_tx_compl(struct ecore_dev *edev, struct qede_tx_queue *txq)
 998 {
 999         uint16_t tx_compl = 0;
1000         uint16_t hw_bd_cons;
1001         int rc;
1002
1003         hw_bd_cons = rte_le_to_cpu_16(*txq->hw_cons_ptr);
1004         rte_compiler_barrier();
1005
1006         while (hw_bd_cons != ecore_chain_get_cons_idx(&txq->tx_pbl)) {
1007                 rc = qede_free_tx_pkt(edev, txq);
1008                 if (rc) {
1009                         DP_NOTICE(edev, false,
1010                                   "hw_bd_cons = %d, chain_cons=%d\n",
1011                                   hw_bd_cons,
1012                                   ecore_chain_get_cons_idx(&txq->tx_pbl));
1013                         break;
1014                 }
1015                 txq->sw_tx_cons++;      /* Making TXD available */
1016                 tx_compl++;
1017         }
1018
1019         PMD_TX_LOG(DEBUG, txq, "Tx compl %u sw_tx_cons %u avail %u\n",
1020                    tx_compl, txq->sw_tx_cons, txq->nb_tx_avail);
1021         return tx_compl;
1022 }
1023
1024 uint16_t
1025 qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1026 {
1027         struct qede_tx_queue *txq = p_txq;
1028         struct qede_dev *qdev = txq->qdev;
1029         struct ecore_dev *edev = &qdev->edev;
1030         struct qede_fastpath *fp;
1031         struct eth_tx_1st_bd *first_bd;
1032         uint16_t nb_tx_pkts;
1033         uint16_t nb_pkt_sent = 0;
1034         uint16_t bd_prod;
1035         uint16_t idx;
1036         uint16_t tx_count;
1037
1038         fp = &qdev->fp_array[QEDE_RSS_COUNT(qdev) + txq->queue_id];
1039
1040         if (unlikely(txq->nb_tx_avail < txq->tx_free_thresh)) {
1041                 PMD_TX_LOG(DEBUG, txq, "send=%u avail=%u free_thresh=%u\n",
1042                            nb_pkts, txq->nb_tx_avail, txq->tx_free_thresh);
1043                 (void)qede_process_tx_compl(edev, txq);
1044         }
1045
1046         nb_tx_pkts = RTE_MIN(nb_pkts, (txq->nb_tx_avail / MAX_NUM_TX_BDS));
1047         if (unlikely(nb_tx_pkts == 0)) {
1048                 PMD_TX_LOG(DEBUG, txq, "Out of BDs nb_pkts=%u avail=%u\n",
1049                            nb_pkts, txq->nb_tx_avail);
1050                 return 0;
1051         }
1052
1053         tx_count = nb_tx_pkts;
1054         while (nb_tx_pkts--) {
1055                 /* Fill the entry in the SW ring and the BDs in the FW ring */
1056                 idx = TX_PROD(txq);
1057                 struct rte_mbuf *mbuf = *tx_pkts++;
1058                 txq->sw_tx_ring[idx].mbuf = mbuf;
1059                 first_bd = (struct eth_tx_1st_bd *)
1060                     ecore_chain_produce(&txq->tx_pbl);
1061                 first_bd->data.bd_flags.bitfields =
1062                     1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT;
1063                 /* Map MBUF linear data for DMA and set in the first BD */
1064                 QEDE_BD_SET_ADDR_LEN(first_bd, rte_mbuf_data_dma_addr(mbuf),
1065                                      mbuf->data_len);
1066
1067                 /* Descriptor based VLAN insertion */
1068                 if (mbuf->ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) {
1069                         first_bd->data.vlan = rte_cpu_to_le_16(mbuf->vlan_tci);
1070                         first_bd->data.bd_flags.bitfields |=
1071                             1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT;
1072                 }
1073
1074                 /* Offload the IP checksum in the hardware */
1075                 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) {
1076                         first_bd->data.bd_flags.bitfields |=
1077                             1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT;
1078                 }
1079
1080                 /* L4 checksum offload (tcp or udp) */
1081                 if (mbuf->ol_flags & (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
1082                         first_bd->data.bd_flags.bitfields |=
1083                             1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT;
1084                         /* IPv6 + extn. -> later */
1085                 }
1086                 first_bd->data.nbds = MAX_NUM_TX_BDS;
1087                 txq->sw_tx_prod++;
1088                 rte_prefetch0(txq->sw_tx_ring[TX_PROD(txq)].mbuf);
1089                 txq->nb_tx_avail--;
1090                 bd_prod =
1091                     rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl));
1092                 nb_pkt_sent++;
1093         }
1094
1095         /* Write value of prod idx into bd_prod */
1096         txq->tx_db.data.bd_prod = bd_prod;
1097         rte_wmb();
1098         rte_compiler_barrier();
1099         DIRECT_REG_WR(edev, txq->doorbell_addr, txq->tx_db.raw);
1100         rte_wmb();
1101
1102         /* Check again for Tx completions */
1103         (void)qede_process_tx_compl(edev, txq);
1104
1105         PMD_TX_LOG(DEBUG, txq, "to_send=%u can_send=%u sent=%u core=%d\n",
1106                    nb_pkts, tx_count, nb_pkt_sent, rte_lcore_id());
1107
1108         return nb_pkt_sent;
1109 }
1110
1111 static void qede_init_fp_queue(struct rte_eth_dev *eth_dev)
1112 {
1113         struct qede_dev *qdev = eth_dev->data->dev_private;
1114         struct qede_fastpath *fp;
1115         uint8_t i, rss_id, txq_index, tc;
1116         int rxq = 0, txq = 0;
1117
1118         for_each_queue(i) {
1119                 fp = &qdev->fp_array[i];
1120                 if (fp->type & QEDE_FASTPATH_RX) {
1121                         fp->rxq = eth_dev->data->rx_queues[i];
1122                         fp->rxq->queue_id = rxq++;
1123                 }
1124
1125                 if (fp->type & QEDE_FASTPATH_TX) {
1126                         for (tc = 0; tc < qdev->num_tc; tc++) {
1127                                 txq_index = tc * QEDE_TSS_CNT(qdev) + txq;
1128                                 fp->txqs[tc] =
1129                                         eth_dev->data->tx_queues[txq_index];
1130                                 fp->txqs[tc]->queue_id = txq_index;
1131                         }
1132                         txq++;
1133                 }
1134         }
1135 }
1136
1137 int qede_dev_start(struct rte_eth_dev *eth_dev)
1138 {
1139         struct qede_dev *qdev = eth_dev->data->dev_private;
1140         struct ecore_dev *edev = &qdev->edev;
1141         struct qed_link_output link_output;
1142         struct qede_fastpath *fp;
1143         int rc, i;
1144
1145         DP_INFO(edev, "Device state is %d\n", qdev->state);
1146
1147         if (qdev->state == QEDE_DEV_START) {
1148                 DP_INFO(edev, "Port is already started\n");
1149                 return 0;
1150         }
1151
1152         if (qdev->state == QEDE_DEV_CONFIG)
1153                 qede_init_fp_queue(eth_dev);
1154
1155         rc = qede_start_queues(eth_dev, true);
1156         if (rc) {
1157                 DP_ERR(edev, "Failed to start queues\n");
1158                 /* TBD: free */
1159                 return rc;
1160         }
1161
1162         /* Bring-up the link */
1163         qede_dev_set_link_state(eth_dev, true);
1164
1165         /* Reset ring */
1166         if (qede_reset_fp_rings(qdev))
1167                 return -ENOMEM;
1168
1169         /* Start/resume traffic */
1170         qdev->ops->fastpath_start(edev);
1171
1172         qdev->state = QEDE_DEV_START;
1173
1174         DP_INFO(edev, "dev_state is QEDE_DEV_START\n");
1175
1176         return 0;
1177 }
1178
1179 static int qede_drain_txq(struct qede_dev *qdev,
1180                           struct qede_tx_queue *txq, bool allow_drain)
1181 {
1182         struct ecore_dev *edev = &qdev->edev;
1183         int rc, cnt = 1000;
1184
1185         while (txq->sw_tx_cons != txq->sw_tx_prod) {
1186                 qede_process_tx_compl(edev, txq);
1187                 if (!cnt) {
1188                         if (allow_drain) {
1189                                 DP_NOTICE(edev, false,
1190                                           "Tx queue[%u] is stuck,"
1191                                           "requesting MCP to drain\n",
1192                                           txq->queue_id);
1193                                 rc = qdev->ops->common->drain(edev);
1194                                 if (rc)
1195                                         return rc;
1196                                 return qede_drain_txq(qdev, txq, false);
1197                         }
1198
1199                         DP_NOTICE(edev, false,
1200                                   "Timeout waiting for tx queue[%d]:"
1201                                   "PROD=%d, CONS=%d\n",
1202                                   txq->queue_id, txq->sw_tx_prod,
1203                                   txq->sw_tx_cons);
1204                         return -ENODEV;
1205                 }
1206                 cnt--;
1207                 DELAY(1000);
1208                 rte_compiler_barrier();
1209         }
1210
1211         /* FW finished processing, wait for HW to transmit all tx packets */
1212         DELAY(2000);
1213
1214         return 0;
1215 }
1216
1217 static int qede_stop_queues(struct qede_dev *qdev)
1218 {
1219         struct qed_update_vport_params vport_update_params;
1220         struct ecore_dev *edev = &qdev->edev;
1221         int rc, tc, i;
1222
1223         /* Disable the vport */
1224         memset(&vport_update_params, 0, sizeof(vport_update_params));
1225         vport_update_params.vport_id = 0;
1226         vport_update_params.update_vport_active_flg = 1;
1227         vport_update_params.vport_active_flg = 0;
1228         vport_update_params.update_rss_flg = 0;
1229
1230         DP_INFO(edev, "Deactivate vport\n");
1231
1232         rc = qdev->ops->vport_update(edev, &vport_update_params);
1233         if (rc) {
1234                 DP_ERR(edev, "Failed to update vport\n");
1235                 return rc;
1236         }
1237
1238         DP_INFO(edev, "Flushing tx queues\n");
1239
1240         /* Flush Tx queues. If needed, request drain from MCP */
1241         for_each_queue(i) {
1242                 struct qede_fastpath *fp = &qdev->fp_array[i];
1243
1244                 if (fp->type & QEDE_FASTPATH_TX) {
1245                         for (tc = 0; tc < qdev->num_tc; tc++) {
1246                                 struct qede_tx_queue *txq = fp->txqs[tc];
1247
1248                                 rc = qede_drain_txq(qdev, txq, true);
1249                                 if (rc)
1250                                         return rc;
1251                         }
1252                 }
1253         }
1254
1255         /* Stop all Queues in reverse order */
1256         for (i = QEDE_QUEUE_CNT(qdev) - 1; i >= 0; i--) {
1257                 struct qed_stop_rxq_params rx_params;
1258
1259                 /* Stop the Tx Queue(s) */
1260                 if (qdev->fp_array[i].type & QEDE_FASTPATH_TX) {
1261                         for (tc = 0; tc < qdev->num_tc; tc++) {
1262                                 struct qed_stop_txq_params tx_params;
1263                                 u8 val;
1264
1265                                 tx_params.rss_id = i;
1266                                 val = qdev->fp_array[i].txqs[tc]->queue_id;
1267                                 tx_params.tx_queue_id = val;
1268
1269                                 DP_INFO(edev, "Stopping tx queues\n");
1270                                 rc = qdev->ops->q_tx_stop(edev, &tx_params);
1271                                 if (rc) {
1272                                         DP_ERR(edev, "Failed to stop TXQ #%d\n",
1273                                                tx_params.tx_queue_id);
1274                                         return rc;
1275                                 }
1276                         }
1277                 }
1278
1279                 /* Stop the Rx Queue */
1280                 if (qdev->fp_array[i].type & QEDE_FASTPATH_RX) {
1281                         memset(&rx_params, 0, sizeof(rx_params));
1282                         rx_params.rss_id = i;
1283                         rx_params.rx_queue_id = qdev->fp_array[i].rxq->queue_id;
1284                         rx_params.eq_completion_only = 1;
1285
1286                         DP_INFO(edev, "Stopping rx queues\n");
1287
1288                         rc = qdev->ops->q_rx_stop(edev, &rx_params);
1289                         if (rc) {
1290                                 DP_ERR(edev, "Failed to stop RXQ #%d\n", i);
1291                                 return rc;
1292                         }
1293                 }
1294         }
1295
1296         return 0;
1297 }
1298
1299 int qede_reset_fp_rings(struct qede_dev *qdev)
1300 {
1301         struct qede_fastpath *fp;
1302         struct qede_tx_queue *txq;
1303         uint8_t tc;
1304         uint16_t id, i;
1305
1306         for_each_queue(id) {
1307                 fp = &qdev->fp_array[id];
1308
1309                 if (fp->type & QEDE_FASTPATH_RX) {
1310                         DP_INFO(&qdev->edev,
1311                                 "Reset FP chain for RSS %u\n", id);
1312                         qede_rx_queue_release_mbufs(fp->rxq);
1313                         ecore_chain_reset(&fp->rxq->rx_bd_ring);
1314                         ecore_chain_reset(&fp->rxq->rx_comp_ring);
1315                         fp->rxq->sw_rx_prod = 0;
1316                         fp->rxq->sw_rx_cons = 0;
1317                         *fp->rxq->hw_cons_ptr = 0;
1318                         for (i = 0; i < fp->rxq->nb_rx_desc; i++) {
1319                                 if (qede_alloc_rx_buffer(fp->rxq)) {
1320                                         DP_ERR(&qdev->edev,
1321                                                "RX buffer allocation failed\n");
1322                                         return -ENOMEM;
1323                                 }
1324                         }
1325                 }
1326                 if (fp->type & QEDE_FASTPATH_TX) {
1327                         for (tc = 0; tc < qdev->num_tc; tc++) {
1328                                 txq = fp->txqs[tc];
1329                                 ecore_chain_reset(&txq->tx_pbl);
1330                                 txq->sw_tx_cons = 0;
1331                                 txq->sw_tx_prod = 0;
1332                                 *txq->hw_cons_ptr = 0;
1333                         }
1334                 }
1335         }
1336
1337         return 0;
1338 }
1339
1340 /* This function frees all memory of a single fp */
1341 static void qede_free_mem_fp(struct rte_eth_dev *eth_dev,
1342                              struct qede_fastpath *fp)
1343 {
1344         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
1345         uint8_t tc;
1346
1347         qede_rx_queue_release(fp->rxq);
1348         for (tc = 0; tc < qdev->num_tc; tc++) {
1349                 qede_tx_queue_release(fp->txqs[tc]);
1350                 eth_dev->data->tx_queues[tc] = NULL;
1351         }
1352 }
1353
1354 void qede_free_mem_load(struct rte_eth_dev *eth_dev)
1355 {
1356         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
1357         struct qede_fastpath *fp;
1358         uint8_t rss_id;
1359
1360         for_each_queue(rss_id) {
1361                 fp = &qdev->fp_array[rss_id];
1362                 qede_free_mem_fp(eth_dev, fp);
1363                 eth_dev->data->rx_queues[rss_id] = NULL;
1364         }
1365 }
1366
1367 void qede_dev_stop(struct rte_eth_dev *eth_dev)
1368 {
1369         struct qede_dev *qdev = eth_dev->data->dev_private;
1370         struct ecore_dev *edev = &qdev->edev;
1371
1372         DP_INFO(edev, "port %u\n", eth_dev->data->port_id);
1373
1374         if (qdev->state != QEDE_DEV_START) {
1375                 DP_INFO(edev, "Device not yet started\n");
1376                 return;
1377         }
1378
1379         if (qede_stop_queues(qdev))
1380                 DP_ERR(edev, "Didn't succeed to close queues\n");
1381
1382         DP_INFO(edev, "Stopped queues\n");
1383
1384         qdev->ops->fastpath_stop(edev);
1385
1386         /* Bring the link down */
1387         qede_dev_set_link_state(eth_dev, false);
1388
1389         qdev->state = QEDE_DEV_STOP;
1390
1391         DP_INFO(edev, "dev_state is QEDE_DEV_STOP\n");
1392 }