drivers/net/qede/qede_rxtx.c

   1 /*
   2  * Copyright (c) 2016 QLogic Corporation.
   3  * All rights reserved.
   4  * www.qlogic.com
   5  *
   6  * See LICENSE.qede_pmd for copyright and licensing details.
   7  */
   8
   9 #include "qede_rxtx.h"
  10
  11 static bool gro_disable = 1;    /* mod_param */
  12
  13 #define QEDE_FASTPATH_TX        (1 << 0)
  14 #define QEDE_FASTPATH_RX        (1 << 1)
  15
  16 static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq)
  17 {
  18         struct rte_mbuf *new_mb = NULL;
  19         struct eth_rx_bd *rx_bd;
  20         dma_addr_t mapping;
  21         uint16_t idx = rxq->sw_rx_prod & NUM_RX_BDS(rxq);
  22
  23         new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
  24         if (unlikely(!new_mb)) {
  25                 PMD_RX_LOG(ERR, rxq,
  26                            "Failed to allocate rx buffer "
  27                            "sw_rx_prod %u sw_rx_cons %u mp entries %u free %u",
  28                            idx, rxq->sw_rx_cons & NUM_RX_BDS(rxq),
  29                            rte_mempool_avail_count(rxq->mb_pool),
  30                            rte_mempool_in_use_count(rxq->mb_pool));
  31                 return -ENOMEM;
  32         }
  33         rxq->sw_rx_ring[idx].mbuf = new_mb;
  34         rxq->sw_rx_ring[idx].page_offset = 0;
  35         mapping = rte_mbuf_data_dma_addr_default(new_mb);
  36         /* Advance PROD and get BD pointer */
  37         rx_bd = (struct eth_rx_bd *)ecore_chain_produce(&rxq->rx_bd_ring);
  38         rx_bd->addr.hi = rte_cpu_to_le_32(U64_HI(mapping));
  39         rx_bd->addr.lo = rte_cpu_to_le_32(U64_LO(mapping));
  40         rxq->sw_rx_prod++;
  41         return 0;
  42 }
  43
  44 static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq)
  45 {
  46         uint16_t i;
  47
  48         if (rxq->sw_rx_ring != NULL) {
  49                 for (i = 0; i < rxq->nb_rx_desc; i++) {
  50                         if (rxq->sw_rx_ring[i].mbuf != NULL) {
  51                                 rte_pktmbuf_free(rxq->sw_rx_ring[i].mbuf);
  52                                 rxq->sw_rx_ring[i].mbuf = NULL;
  53                         }
  54                 }
  55         }
  56 }
  57
  58 void qede_rx_queue_release(void *rx_queue)
  59 {
  60         struct qede_rx_queue *rxq = rx_queue;
  61
  62         if (rxq != NULL) {
  63                 qede_rx_queue_release_mbufs(rxq);
  64                 rte_free(rxq->sw_rx_ring);
  65                 rxq->sw_rx_ring = NULL;
  66                 rte_free(rxq);
  67                 rxq = NULL;
  68         }
  69 }
  70
  71 static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq)
  72 {
  73         unsigned int i;
  74
  75         PMD_TX_LOG(DEBUG, txq, "releasing %u mbufs\n", txq->nb_tx_desc);
  76
  77         if (txq->sw_tx_ring) {
  78                 for (i = 0; i < txq->nb_tx_desc; i++) {
  79                         if (txq->sw_tx_ring[i].mbuf) {
  80                                 rte_pktmbuf_free(txq->sw_tx_ring[i].mbuf);
  81                                 txq->sw_tx_ring[i].mbuf = NULL;
  82                         }
  83                 }
  84         }
  85 }
  86
  87 int
  88 qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
  89                     uint16_t nb_desc, unsigned int socket_id,
  90                     const struct rte_eth_rxconf *rx_conf,
  91                     struct rte_mempool *mp)
  92 {
  93         struct qede_dev *qdev = dev->data->dev_private;
  94         struct ecore_dev *edev = &qdev->edev;
  95         struct rte_eth_dev_data *eth_data = dev->data;
  96         struct qede_rx_queue *rxq;
  97         uint16_t pkt_len = (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len;
  98         size_t size;
  99         uint16_t data_size;
 100         int rc;
 101         int i;
 102
 103         PMD_INIT_FUNC_TRACE(edev);
 104
 105         /* Note: Ring size/align is controlled by struct rte_eth_desc_lim */
 106         if (!rte_is_power_of_2(nb_desc)) {
 107                 DP_ERR(edev, "Ring size %u is not power of 2\n",
 108                           nb_desc);
 109                 return -EINVAL;
 110         }
 111
 112         /* Free memory prior to re-allocation if needed... */
 113         if (dev->data->rx_queues[queue_idx] != NULL) {
 114                 qede_rx_queue_release(dev->data->rx_queues[queue_idx]);
 115                 dev->data->rx_queues[queue_idx] = NULL;
 116         }
 117
 118         /* First allocate the rx queue data structure */
 119         rxq = rte_zmalloc_socket("qede_rx_queue", sizeof(struct qede_rx_queue),
 120                                  RTE_CACHE_LINE_SIZE, socket_id);
 121
 122         if (!rxq) {
 123                 DP_ERR(edev, "Unable to allocate memory for rxq on socket %u",
 124                           socket_id);
 125                 return -ENOMEM;
 126         }
 127
 128         rxq->qdev = qdev;
 129         rxq->mb_pool = mp;
 130         rxq->nb_rx_desc = nb_desc;
 131         rxq->queue_id = queue_idx;
 132         rxq->port_id = dev->data->port_id;
 133
 134         /* Sanity check */
 135         data_size = (uint16_t)rte_pktmbuf_data_room_size(mp) -
 136                                 RTE_PKTMBUF_HEADROOM;
 137
 138         qdev->mtu = pkt_len;
 139         rxq->rx_buf_size = data_size;
 140
 141         DP_INFO(edev, "MTU = %u ; RX buffer = %u\n",
 142                 qdev->mtu, rxq->rx_buf_size);
 143
 144         if (pkt_len > ETHER_MAX_LEN) {
 145                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
 146                 DP_NOTICE(edev, false, "jumbo frame enabled\n");
 147         } else {
 148                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
 149         }
 150
 151         /* Allocate the parallel driver ring for Rx buffers */
 152         size = sizeof(*rxq->sw_rx_ring) * rxq->nb_rx_desc;
 153         rxq->sw_rx_ring = rte_zmalloc_socket("sw_rx_ring", size,
 154                                              RTE_CACHE_LINE_SIZE, socket_id);
 155         if (!rxq->sw_rx_ring) {
 156                 DP_NOTICE(edev, false,
 157                           "Unable to alloc memory for sw_rx_ring on socket %u\n",
 158                           socket_id);
 159                 rte_free(rxq);
 160                 rxq = NULL;
 161                 return -ENOMEM;
 162         }
 163
 164         /* Allocate FW Rx ring  */
 165         rc = qdev->ops->common->chain_alloc(edev,
 166                                             ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
 167                                             ECORE_CHAIN_MODE_NEXT_PTR,
 168                                             ECORE_CHAIN_CNT_TYPE_U16,
 169                                             rxq->nb_rx_desc,
 170                                             sizeof(struct eth_rx_bd),
 171                                             &rxq->rx_bd_ring);
 172
 173         if (rc != ECORE_SUCCESS) {
 174                 DP_NOTICE(edev, false,
 175                           "Unable to alloc memory for rxbd ring on socket %u\n",
 176                           socket_id);
 177                 rte_free(rxq->sw_rx_ring);
 178                 rxq->sw_rx_ring = NULL;
 179                 rte_free(rxq);
 180                 rxq = NULL;
 181                 return -ENOMEM;
 182         }
 183
 184         /* Allocate FW completion ring */
 185         rc = qdev->ops->common->chain_alloc(edev,
 186                                             ECORE_CHAIN_USE_TO_CONSUME,
 187                                             ECORE_CHAIN_MODE_PBL,
 188                                             ECORE_CHAIN_CNT_TYPE_U16,
 189                                             rxq->nb_rx_desc,
 190                                             sizeof(union eth_rx_cqe),
 191                                             &rxq->rx_comp_ring);
 192
 193         if (rc != ECORE_SUCCESS) {
 194                 DP_NOTICE(edev, false,
 195                           "Unable to alloc memory for cqe ring on socket %u\n",
 196                           socket_id);
 197                 /* TBD: Freeing RX BD ring */
 198                 rte_free(rxq->sw_rx_ring);
 199                 rxq->sw_rx_ring = NULL;
 200                 rte_free(rxq);
 201                 return -ENOMEM;
 202         }
 203
 204         /* Allocate buffers for the Rx ring */
 205         for (i = 0; i < rxq->nb_rx_desc; i++) {
 206                 rc = qede_alloc_rx_buffer(rxq);
 207                 if (rc) {
 208                         DP_NOTICE(edev, false,
 209                                   "RX buffer allocation failed at idx=%d\n", i);
 210                         goto err4;
 211                 }
 212         }
 213
 214         dev->data->rx_queues[queue_idx] = rxq;
 215
 216         DP_INFO(edev, "rxq %d num_desc %u rx_buf_size=%u socket %u\n",
 217                   queue_idx, nb_desc, qdev->mtu, socket_id);
 218
 219         return 0;
 220 err4:
 221         qede_rx_queue_release(rxq);
 222         return -ENOMEM;
 223 }
 224
 225 void qede_tx_queue_release(void *tx_queue)
 226 {
 227         struct qede_tx_queue *txq = tx_queue;
 228
 229         if (txq != NULL) {
 230                 qede_tx_queue_release_mbufs(txq);
 231                 if (txq->sw_tx_ring) {
 232                         rte_free(txq->sw_tx_ring);
 233                         txq->sw_tx_ring = NULL;
 234                 }
 235                 rte_free(txq);
 236         }
 237         txq = NULL;
 238 }
 239
 240 int
 241 qede_tx_queue_setup(struct rte_eth_dev *dev,
 242                     uint16_t queue_idx,
 243                     uint16_t nb_desc,
 244                     unsigned int socket_id,
 245                     const struct rte_eth_txconf *tx_conf)
 246 {
 247         struct qede_dev *qdev = dev->data->dev_private;
 248         struct ecore_dev *edev = &qdev->edev;
 249         struct qede_tx_queue *txq;
 250         int rc;
 251
 252         PMD_INIT_FUNC_TRACE(edev);
 253
 254         if (!rte_is_power_of_2(nb_desc)) {
 255                 DP_ERR(edev, "Ring size %u is not power of 2\n",
 256                        nb_desc);
 257                 return -EINVAL;
 258         }
 259
 260         /* Free memory prior to re-allocation if needed... */
 261         if (dev->data->tx_queues[queue_idx] != NULL) {
 262                 qede_tx_queue_release(dev->data->tx_queues[queue_idx]);
 263                 dev->data->tx_queues[queue_idx] = NULL;
 264         }
 265
 266         txq = rte_zmalloc_socket("qede_tx_queue", sizeof(struct qede_tx_queue),
 267                                  RTE_CACHE_LINE_SIZE, socket_id);
 268
 269         if (txq == NULL) {
 270                 DP_ERR(edev,
 271                        "Unable to allocate memory for txq on socket %u",
 272                        socket_id);
 273                 return -ENOMEM;
 274         }
 275
 276         txq->nb_tx_desc = nb_desc;
 277         txq->qdev = qdev;
 278         txq->port_id = dev->data->port_id;
 279
 280         rc = qdev->ops->common->chain_alloc(edev,
 281                                             ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
 282                                             ECORE_CHAIN_MODE_PBL,
 283                                             ECORE_CHAIN_CNT_TYPE_U16,
 284                                             txq->nb_tx_desc,
 285                                             sizeof(union eth_tx_bd_types),
 286                                             &txq->tx_pbl);
 287         if (rc != ECORE_SUCCESS) {
 288                 DP_ERR(edev,
 289                        "Unable to allocate memory for txbd ring on socket %u",
 290                        socket_id);
 291                 qede_tx_queue_release(txq);
 292                 return -ENOMEM;
 293         }
 294
 295         /* Allocate software ring */
 296         txq->sw_tx_ring = rte_zmalloc_socket("txq->sw_tx_ring",
 297                                              (sizeof(struct qede_tx_entry) *
 298                                               txq->nb_tx_desc),
 299                                              RTE_CACHE_LINE_SIZE, socket_id);
 300
 301         if (!txq->sw_tx_ring) {
 302                 DP_ERR(edev,
 303                        "Unable to allocate memory for txbd ring on socket %u",
 304                        socket_id);
 305                 qede_tx_queue_release(txq);
 306                 return -ENOMEM;
 307         }
 308
 309         txq->queue_id = queue_idx;
 310
 311         txq->nb_tx_avail = txq->nb_tx_desc;
 312
 313         txq->tx_free_thresh =
 314             tx_conf->tx_free_thresh ? tx_conf->tx_free_thresh :
 315             (txq->nb_tx_desc - QEDE_DEFAULT_TX_FREE_THRESH);
 316
 317         dev->data->tx_queues[queue_idx] = txq;
 318
 319         DP_INFO(edev,
 320                   "txq %u num_desc %u tx_free_thresh %u socket %u\n",
 321                   queue_idx, nb_desc, txq->tx_free_thresh, socket_id);
 322
 323         return 0;
 324 }
 325
 326 /* This function inits fp content and resets the SB, RXQ and TXQ arrays */
 327 static void qede_init_fp(struct qede_dev *qdev)
 328 {
 329         struct qede_fastpath *fp;
 330         uint8_t i, rss_id, tc;
 331         int fp_rx = qdev->fp_num_rx, rxq = 0, txq = 0;
 332
 333         memset((void *)qdev->fp_array, 0, (QEDE_QUEUE_CNT(qdev) *
 334                                            sizeof(*qdev->fp_array)));
 335         memset((void *)qdev->sb_array, 0, (QEDE_QUEUE_CNT(qdev) *
 336                                            sizeof(*qdev->sb_array)));
 337         for_each_queue(i) {
 338                 fp = &qdev->fp_array[i];
 339                 if (fp_rx) {
 340                         fp->type = QEDE_FASTPATH_RX;
 341                         fp_rx--;
 342                 } else{
 343                         fp->type = QEDE_FASTPATH_TX;
 344                 }
 345                 fp->qdev = qdev;
 346                 fp->id = i;
 347                 fp->sb_info = &qdev->sb_array[i];
 348                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", "qdev", i);
 349         }
 350
 351         qdev->gro_disable = gro_disable;
 352 }
 353
 354 void qede_free_fp_arrays(struct qede_dev *qdev)
 355 {
 356         /* It asseumes qede_free_mem_load() is called before */
 357         if (qdev->fp_array != NULL) {
 358                 rte_free(qdev->fp_array);
 359                 qdev->fp_array = NULL;
 360         }
 361
 362         if (qdev->sb_array != NULL) {
 363                 rte_free(qdev->sb_array);
 364                 qdev->sb_array = NULL;
 365         }
 366 }
 367
 368 int qede_alloc_fp_array(struct qede_dev *qdev)
 369 {
 370         struct qede_fastpath *fp;
 371         struct ecore_dev *edev = &qdev->edev;
 372         int i;
 373
 374         qdev->fp_array = rte_calloc("fp", QEDE_QUEUE_CNT(qdev),
 375                                     sizeof(*qdev->fp_array),
 376                                     RTE_CACHE_LINE_SIZE);
 377
 378         if (!qdev->fp_array) {
 379                 DP_ERR(edev, "fp array allocation failed\n");
 380                 return -ENOMEM;
 381         }
 382
 383         qdev->sb_array = rte_calloc("sb", QEDE_QUEUE_CNT(qdev),
 384                                     sizeof(*qdev->sb_array),
 385                                     RTE_CACHE_LINE_SIZE);
 386
 387         if (!qdev->sb_array) {
 388                 DP_ERR(edev, "sb array allocation failed\n");
 389                 rte_free(qdev->fp_array);
 390                 return -ENOMEM;
 391         }
 392
 393         return 0;
 394 }
 395
 396 /* This function allocates fast-path status block memory */
 397 static int
 398 qede_alloc_mem_sb(struct qede_dev *qdev, struct ecore_sb_info *sb_info,
 399                   uint16_t sb_id)
 400 {
 401         struct ecore_dev *edev = &qdev->edev;
 402         struct status_block *sb_virt;
 403         dma_addr_t sb_phys;
 404         int rc;
 405
 406         sb_virt = OSAL_DMA_ALLOC_COHERENT(edev, &sb_phys, sizeof(*sb_virt));
 407
 408         if (!sb_virt) {
 409                 DP_ERR(edev, "Status block allocation failed\n");
 410                 return -ENOMEM;
 411         }
 412
 413         rc = qdev->ops->common->sb_init(edev, sb_info,
 414                                         sb_virt, sb_phys, sb_id,
 415                                         QED_SB_TYPE_L2_QUEUE);
 416         if (rc) {
 417                 DP_ERR(edev, "Status block initialization failed\n");
 418                 /* TBD: No dma_free_coherent possible */
 419                 return rc;
 420         }
 421
 422         return 0;
 423 }
 424
 425 int qede_alloc_fp_resc(struct qede_dev *qdev)
 426 {
 427         struct qede_fastpath *fp;
 428         int rc, i;
 429
 430         if (qdev->fp_array)
 431                 qede_free_fp_arrays(qdev);
 432
 433         rc = qede_alloc_fp_array(qdev);
 434         if (rc != 0)
 435                 return rc;
 436
 437         qede_init_fp(qdev);
 438
 439         for (i = 0; i < QEDE_QUEUE_CNT(qdev); i++) {
 440                 fp = &qdev->fp_array[i];
 441                 if (qede_alloc_mem_sb(qdev, fp->sb_info, i)) {
 442                         qede_free_fp_arrays(qdev);
 443                         return -ENOMEM;
 444                 }
 445         }
 446
 447         return 0;
 448 }
 449
 450 void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev)
 451 {
 452         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
 453
 454         qede_free_mem_load(eth_dev);
 455         qede_free_fp_arrays(qdev);
 456 }
 457
 458 static inline void
 459 qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq)
 460 {
 461         uint16_t bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring);
 462         uint16_t cqe_prod = ecore_chain_get_prod_idx(&rxq->rx_comp_ring);
 463         struct eth_rx_prod_data rx_prods = { 0 };
 464
 465         /* Update producers */
 466         rx_prods.bd_prod = rte_cpu_to_le_16(bd_prod);
 467         rx_prods.cqe_prod = rte_cpu_to_le_16(cqe_prod);
 468
 469         /* Make sure that the BD and SGE data is updated before updating the
 470          * producers since FW might read the BD/SGE right after the producer
 471          * is updated.
 472          */
 473         rte_wmb();
 474
 475         internal_ram_wr(rxq->hw_rxq_prod_addr, sizeof(rx_prods),
 476                         (uint32_t *)&rx_prods);
 477
 478         /* mmiowb is needed to synchronize doorbell writes from more than one
 479          * processor. It guarantees that the write arrives to the device before
 480          * the napi lock is released and another qede_poll is called (possibly
 481          * on another CPU). Without this barrier, the next doorbell can bypass
 482          * this doorbell. This is applicable to IA64/Altix systems.
 483          */
 484         rte_wmb();
 485
 486         PMD_RX_LOG(DEBUG, rxq, "bd_prod %u  cqe_prod %u\n", bd_prod, cqe_prod);
 487 }
 488
 489 static inline uint32_t
 490 qede_rxfh_indir_default(uint32_t index, uint32_t n_rx_rings)
 491 {
 492         return index % n_rx_rings;
 493 }
 494
 495 static void qede_prandom_bytes(uint32_t *buff, size_t bytes)
 496 {
 497         unsigned int i;
 498
 499         srand((unsigned int)time(NULL));
 500
 501         for (i = 0; i < ECORE_RSS_KEY_SIZE; i++)
 502                 buff[i] = rand();
 503 }
 504
 505 static bool
 506 qede_check_vport_rss_enable(struct rte_eth_dev *eth_dev,
 507                             struct qed_update_vport_rss_params *rss_params)
 508 {
 509         struct rte_eth_rss_conf rss_conf;
 510         enum rte_eth_rx_mq_mode mode = eth_dev->data->dev_conf.rxmode.mq_mode;
 511         struct qede_dev *qdev = eth_dev->data->dev_private;
 512         struct ecore_dev *edev = &qdev->edev;
 513         uint8_t rss_caps;
 514         unsigned int i;
 515         uint64_t hf;
 516         uint32_t *key;
 517
 518         PMD_INIT_FUNC_TRACE(edev);
 519
 520         rss_conf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 521         key = (uint32_t *)rss_conf.rss_key;
 522         hf = rss_conf.rss_hf;
 523
 524         /* Check if RSS conditions are met.
 525          * Note: Even though its meaningless to enable RSS with one queue, it
 526          * could be used to produce RSS Hash, so skipping that check.
 527          */
 528         if (!(mode & ETH_MQ_RX_RSS)) {
 529                 DP_INFO(edev, "RSS flag is not set\n");
 530                 return false;
 531         }
 532
 533         if (hf == 0) {
 534                 DP_INFO(edev, "Request to disable RSS\n");
 535                 return false;
 536         }
 537
 538         memset(rss_params, 0, sizeof(*rss_params));
 539
 540         for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++)
 541                 rss_params->rss_ind_table[i] = qede_rxfh_indir_default(i,
 542                                                         QEDE_RSS_COUNT(qdev));
 543
 544         if (!key)
 545                 qede_prandom_bytes(rss_params->rss_key,
 546                                    sizeof(rss_params->rss_key));
 547         else
 548                 memcpy(rss_params->rss_key, rss_conf.rss_key,
 549                        rss_conf.rss_key_len);
 550
 551         qede_init_rss_caps(&rss_caps, hf);
 552
 553         rss_params->rss_caps = rss_caps;
 554
 555         DP_INFO(edev, "RSS conditions are met\n");
 556
 557         return true;
 558 }
 559
 560 static int qede_start_queues(struct rte_eth_dev *eth_dev, bool clear_stats)
 561 {
 562         struct qede_dev *qdev = eth_dev->data->dev_private;
 563         struct ecore_dev *edev = &qdev->edev;
 564         struct qed_update_vport_rss_params *rss_params = &qdev->rss_params;
 565         struct qed_dev_info *qed_info = &qdev->dev_info.common;
 566         struct qed_update_vport_params vport_update_params;
 567         struct qede_tx_queue *txq;
 568         struct qede_fastpath *fp;
 569         dma_addr_t p_phys_table;
 570         int txq_index;
 571         uint16_t page_cnt;
 572         int vlan_removal_en = 1;
 573         int rc, tc, i;
 574
 575         for_each_queue(i) {
 576                 fp = &qdev->fp_array[i];
 577                 if (fp->type & QEDE_FASTPATH_RX) {
 578                         p_phys_table = ecore_chain_get_pbl_phys(&fp->rxq->
 579                                                                 rx_comp_ring);
 580                         page_cnt = ecore_chain_get_page_cnt(&fp->rxq->
 581                                                                 rx_comp_ring);
 582
 583                         ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
 584
 585                         rc = qdev->ops->q_rx_start(edev, i, fp->rxq->queue_id,
 586                                            0,
 587                                            fp->sb_info->igu_sb_id,
 588                                            RX_PI,
 589                                            fp->rxq->rx_buf_size,
 590                                            fp->rxq->rx_bd_ring.p_phys_addr,
 591                                            p_phys_table,
 592                                            page_cnt,
 593                                            &fp->rxq->hw_rxq_prod_addr);
 594                         if (rc) {
 595                                 DP_ERR(edev, "Start rxq #%d failed %d\n",
 596                                        fp->rxq->queue_id, rc);
 597                                 return rc;
 598                         }
 599
 600                         fp->rxq->hw_cons_ptr =
 601                                         &fp->sb_info->sb_virt->pi_array[RX_PI];
 602
 603                         qede_update_rx_prod(qdev, fp->rxq);
 604                 }
 605
 606                 if (!(fp->type & QEDE_FASTPATH_TX))
 607                         continue;
 608                 for (tc = 0; tc < qdev->num_tc; tc++) {
 609                         txq = fp->txqs[tc];
 610                         txq_index = tc * QEDE_RSS_COUNT(qdev) + i;
 611
 612                         p_phys_table = ecore_chain_get_pbl_phys(&txq->tx_pbl);
 613                         page_cnt = ecore_chain_get_page_cnt(&txq->tx_pbl);
 614                         rc = qdev->ops->q_tx_start(edev, i, txq->queue_id,
 615                                                    0,
 616                                                    fp->sb_info->igu_sb_id,
 617                                                    TX_PI(tc),
 618                                                    p_phys_table, page_cnt,
 619                                                    &txq->doorbell_addr);
 620                         if (rc) {
 621                                 DP_ERR(edev, "Start txq %u failed %d\n",
 622                                        txq_index, rc);
 623                                 return rc;
 624                         }
 625
 626                         txq->hw_cons_ptr =
 627                             &fp->sb_info->sb_virt->pi_array[TX_PI(tc)];
 628                         SET_FIELD(txq->tx_db.data.params,
 629                                   ETH_DB_DATA_DEST, DB_DEST_XCM);
 630                         SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD,
 631                                   DB_AGG_CMD_SET);
 632                         SET_FIELD(txq->tx_db.data.params,
 633                                   ETH_DB_DATA_AGG_VAL_SEL,
 634                                   DQ_XCM_ETH_TX_BD_PROD_CMD);
 635
 636                         txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
 637                 }
 638         }
 639
 640         /* Prepare and send the vport enable */
 641         memset(&vport_update_params, 0, sizeof(vport_update_params));
 642         /* Update MTU via vport update */
 643         vport_update_params.mtu = qdev->mtu;
 644         vport_update_params.vport_id = 0;
 645         vport_update_params.update_vport_active_flg = 1;
 646         vport_update_params.vport_active_flg = 1;
 647
 648         /* @DPDK */
 649         if (qed_info->mf_mode == MF_NPAR && qed_info->tx_switching) {
 650                 /* TBD: Check SRIOV enabled for VF */
 651                 vport_update_params.update_tx_switching_flg = 1;
 652                 vport_update_params.tx_switching_flg = 1;
 653         }
 654
 655         if (qede_check_vport_rss_enable(eth_dev, rss_params)) {
 656                 vport_update_params.update_rss_flg = 1;
 657                 qdev->rss_enabled = 1;
 658         } else {
 659                 qdev->rss_enabled = 0;
 660         }
 661
 662         rte_memcpy(&vport_update_params.rss_params, rss_params,
 663                sizeof(*rss_params));
 664
 665         rc = qdev->ops->vport_update(edev, &vport_update_params);
 666         if (rc) {
 667                 DP_ERR(edev, "Update V-PORT failed %d\n", rc);
 668                 return rc;
 669         }
 670
 671         return 0;
 672 }
 673
 674 #ifdef ENC_SUPPORTED
 675 static bool qede_tunn_exist(uint16_t flag)
 676 {
 677         return !!((PARSING_AND_ERR_FLAGS_TUNNELEXIST_MASK <<
 678                     PARSING_AND_ERR_FLAGS_TUNNELEXIST_SHIFT) & flag);
 679 }
 680
 681 static inline uint8_t qede_check_tunn_csum(uint16_t flag)
 682 {
 683         uint8_t tcsum = 0;
 684         uint16_t csum_flag = 0;
 685
 686         if ((PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_MASK <<
 687              PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_SHIFT) & flag)
 688                 csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_MASK <<
 689                     PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT;
 690
 691         if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK <<
 692              PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) {
 693                 csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK <<
 694                     PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT;
 695                 tcsum = QEDE_TUNN_CSUM_UNNECESSARY;
 696         }
 697
 698         csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_MASK <<
 699             PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_SHIFT |
 700             PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK <<
 701             PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT;
 702
 703         if (csum_flag & flag)
 704                 return QEDE_CSUM_ERROR;
 705
 706         return QEDE_CSUM_UNNECESSARY | tcsum;
 707 }
 708 #else
 709 static inline uint8_t qede_tunn_exist(uint16_t flag)
 710 {
 711         return 0;
 712 }
 713
 714 static inline uint8_t qede_check_tunn_csum(uint16_t flag)
 715 {
 716         return 0;
 717 }
 718 #endif
 719
 720 static inline uint8_t qede_check_notunn_csum(uint16_t flag)
 721 {
 722         uint8_t csum = 0;
 723         uint16_t csum_flag = 0;
 724
 725         if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK <<
 726              PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) {
 727                 csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK <<
 728                     PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT;
 729                 csum = QEDE_CSUM_UNNECESSARY;
 730         }
 731
 732         csum_flag |= PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK <<
 733             PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT;
 734
 735         if (csum_flag & flag)
 736                 return QEDE_CSUM_ERROR;
 737
 738         return csum;
 739 }
 740
 741 static inline uint8_t qede_check_csum(uint16_t flag)
 742 {
 743         if (likely(!qede_tunn_exist(flag)))
 744                 return qede_check_notunn_csum(flag);
 745         else
 746                 return qede_check_tunn_csum(flag);
 747 }
 748
 749 static inline void qede_rx_bd_ring_consume(struct qede_rx_queue *rxq)
 750 {
 751         ecore_chain_consume(&rxq->rx_bd_ring);
 752         rxq->sw_rx_cons++;
 753 }
 754
 755 static inline void
 756 qede_reuse_page(struct qede_dev *qdev,
 757                 struct qede_rx_queue *rxq, struct qede_rx_entry *curr_cons)
 758 {
 759         struct eth_rx_bd *rx_bd_prod = ecore_chain_produce(&rxq->rx_bd_ring);
 760         uint16_t idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
 761         struct qede_rx_entry *curr_prod;
 762         dma_addr_t new_mapping;
 763
 764         curr_prod = &rxq->sw_rx_ring[idx];
 765         *curr_prod = *curr_cons;
 766
 767         new_mapping = rte_mbuf_data_dma_addr_default(curr_prod->mbuf) +
 768                       curr_prod->page_offset;
 769
 770         rx_bd_prod->addr.hi = rte_cpu_to_le_32(U64_HI(new_mapping));
 771         rx_bd_prod->addr.lo = rte_cpu_to_le_32(U64_LO(new_mapping));
 772
 773         rxq->sw_rx_prod++;
 774 }
 775
 776 static inline void
 777 qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq,
 778                         struct qede_dev *qdev, uint8_t count)
 779 {
 780         struct qede_rx_entry *curr_cons;
 781
 782         for (; count > 0; count--) {
 783                 curr_cons = &rxq->sw_rx_ring[rxq->sw_rx_cons & NUM_RX_BDS(rxq)];
 784                 qede_reuse_page(qdev, rxq, curr_cons);
 785                 qede_rx_bd_ring_consume(rxq);
 786         }
 787 }
 788
 789 static inline uint32_t qede_rx_cqe_to_pkt_type(uint16_t flags)
 790 {
 791         uint32_t p_type;
 792         /* TBD - L4 indications needed ? */
 793         uint16_t protocol = ((PARSING_AND_ERR_FLAGS_L3TYPE_MASK <<
 794                               PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT) & flags);
 795
 796         /* protocol = 3 means LLC/SNAP over Ethernet */
 797         if (unlikely(protocol == 0 || protocol == 3))
 798                 p_type = RTE_PTYPE_UNKNOWN;
 799         else if (protocol == 1)
 800                 p_type = RTE_PTYPE_L3_IPV4;
 801         else if (protocol == 2)
 802                 p_type = RTE_PTYPE_L3_IPV6;
 803
 804         return RTE_PTYPE_L2_ETHER | p_type;
 805 }
 806
 807 uint16_t
 808 qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 809 {
 810         struct qede_rx_queue *rxq = p_rxq;
 811         struct qede_dev *qdev = rxq->qdev;
 812         struct ecore_dev *edev = &qdev->edev;
 813         struct qede_fastpath *fp = &qdev->fp_array[rxq->queue_id];
 814         uint16_t hw_comp_cons, sw_comp_cons, sw_rx_index;
 815         uint16_t rx_pkt = 0;
 816         union eth_rx_cqe *cqe;
 817         struct eth_fast_path_rx_reg_cqe *fp_cqe;
 818         register struct rte_mbuf *rx_mb = NULL;
 819         enum eth_rx_cqe_type cqe_type;
 820         uint16_t len, pad;
 821         uint16_t preload_idx;
 822         uint8_t csum_flag;
 823         uint16_t parse_flag;
 824         enum rss_hash_type htype;
 825
 826         hw_comp_cons = rte_le_to_cpu_16(*rxq->hw_cons_ptr);
 827         sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
 828
 829         rte_rmb();
 830
 831         if (hw_comp_cons == sw_comp_cons)
 832                 return 0;
 833
 834         while (sw_comp_cons != hw_comp_cons) {
 835                 /* Get the CQE from the completion ring */
 836                 cqe =
 837                     (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring);
 838                 cqe_type = cqe->fast_path_regular.type;
 839
 840                 if (unlikely(cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH)) {
 841                         PMD_RX_LOG(DEBUG, rxq, "Got a slowath CQE\n");
 842
 843                         qdev->ops->eth_cqe_completion(edev, fp->id,
 844                                 (struct eth_slow_path_rx_cqe *)cqe);
 845                         goto next_cqe;
 846                 }
 847
 848                 /* Get the data from the SW ring */
 849                 sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
 850                 rx_mb = rxq->sw_rx_ring[sw_rx_index].mbuf;
 851                 assert(rx_mb != NULL);
 852
 853                 /* non GRO */
 854                 fp_cqe = &cqe->fast_path_regular;
 855
 856                 len = rte_le_to_cpu_16(fp_cqe->len_on_first_bd);
 857                 pad = fp_cqe->placement_offset;
 858                 assert((len + pad) <= rx_mb->buf_len);
 859
 860                 PMD_RX_LOG(DEBUG, rxq,
 861                            "CQE type = 0x%x, flags = 0x%x, vlan = 0x%x"
 862                            " len = %u, parsing_flags = %d\n",
 863                            cqe_type, fp_cqe->bitfields,
 864                            rte_le_to_cpu_16(fp_cqe->vlan_tag),
 865                            len, rte_le_to_cpu_16(fp_cqe->pars_flags.flags));
 866
 867                 /* If this is an error packet then drop it */
 868                 parse_flag =
 869                     rte_le_to_cpu_16(cqe->fast_path_regular.pars_flags.flags);
 870                 csum_flag = qede_check_csum(parse_flag);
 871                 if (unlikely(csum_flag == QEDE_CSUM_ERROR)) {
 872                         PMD_RX_LOG(ERR, rxq,
 873                                    "CQE in CONS = %u has error, flags = 0x%x "
 874                                    "dropping incoming packet\n",
 875                                    sw_comp_cons, parse_flag);
 876                         rxq->rx_hw_errors++;
 877                         qede_recycle_rx_bd_ring(rxq, qdev, fp_cqe->bd_num);
 878                         goto next_cqe;
 879                 }
 880
 881                 if (unlikely(qede_alloc_rx_buffer(rxq) != 0)) {
 882                         PMD_RX_LOG(ERR, rxq,
 883                                    "New buffer allocation failed,"
 884                                    "dropping incoming packet\n");
 885                         qede_recycle_rx_bd_ring(rxq, qdev, fp_cqe->bd_num);
 886                         rte_eth_devices[rxq->port_id].
 887                             data->rx_mbuf_alloc_failed++;
 888                         rxq->rx_alloc_errors++;
 889                         break;
 890                 }
 891
 892                 qede_rx_bd_ring_consume(rxq);
 893
 894                 /* Prefetch next mbuf while processing current one. */
 895                 preload_idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
 896                 rte_prefetch0(rxq->sw_rx_ring[preload_idx].mbuf);
 897
 898                 if (fp_cqe->bd_num != 1)
 899                         PMD_RX_LOG(DEBUG, rxq,
 900                                    "Jumbo-over-BD packet not supported\n");
 901
 902                 /* Update MBUF fields */
 903                 rx_mb->ol_flags = 0;
 904                 rx_mb->data_off = pad + RTE_PKTMBUF_HEADROOM;
 905                 rx_mb->nb_segs = 1;
 906                 rx_mb->data_len = len;
 907                 rx_mb->pkt_len = len;
 908                 rx_mb->port = rxq->port_id;
 909                 rx_mb->packet_type = qede_rx_cqe_to_pkt_type(parse_flag);
 910
 911                 htype = (uint8_t)GET_FIELD(fp_cqe->bitfields,
 912                                 ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE);
 913                 if (qdev->rss_enabled && htype) {
 914                         rx_mb->ol_flags |= PKT_RX_RSS_HASH;
 915                         rx_mb->hash.rss = rte_le_to_cpu_32(fp_cqe->rss_hash);
 916                         PMD_RX_LOG(DEBUG, rxq, "Hash result 0x%x\n",
 917                                    rx_mb->hash.rss);
 918                 }
 919
 920                 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
 921
 922                 if (CQE_HAS_VLAN(parse_flag)) {
 923                         rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag);
 924                         rx_mb->ol_flags |= PKT_RX_VLAN_PKT;
 925                 }
 926
 927                 if (CQE_HAS_OUTER_VLAN(parse_flag)) {
 928                         /* FW does not provide indication of Outer VLAN tag,
 929                          * which is always stripped, so vlan_tci_outer is set
 930                          * to 0. Here vlan_tag represents inner VLAN tag.
 931                          */
 932                         rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag);
 933                         rx_mb->ol_flags |= PKT_RX_QINQ_PKT;
 934                         rx_mb->vlan_tci_outer = 0;
 935                 }
 936
 937                 rx_pkts[rx_pkt] = rx_mb;
 938                 rx_pkt++;
 939 next_cqe:
 940                 ecore_chain_recycle_consumed(&rxq->rx_comp_ring);
 941                 sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
 942                 if (rx_pkt == nb_pkts) {
 943                         PMD_RX_LOG(DEBUG, rxq,
 944                                    "Budget reached nb_pkts=%u received=%u\n",
 945                                    rx_pkt, nb_pkts);
 946                         break;
 947                 }
 948         }
 949
 950         qede_update_rx_prod(qdev, rxq);
 951
 952         PMD_RX_LOG(DEBUG, rxq, "rx_pkts=%u core=%d\n", rx_pkt, rte_lcore_id());
 953
 954         return rx_pkt;
 955 }
 956
 957 static inline int
 958 qede_free_tx_pkt(struct ecore_dev *edev, struct qede_tx_queue *txq)
 959 {
 960         uint16_t idx = TX_CONS(txq);
 961         struct eth_tx_bd *tx_data_bd;
 962         struct rte_mbuf *mbuf = txq->sw_tx_ring[idx].mbuf;
 963
 964         if (unlikely(!mbuf)) {
 965                 PMD_TX_LOG(ERR, txq,
 966                            "null mbuf nb_tx_desc %u nb_tx_avail %u "
 967                            "sw_tx_cons %u sw_tx_prod %u\n",
 968                            txq->nb_tx_desc, txq->nb_tx_avail, idx,
 969                            TX_PROD(txq));
 970                 return -1;
 971         }
 972
 973         /* Free now */
 974         rte_pktmbuf_free_seg(mbuf);
 975         txq->sw_tx_ring[idx].mbuf = NULL;
 976         ecore_chain_consume(&txq->tx_pbl);
 977         txq->nb_tx_avail++;
 978
 979         return 0;
 980 }
 981
 982 static inline uint16_t
 983 qede_process_tx_compl(struct ecore_dev *edev, struct qede_tx_queue *txq)
 984 {
 985         uint16_t tx_compl = 0;
 986         uint16_t hw_bd_cons;
 987         int rc;
 988
 989         hw_bd_cons = rte_le_to_cpu_16(*txq->hw_cons_ptr);
 990         rte_compiler_barrier();
 991
 992         while (hw_bd_cons != ecore_chain_get_cons_idx(&txq->tx_pbl)) {
 993                 rc = qede_free_tx_pkt(edev, txq);
 994                 if (rc) {
 995                         DP_NOTICE(edev, false,
 996                                   "hw_bd_cons = %d, chain_cons=%d\n",
 997                                   hw_bd_cons,
 998                                   ecore_chain_get_cons_idx(&txq->tx_pbl));
 999                         break;
1000                 }
1001                 txq->sw_tx_cons++;      /* Making TXD available */
1002                 tx_compl++;
1003         }
1004
1005         PMD_TX_LOG(DEBUG, txq, "Tx compl %u sw_tx_cons %u avail %u\n",
1006                    tx_compl, txq->sw_tx_cons, txq->nb_tx_avail);
1007         return tx_compl;
1008 }
1009
1010 uint16_t
1011 qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1012 {
1013         struct qede_tx_queue *txq = p_txq;
1014         struct qede_dev *qdev = txq->qdev;
1015         struct ecore_dev *edev = &qdev->edev;
1016         struct qede_fastpath *fp;
1017         struct eth_tx_1st_bd *first_bd;
1018         uint16_t nb_tx_pkts;
1019         uint16_t nb_pkt_sent = 0;
1020         uint16_t bd_prod;
1021         uint16_t idx;
1022         uint16_t tx_count;
1023
1024         fp = &qdev->fp_array[QEDE_RSS_COUNT(qdev) + txq->queue_id];
1025
1026         if (unlikely(txq->nb_tx_avail < txq->tx_free_thresh)) {
1027                 PMD_TX_LOG(DEBUG, txq, "send=%u avail=%u free_thresh=%u\n",
1028                            nb_pkts, txq->nb_tx_avail, txq->tx_free_thresh);
1029                 (void)qede_process_tx_compl(edev, txq);
1030         }
1031
1032         nb_tx_pkts = RTE_MIN(nb_pkts, (txq->nb_tx_avail / MAX_NUM_TX_BDS));
1033         if (unlikely(nb_tx_pkts == 0)) {
1034                 PMD_TX_LOG(DEBUG, txq, "Out of BDs nb_pkts=%u avail=%u\n",
1035                            nb_pkts, txq->nb_tx_avail);
1036                 return 0;
1037         }
1038
1039         tx_count = nb_tx_pkts;
1040         while (nb_tx_pkts--) {
1041                 /* Fill the entry in the SW ring and the BDs in the FW ring */
1042                 idx = TX_PROD(txq);
1043                 struct rte_mbuf *mbuf = *tx_pkts++;
1044                 txq->sw_tx_ring[idx].mbuf = mbuf;
1045                 first_bd = (struct eth_tx_1st_bd *)
1046                     ecore_chain_produce(&txq->tx_pbl);
1047                 first_bd->data.bd_flags.bitfields =
1048                     1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT;
1049                 /* Map MBUF linear data for DMA and set in the first BD */
1050                 QEDE_BD_SET_ADDR_LEN(first_bd, rte_mbuf_data_dma_addr(mbuf),
1051                                      mbuf->data_len);
1052
1053                 /* Descriptor based VLAN insertion */
1054                 if (mbuf->ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) {
1055                         first_bd->data.vlan = rte_cpu_to_le_16(mbuf->vlan_tci);
1056                         first_bd->data.bd_flags.bitfields |=
1057                             1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT;
1058                 }
1059
1060                 /* Offload the IP checksum in the hardware */
1061                 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) {
1062                         first_bd->data.bd_flags.bitfields |=
1063                             1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT;
1064                 }
1065
1066                 /* L4 checksum offload (tcp or udp) */
1067                 if (mbuf->ol_flags & (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
1068                         first_bd->data.bd_flags.bitfields |=
1069                             1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT;
1070                         /* IPv6 + extn. -> later */
1071                 }
1072                 first_bd->data.nbds = MAX_NUM_TX_BDS;
1073                 txq->sw_tx_prod++;
1074                 rte_prefetch0(txq->sw_tx_ring[TX_PROD(txq)].mbuf);
1075                 txq->nb_tx_avail--;
1076                 bd_prod =
1077                     rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl));
1078                 nb_pkt_sent++;
1079         }
1080
1081         /* Write value of prod idx into bd_prod */
1082         txq->tx_db.data.bd_prod = bd_prod;
1083         rte_wmb();
1084         rte_compiler_barrier();
1085         DIRECT_REG_WR(edev, txq->doorbell_addr, txq->tx_db.raw);
1086         rte_wmb();
1087
1088         /* Check again for Tx completions */
1089         (void)qede_process_tx_compl(edev, txq);
1090
1091         PMD_TX_LOG(DEBUG, txq, "to_send=%u can_send=%u sent=%u core=%d\n",
1092                    nb_pkts, tx_count, nb_pkt_sent, rte_lcore_id());
1093
1094         return nb_pkt_sent;
1095 }
1096
1097 static void qede_init_fp_queue(struct rte_eth_dev *eth_dev)
1098 {
1099         struct qede_dev *qdev = eth_dev->data->dev_private;
1100         struct qede_fastpath *fp;
1101         uint8_t i, rss_id, txq_index, tc;
1102         int rxq = 0, txq = 0;
1103
1104         for_each_queue(i) {
1105                 fp = &qdev->fp_array[i];
1106                 if (fp->type & QEDE_FASTPATH_RX) {
1107                         fp->rxq = eth_dev->data->rx_queues[i];
1108                         fp->rxq->queue_id = rxq++;
1109                 }
1110
1111                 if (fp->type & QEDE_FASTPATH_TX) {
1112                         for (tc = 0; tc < qdev->num_tc; tc++) {
1113                                 txq_index = tc * QEDE_TSS_COUNT(qdev) + txq;
1114                                 fp->txqs[tc] =
1115                                         eth_dev->data->tx_queues[txq_index];
1116                                 fp->txqs[tc]->queue_id = txq_index;
1117                         }
1118                         txq++;
1119                 }
1120         }
1121 }
1122
1123 int qede_dev_start(struct rte_eth_dev *eth_dev)
1124 {
1125         struct qede_dev *qdev = eth_dev->data->dev_private;
1126         struct ecore_dev *edev = &qdev->edev;
1127         struct qed_link_output link_output;
1128         struct qede_fastpath *fp;
1129         int rc, i;
1130
1131         DP_INFO(edev, "Device state is %d\n", qdev->state);
1132
1133         if (qdev->state == QEDE_DEV_START) {
1134                 DP_INFO(edev, "Port is already started\n");
1135                 return 0;
1136         }
1137
1138         if (qdev->state == QEDE_DEV_CONFIG)
1139                 qede_init_fp_queue(eth_dev);
1140
1141         rc = qede_start_queues(eth_dev, true);
1142         if (rc) {
1143                 DP_ERR(edev, "Failed to start queues\n");
1144                 /* TBD: free */
1145                 return rc;
1146         }
1147
1148         /* Bring-up the link */
1149         qede_dev_set_link_state(eth_dev, true);
1150
1151         /* Reset ring */
1152         if (qede_reset_fp_rings(qdev))
1153                 return -ENOMEM;
1154
1155         /* Start/resume traffic */
1156         qdev->ops->fastpath_start(edev);
1157
1158         qdev->state = QEDE_DEV_START;
1159
1160         DP_INFO(edev, "dev_state is QEDE_DEV_START\n");
1161
1162         return 0;
1163 }
1164
1165 static int qede_drain_txq(struct qede_dev *qdev,
1166                           struct qede_tx_queue *txq, bool allow_drain)
1167 {
1168         struct ecore_dev *edev = &qdev->edev;
1169         int rc, cnt = 1000;
1170
1171         while (txq->sw_tx_cons != txq->sw_tx_prod) {
1172                 qede_process_tx_compl(edev, txq);
1173                 if (!cnt) {
1174                         if (allow_drain) {
1175                                 DP_NOTICE(edev, false,
1176                                           "Tx queue[%u] is stuck,"
1177                                           "requesting MCP to drain\n",
1178                                           txq->queue_id);
1179                                 rc = qdev->ops->common->drain(edev);
1180                                 if (rc)
1181                                         return rc;
1182                                 return qede_drain_txq(qdev, txq, false);
1183                         }
1184
1185                         DP_NOTICE(edev, false,
1186                                   "Timeout waiting for tx queue[%d]:"
1187                                   "PROD=%d, CONS=%d\n",
1188                                   txq->queue_id, txq->sw_tx_prod,
1189                                   txq->sw_tx_cons);
1190                         return -ENODEV;
1191                 }
1192                 cnt--;
1193                 DELAY(1000);
1194                 rte_compiler_barrier();
1195         }
1196
1197         /* FW finished processing, wait for HW to transmit all tx packets */
1198         DELAY(2000);
1199
1200         return 0;
1201 }
1202
1203 static int qede_stop_queues(struct qede_dev *qdev)
1204 {
1205         struct qed_update_vport_params vport_update_params;
1206         struct ecore_dev *edev = &qdev->edev;
1207         int rc, tc, i;
1208
1209         /* Disable the vport */
1210         memset(&vport_update_params, 0, sizeof(vport_update_params));
1211         vport_update_params.vport_id = 0;
1212         vport_update_params.update_vport_active_flg = 1;
1213         vport_update_params.vport_active_flg = 0;
1214         vport_update_params.update_rss_flg = 0;
1215
1216         DP_INFO(edev, "Deactivate vport\n");
1217
1218         rc = qdev->ops->vport_update(edev, &vport_update_params);
1219         if (rc) {
1220                 DP_ERR(edev, "Failed to update vport\n");
1221                 return rc;
1222         }
1223
1224         DP_INFO(edev, "Flushing tx queues\n");
1225
1226         /* Flush Tx queues. If needed, request drain from MCP */
1227         for_each_queue(i) {
1228                 struct qede_fastpath *fp = &qdev->fp_array[i];
1229
1230                 if (fp->type & QEDE_FASTPATH_TX) {
1231                         for (tc = 0; tc < qdev->num_tc; tc++) {
1232                                 struct qede_tx_queue *txq = fp->txqs[tc];
1233
1234                                 rc = qede_drain_txq(qdev, txq, true);
1235                                 if (rc)
1236                                         return rc;
1237                         }
1238                 }
1239         }
1240
1241         /* Stop all Queues in reverse order */
1242         for (i = QEDE_QUEUE_CNT(qdev) - 1; i >= 0; i--) {
1243                 struct qed_stop_rxq_params rx_params;
1244
1245                 /* Stop the Tx Queue(s) */
1246                 if (qdev->fp_array[i].type & QEDE_FASTPATH_TX) {
1247                         for (tc = 0; tc < qdev->num_tc; tc++) {
1248                                 struct qed_stop_txq_params tx_params;
1249                                 u8 val;
1250
1251                                 tx_params.rss_id = i;
1252                                 val = qdev->fp_array[i].txqs[tc]->queue_id;
1253                                 tx_params.tx_queue_id = val;
1254
1255                                 DP_INFO(edev, "Stopping tx queues\n");
1256                                 rc = qdev->ops->q_tx_stop(edev, &tx_params);
1257                                 if (rc) {
1258                                         DP_ERR(edev, "Failed to stop TXQ #%d\n",
1259                                                tx_params.tx_queue_id);
1260                                         return rc;
1261                                 }
1262                         }
1263                 }
1264
1265                 /* Stop the Rx Queue */
1266                 if (qdev->fp_array[i].type & QEDE_FASTPATH_RX) {
1267                         memset(&rx_params, 0, sizeof(rx_params));
1268                         rx_params.rss_id = i;
1269                         rx_params.rx_queue_id = qdev->fp_array[i].rxq->queue_id;
1270                         rx_params.eq_completion_only = 1;
1271
1272                         DP_INFO(edev, "Stopping rx queues\n");
1273
1274                         rc = qdev->ops->q_rx_stop(edev, &rx_params);
1275                         if (rc) {
1276                                 DP_ERR(edev, "Failed to stop RXQ #%d\n", i);
1277                                 return rc;
1278                         }
1279                 }
1280         }
1281
1282         return 0;
1283 }
1284
1285 int qede_reset_fp_rings(struct qede_dev *qdev)
1286 {
1287         struct qede_fastpath *fp;
1288         struct qede_tx_queue *txq;
1289         uint8_t tc;
1290         uint16_t id, i;
1291
1292         for_each_queue(id) {
1293                 fp = &qdev->fp_array[id];
1294
1295                 if (fp->type & QEDE_FASTPATH_RX) {
1296                         DP_INFO(&qdev->edev,
1297                                 "Reset FP chain for RSS %u\n", id);
1298                         qede_rx_queue_release_mbufs(fp->rxq);
1299                         ecore_chain_reset(&fp->rxq->rx_bd_ring);
1300                         ecore_chain_reset(&fp->rxq->rx_comp_ring);
1301                         fp->rxq->sw_rx_prod = 0;
1302                         fp->rxq->sw_rx_cons = 0;
1303                         *fp->rxq->hw_cons_ptr = 0;
1304                         for (i = 0; i < fp->rxq->nb_rx_desc; i++) {
1305                                 if (qede_alloc_rx_buffer(fp->rxq)) {
1306                                         DP_ERR(&qdev->edev,
1307                                                "RX buffer allocation failed\n");
1308                                         return -ENOMEM;
1309                                 }
1310                         }
1311                 }
1312                 if (fp->type & QEDE_FASTPATH_TX) {
1313                         for (tc = 0; tc < qdev->num_tc; tc++) {
1314                                 txq = fp->txqs[tc];
1315                                 qede_tx_queue_release_mbufs(txq);
1316                                 ecore_chain_reset(&txq->tx_pbl);
1317                                 txq->sw_tx_cons = 0;
1318                                 txq->sw_tx_prod = 0;
1319                                 *txq->hw_cons_ptr = 0;
1320                         }
1321                 }
1322         }
1323
1324         return 0;
1325 }
1326
1327 /* This function frees all memory of a single fp */
1328 void qede_free_mem_load(struct rte_eth_dev *eth_dev)
1329 {
1330         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
1331         struct qede_fastpath *fp;
1332         uint16_t txq_idx;
1333         uint8_t id;
1334         uint8_t tc;
1335
1336         for_each_queue(id) {
1337                 fp = &qdev->fp_array[id];
1338                 if (fp->type & QEDE_FASTPATH_RX) {
1339                         qede_rx_queue_release(fp->rxq);
1340                         eth_dev->data->rx_queues[id] = NULL;
1341                 } else {
1342                         for (tc = 0; tc < qdev->num_tc; tc++) {
1343                                 txq_idx = fp->txqs[tc]->queue_id;
1344                                 qede_tx_queue_release(fp->txqs[tc]);
1345                                 eth_dev->data->tx_queues[txq_idx] = NULL;
1346                         }
1347                 }
1348         }
1349 }
1350
1351 void qede_dev_stop(struct rte_eth_dev *eth_dev)
1352 {
1353         struct qede_dev *qdev = eth_dev->data->dev_private;
1354         struct ecore_dev *edev = &qdev->edev;
1355
1356         DP_INFO(edev, "port %u\n", eth_dev->data->port_id);
1357
1358         if (qdev->state != QEDE_DEV_START) {
1359                 DP_INFO(edev, "Device not yet started\n");
1360                 return;
1361         }
1362
1363         if (qede_stop_queues(qdev))
1364                 DP_ERR(edev, "Didn't succeed to close queues\n");
1365
1366         DP_INFO(edev, "Stopped queues\n");
1367
1368         qdev->ops->fastpath_stop(edev);
1369
1370         /* Bring the link down */
1371         qede_dev_set_link_state(eth_dev, false);
1372
1373         qdev->state = QEDE_DEV_STOP;
1374
1375         DP_INFO(edev, "dev_state is QEDE_DEV_STOP\n");
1376 }