drivers/net/qede/qede_rxtx.c

   1 /*
   2  * Copyright (c) 2016 QLogic Corporation.
   3  * All rights reserved.
   4  * www.qlogic.com
   5  *
   6  * See LICENSE.qede_pmd for copyright and licensing details.
   7  */
   8
   9 #include "qede_rxtx.h"
  10
  11 static bool gro_disable = 1;    /* mod_param */
  12
  13 #define QEDE_FASTPATH_TX        (1 << 0)
  14 #define QEDE_FASTPATH_RX        (1 << 1)
  15
  16 static inline int qede_alloc_rx_buffer(struct qede_rx_queue *rxq)
  17 {
  18         struct rte_mbuf *new_mb = NULL;
  19         struct eth_rx_bd *rx_bd;
  20         dma_addr_t mapping;
  21         uint16_t idx = rxq->sw_rx_prod & NUM_RX_BDS(rxq);
  22
  23         new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
  24         if (unlikely(!new_mb)) {
  25                 PMD_RX_LOG(ERR, rxq,
  26                            "Failed to allocate rx buffer "
  27                            "sw_rx_prod %u sw_rx_cons %u mp entries %u free %u",
  28                            idx, rxq->sw_rx_cons & NUM_RX_BDS(rxq),
  29                            rte_mempool_avail_count(rxq->mb_pool),
  30                            rte_mempool_in_use_count(rxq->mb_pool));
  31                 return -ENOMEM;
  32         }
  33         rxq->sw_rx_ring[idx].mbuf = new_mb;
  34         rxq->sw_rx_ring[idx].page_offset = 0;
  35         mapping = rte_mbuf_data_dma_addr_default(new_mb);
  36         /* Advance PROD and get BD pointer */
  37         rx_bd = (struct eth_rx_bd *)ecore_chain_produce(&rxq->rx_bd_ring);
  38         rx_bd->addr.hi = rte_cpu_to_le_32(U64_HI(mapping));
  39         rx_bd->addr.lo = rte_cpu_to_le_32(U64_LO(mapping));
  40         rxq->sw_rx_prod++;
  41         return 0;
  42 }
  43
  44 static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq)
  45 {
  46         uint16_t i;
  47
  48         if (rxq->sw_rx_ring != NULL) {
  49                 for (i = 0; i < rxq->nb_rx_desc; i++) {
  50                         if (rxq->sw_rx_ring[i].mbuf != NULL) {
  51                                 rte_pktmbuf_free(rxq->sw_rx_ring[i].mbuf);
  52                                 rxq->sw_rx_ring[i].mbuf = NULL;
  53                         }
  54                 }
  55         }
  56 }
  57
  58 void qede_rx_queue_release(void *rx_queue)
  59 {
  60         struct qede_rx_queue *rxq = rx_queue;
  61
  62         if (rxq != NULL) {
  63                 qede_rx_queue_release_mbufs(rxq);
  64                 rte_free(rxq->sw_rx_ring);
  65                 rxq->sw_rx_ring = NULL;
  66                 rte_free(rxq);
  67                 rx_queue = NULL;
  68         }
  69 }
  70
  71 static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq)
  72 {
  73         unsigned int i;
  74
  75         PMD_TX_LOG(DEBUG, txq, "releasing %u mbufs\n", txq->nb_tx_desc);
  76
  77         if (txq->sw_tx_ring) {
  78                 for (i = 0; i < txq->nb_tx_desc; i++) {
  79                         if (txq->sw_tx_ring[i].mbuf) {
  80                                 rte_pktmbuf_free(txq->sw_tx_ring[i].mbuf);
  81                                 txq->sw_tx_ring[i].mbuf = NULL;
  82                         }
  83                 }
  84         }
  85 }
  86
  87 int
  88 qede_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
  89                     uint16_t nb_desc, unsigned int socket_id,
  90                     const struct rte_eth_rxconf *rx_conf,
  91                     struct rte_mempool *mp)
  92 {
  93         struct qede_dev *qdev = dev->data->dev_private;
  94         struct ecore_dev *edev = &qdev->edev;
  95         struct rte_eth_dev_data *eth_data = dev->data;
  96         struct qede_rx_queue *rxq;
  97         uint16_t pkt_len = (uint16_t)dev->data->dev_conf.rxmode.max_rx_pkt_len;
  98         size_t size;
  99         uint16_t data_size;
 100         int rc;
 101         int i;
 102
 103         PMD_INIT_FUNC_TRACE(edev);
 104
 105         /* Note: Ring size/align is controlled by struct rte_eth_desc_lim */
 106         if (!rte_is_power_of_2(nb_desc)) {
 107                 DP_ERR(edev, "Ring size %u is not power of 2\n",
 108                           nb_desc);
 109                 return -EINVAL;
 110         }
 111
 112         /* Free memory prior to re-allocation if needed... */
 113         if (dev->data->rx_queues[queue_idx] != NULL) {
 114                 qede_rx_queue_release(dev->data->rx_queues[queue_idx]);
 115                 dev->data->rx_queues[queue_idx] = NULL;
 116         }
 117
 118         /* First allocate the rx queue data structure */
 119         rxq = rte_zmalloc_socket("qede_rx_queue", sizeof(struct qede_rx_queue),
 120                                  RTE_CACHE_LINE_SIZE, socket_id);
 121
 122         if (!rxq) {
 123                 DP_ERR(edev, "Unable to allocate memory for rxq on socket %u",
 124                           socket_id);
 125                 return -ENOMEM;
 126         }
 127
 128         rxq->qdev = qdev;
 129         rxq->mb_pool = mp;
 130         rxq->nb_rx_desc = nb_desc;
 131         rxq->queue_id = queue_idx;
 132         rxq->port_id = dev->data->port_id;
 133
 134         /* Sanity check */
 135         data_size = (uint16_t)rte_pktmbuf_data_room_size(mp) -
 136                                 RTE_PKTMBUF_HEADROOM;
 137
 138         qdev->mtu = pkt_len;
 139         rxq->rx_buf_size = data_size;
 140
 141         DP_INFO(edev, "MTU = %u ; RX buffer = %u\n",
 142                 qdev->mtu, rxq->rx_buf_size);
 143
 144         if (pkt_len > ETHER_MAX_LEN) {
 145                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
 146                 DP_NOTICE(edev, false, "jumbo frame enabled\n");
 147         } else {
 148                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
 149         }
 150
 151         /* Allocate the parallel driver ring for Rx buffers */
 152         size = sizeof(*rxq->sw_rx_ring) * rxq->nb_rx_desc;
 153         rxq->sw_rx_ring = rte_zmalloc_socket("sw_rx_ring", size,
 154                                              RTE_CACHE_LINE_SIZE, socket_id);
 155         if (!rxq->sw_rx_ring) {
 156                 DP_NOTICE(edev, false,
 157                           "Unable to alloc memory for sw_rx_ring on socket %u\n",
 158                           socket_id);
 159                 rte_free(rxq);
 160                 rxq = NULL;
 161                 return -ENOMEM;
 162         }
 163
 164         /* Allocate FW Rx ring  */
 165         rc = qdev->ops->common->chain_alloc(edev,
 166                                             ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
 167                                             ECORE_CHAIN_MODE_NEXT_PTR,
 168                                             ECORE_CHAIN_CNT_TYPE_U16,
 169                                             rxq->nb_rx_desc,
 170                                             sizeof(struct eth_rx_bd),
 171                                             &rxq->rx_bd_ring);
 172
 173         if (rc != ECORE_SUCCESS) {
 174                 DP_NOTICE(edev, false,
 175                           "Unable to alloc memory for rxbd ring on socket %u\n",
 176                           socket_id);
 177                 rte_free(rxq->sw_rx_ring);
 178                 rxq->sw_rx_ring = NULL;
 179                 rte_free(rxq);
 180                 rxq = NULL;
 181                 return -ENOMEM;
 182         }
 183
 184         /* Allocate FW completion ring */
 185         rc = qdev->ops->common->chain_alloc(edev,
 186                                             ECORE_CHAIN_USE_TO_CONSUME,
 187                                             ECORE_CHAIN_MODE_PBL,
 188                                             ECORE_CHAIN_CNT_TYPE_U16,
 189                                             rxq->nb_rx_desc,
 190                                             sizeof(union eth_rx_cqe),
 191                                             &rxq->rx_comp_ring);
 192
 193         if (rc != ECORE_SUCCESS) {
 194                 DP_NOTICE(edev, false,
 195                           "Unable to alloc memory for cqe ring on socket %u\n",
 196                           socket_id);
 197                 /* TBD: Freeing RX BD ring */
 198                 rte_free(rxq->sw_rx_ring);
 199                 rxq->sw_rx_ring = NULL;
 200                 rte_free(rxq);
 201                 return -ENOMEM;
 202         }
 203
 204         /* Allocate buffers for the Rx ring */
 205         for (i = 0; i < rxq->nb_rx_desc; i++) {
 206                 rc = qede_alloc_rx_buffer(rxq);
 207                 if (rc) {
 208                         DP_NOTICE(edev, false,
 209                                   "RX buffer allocation failed at idx=%d\n", i);
 210                         goto err4;
 211                 }
 212         }
 213
 214         dev->data->rx_queues[queue_idx] = rxq;
 215
 216         DP_INFO(edev, "rxq %d num_desc %u rx_buf_size=%u socket %u\n",
 217                   queue_idx, nb_desc, qdev->mtu, socket_id);
 218
 219         return 0;
 220 err4:
 221         qede_rx_queue_release(rxq);
 222         return -ENOMEM;
 223 }
 224
 225 void qede_tx_queue_release(void *tx_queue)
 226 {
 227         struct qede_tx_queue *txq = tx_queue;
 228
 229         if (txq != NULL) {
 230                 qede_tx_queue_release_mbufs(txq);
 231                 if (txq->sw_tx_ring) {
 232                         rte_free(txq->sw_tx_ring);
 233                         txq->sw_tx_ring = NULL;
 234                 }
 235                 rte_free(txq);
 236         }
 237         tx_queue = NULL;
 238 }
 239
 240 int
 241 qede_tx_queue_setup(struct rte_eth_dev *dev,
 242                     uint16_t queue_idx,
 243                     uint16_t nb_desc,
 244                     unsigned int socket_id,
 245                     const struct rte_eth_txconf *tx_conf)
 246 {
 247         struct qede_dev *qdev = dev->data->dev_private;
 248         struct ecore_dev *edev = &qdev->edev;
 249         struct qede_tx_queue *txq;
 250         int rc;
 251
 252         PMD_INIT_FUNC_TRACE(edev);
 253
 254         if (!rte_is_power_of_2(nb_desc)) {
 255                 DP_ERR(edev, "Ring size %u is not power of 2\n",
 256                        nb_desc);
 257                 return -EINVAL;
 258         }
 259
 260         /* Free memory prior to re-allocation if needed... */
 261         if (dev->data->tx_queues[queue_idx] != NULL) {
 262                 qede_tx_queue_release(dev->data->tx_queues[queue_idx]);
 263                 dev->data->tx_queues[queue_idx] = NULL;
 264         }
 265
 266         txq = rte_zmalloc_socket("qede_tx_queue", sizeof(struct qede_tx_queue),
 267                                  RTE_CACHE_LINE_SIZE, socket_id);
 268
 269         if (txq == NULL) {
 270                 DP_ERR(edev,
 271                        "Unable to allocate memory for txq on socket %u",
 272                        socket_id);
 273                 return -ENOMEM;
 274         }
 275
 276         txq->nb_tx_desc = nb_desc;
 277         txq->qdev = qdev;
 278         txq->port_id = dev->data->port_id;
 279
 280         rc = qdev->ops->common->chain_alloc(edev,
 281                                             ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
 282                                             ECORE_CHAIN_MODE_PBL,
 283                                             ECORE_CHAIN_CNT_TYPE_U16,
 284                                             txq->nb_tx_desc,
 285                                             sizeof(union eth_tx_bd_types),
 286                                             &txq->tx_pbl);
 287         if (rc != ECORE_SUCCESS) {
 288                 DP_ERR(edev,
 289                        "Unable to allocate memory for txbd ring on socket %u",
 290                        socket_id);
 291                 qede_tx_queue_release(txq);
 292                 return -ENOMEM;
 293         }
 294
 295         /* Allocate software ring */
 296         txq->sw_tx_ring = rte_zmalloc_socket("txq->sw_tx_ring",
 297                                              (sizeof(struct qede_tx_entry) *
 298                                               txq->nb_tx_desc),
 299                                              RTE_CACHE_LINE_SIZE, socket_id);
 300
 301         if (!txq->sw_tx_ring) {
 302                 DP_ERR(edev,
 303                        "Unable to allocate memory for txbd ring on socket %u",
 304                        socket_id);
 305                 qede_tx_queue_release(txq);
 306                 return -ENOMEM;
 307         }
 308
 309         txq->queue_id = queue_idx;
 310
 311         txq->nb_tx_avail = txq->nb_tx_desc;
 312
 313         txq->tx_free_thresh =
 314             tx_conf->tx_free_thresh ? tx_conf->tx_free_thresh :
 315             (txq->nb_tx_desc - QEDE_DEFAULT_TX_FREE_THRESH);
 316
 317         dev->data->tx_queues[queue_idx] = txq;
 318
 319         DP_INFO(edev,
 320                   "txq %u num_desc %u tx_free_thresh %u socket %u\n",
 321                   queue_idx, nb_desc, txq->tx_free_thresh, socket_id);
 322
 323         return 0;
 324 }
 325
 326 /* This function inits fp content and resets the SB, RXQ and TXQ arrays */
 327 static void qede_init_fp(struct qede_dev *qdev)
 328 {
 329         struct qede_fastpath *fp;
 330         uint8_t i, rss_id, tc;
 331         int fp_rx = qdev->fp_num_rx, rxq = 0, txq = 0;
 332
 333         memset((void *)qdev->fp_array, 0, (QEDE_QUEUE_CNT(qdev) *
 334                                            sizeof(*qdev->fp_array)));
 335         memset((void *)qdev->sb_array, 0, (QEDE_QUEUE_CNT(qdev) *
 336                                            sizeof(*qdev->sb_array)));
 337         for_each_queue(i) {
 338                 fp = &qdev->fp_array[i];
 339                 if (fp_rx) {
 340                         fp->type = QEDE_FASTPATH_RX;
 341                         fp_rx--;
 342                 } else{
 343                         fp->type = QEDE_FASTPATH_TX;
 344                 }
 345                 fp->qdev = qdev;
 346                 fp->id = i;
 347                 fp->sb_info = &qdev->sb_array[i];
 348                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", "qdev", i);
 349         }
 350
 351         qdev->gro_disable = gro_disable;
 352 }
 353
 354 void qede_free_fp_arrays(struct qede_dev *qdev)
 355 {
 356         /* It asseumes qede_free_mem_load() is called before */
 357         if (qdev->fp_array != NULL) {
 358                 rte_free(qdev->fp_array);
 359                 qdev->fp_array = NULL;
 360         }
 361
 362         if (qdev->sb_array != NULL) {
 363                 rte_free(qdev->sb_array);
 364                 qdev->sb_array = NULL;
 365         }
 366 }
 367
 368 int qede_alloc_fp_array(struct qede_dev *qdev)
 369 {
 370         struct qede_fastpath *fp;
 371         struct ecore_dev *edev = &qdev->edev;
 372         int i;
 373
 374         qdev->fp_array = rte_calloc("fp", QEDE_QUEUE_CNT(qdev),
 375                                     sizeof(*qdev->fp_array),
 376                                     RTE_CACHE_LINE_SIZE);
 377
 378         if (!qdev->fp_array) {
 379                 DP_ERR(edev, "fp array allocation failed\n");
 380                 return -ENOMEM;
 381         }
 382
 383         qdev->sb_array = rte_calloc("sb", QEDE_QUEUE_CNT(qdev),
 384                                     sizeof(*qdev->sb_array),
 385                                     RTE_CACHE_LINE_SIZE);
 386
 387         if (!qdev->sb_array) {
 388                 DP_ERR(edev, "sb array allocation failed\n");
 389                 rte_free(qdev->fp_array);
 390                 return -ENOMEM;
 391         }
 392
 393         return 0;
 394 }
 395
 396 /* This function allocates fast-path status block memory */
 397 static int
 398 qede_alloc_mem_sb(struct qede_dev *qdev, struct ecore_sb_info *sb_info,
 399                   uint16_t sb_id)
 400 {
 401         struct ecore_dev *edev = &qdev->edev;
 402         struct status_block *sb_virt;
 403         dma_addr_t sb_phys;
 404         int rc;
 405
 406         sb_virt = OSAL_DMA_ALLOC_COHERENT(edev, &sb_phys, sizeof(*sb_virt));
 407
 408         if (!sb_virt) {
 409                 DP_ERR(edev, "Status block allocation failed\n");
 410                 return -ENOMEM;
 411         }
 412
 413         rc = qdev->ops->common->sb_init(edev, sb_info,
 414                                         sb_virt, sb_phys, sb_id,
 415                                         QED_SB_TYPE_L2_QUEUE);
 416         if (rc) {
 417                 DP_ERR(edev, "Status block initialization failed\n");
 418                 /* TBD: No dma_free_coherent possible */
 419                 return rc;
 420         }
 421
 422         return 0;
 423 }
 424
 425 int qede_alloc_fp_resc(struct qede_dev *qdev)
 426 {
 427         struct qede_fastpath *fp;
 428         int rc, i;
 429
 430         if (qdev->fp_array)
 431                 qede_free_fp_arrays(qdev);
 432
 433         rc = qede_alloc_fp_array(qdev);
 434         if (rc != 0)
 435                 return rc;
 436
 437         qede_init_fp(qdev);
 438
 439         for (i = 0; i < QEDE_QUEUE_CNT(qdev); i++) {
 440                 fp = &qdev->fp_array[i];
 441                 if (qede_alloc_mem_sb(qdev, fp->sb_info, i)) {
 442                         qede_free_fp_arrays(qdev);
 443                         return -ENOMEM;
 444                 }
 445         }
 446
 447         return 0;
 448 }
 449
 450 void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev)
 451 {
 452         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
 453
 454         qede_free_mem_load(eth_dev);
 455         qede_free_fp_arrays(qdev);
 456 }
 457
 458 static inline void
 459 qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq)
 460 {
 461         uint16_t bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring);
 462         uint16_t cqe_prod = ecore_chain_get_prod_idx(&rxq->rx_comp_ring);
 463         struct eth_rx_prod_data rx_prods = { 0 };
 464
 465         /* Update producers */
 466         rx_prods.bd_prod = rte_cpu_to_le_16(bd_prod);
 467         rx_prods.cqe_prod = rte_cpu_to_le_16(cqe_prod);
 468
 469         /* Make sure that the BD and SGE data is updated before updating the
 470          * producers since FW might read the BD/SGE right after the producer
 471          * is updated.
 472          */
 473         rte_wmb();
 474
 475         internal_ram_wr(rxq->hw_rxq_prod_addr, sizeof(rx_prods),
 476                         (uint32_t *)&rx_prods);
 477
 478         /* mmiowb is needed to synchronize doorbell writes from more than one
 479          * processor. It guarantees that the write arrives to the device before
 480          * the napi lock is released and another qede_poll is called (possibly
 481          * on another CPU). Without this barrier, the next doorbell can bypass
 482          * this doorbell. This is applicable to IA64/Altix systems.
 483          */
 484         rte_wmb();
 485
 486         PMD_RX_LOG(DEBUG, rxq, "bd_prod %u  cqe_prod %u\n", bd_prod, cqe_prod);
 487 }
 488
 489 static inline uint32_t
 490 qede_rxfh_indir_default(uint32_t index, uint32_t n_rx_rings)
 491 {
 492         return index % n_rx_rings;
 493 }
 494
 495 static void qede_prandom_bytes(uint32_t *buff, size_t bytes)
 496 {
 497         unsigned int i;
 498
 499         srand((unsigned int)time(NULL));
 500
 501         for (i = 0; i < ECORE_RSS_KEY_SIZE; i++)
 502                 buff[i] = rand();
 503 }
 504
 505 static int
 506 qede_config_rss(struct rte_eth_dev *eth_dev,
 507                 struct qed_update_vport_rss_params *rss_params)
 508 {
 509         struct rte_eth_rss_conf rss_conf;
 510         enum rte_eth_rx_mq_mode mode = eth_dev->data->dev_conf.rxmode.mq_mode;
 511         struct qede_dev *qdev = eth_dev->data->dev_private;
 512         struct ecore_dev *edev = &qdev->edev;
 513         uint8_t rss_caps;
 514         unsigned int i;
 515         uint64_t hf;
 516         uint32_t *key;
 517
 518         rss_conf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 519         key = (uint32_t *)rss_conf.rss_key;
 520         hf = rss_conf.rss_hf;
 521         PMD_INIT_FUNC_TRACE(edev);
 522
 523         /* Check if RSS conditions are met.
 524          * Note: Even though its meaningless to enable RSS with one queue, it
 525          * could be used to produce RSS Hash, so skipping that check.
 526          */
 527
 528         if (!(mode & ETH_MQ_RX_RSS)) {
 529                 DP_INFO(edev, "RSS flag is not set\n");
 530                 return -EINVAL;
 531         }
 532
 533         DP_INFO(edev, "RSS flag is set\n");
 534
 535         if (rss_conf.rss_hf == 0)
 536                 DP_NOTICE(edev, false, "RSS hash function = 0, disables RSS\n");
 537
 538         if (rss_conf.rss_key != NULL)
 539                 memcpy(qdev->rss_params.rss_key, rss_conf.rss_key,
 540                        rss_conf.rss_key_len);
 541
 542         memset(rss_params, 0, sizeof(*rss_params));
 543
 544         for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++)
 545                 rss_params->rss_ind_table[i] = qede_rxfh_indir_default(i,
 546                                                         QEDE_RSS_CNT(qdev));
 547
 548         /* key and protocols */
 549         if (rss_conf.rss_key == NULL)
 550                 qede_prandom_bytes(rss_params->rss_key,
 551                                    sizeof(rss_params->rss_key));
 552         else
 553                 memcpy(rss_params->rss_key, rss_conf.rss_key,
 554                        rss_conf.rss_key_len);
 555
 556         rss_caps = 0;
 557         rss_caps |= (hf & ETH_RSS_IPV4)              ? ECORE_RSS_IPV4 : 0;
 558         rss_caps |= (hf & ETH_RSS_IPV6)              ? ECORE_RSS_IPV6 : 0;
 559         rss_caps |= (hf & ETH_RSS_IPV6_EX)           ? ECORE_RSS_IPV6 : 0;
 560         rss_caps |= (hf & ETH_RSS_NONFRAG_IPV4_TCP)  ? ECORE_RSS_IPV4_TCP : 0;
 561         rss_caps |= (hf & ETH_RSS_NONFRAG_IPV6_TCP)  ? ECORE_RSS_IPV6_TCP : 0;
 562         rss_caps |= (hf & ETH_RSS_IPV6_TCP_EX)       ? ECORE_RSS_IPV6_TCP : 0;
 563
 564         rss_params->rss_caps = rss_caps;
 565
 566         DP_INFO(edev, "RSS check passes\n");
 567
 568         return 0;
 569 }
 570
 571 static int qede_start_queues(struct rte_eth_dev *eth_dev, bool clear_stats)
 572 {
 573         struct qede_dev *qdev = eth_dev->data->dev_private;
 574         struct ecore_dev *edev = &qdev->edev;
 575         struct qed_update_vport_rss_params *rss_params = &qdev->rss_params;
 576         struct qed_dev_info *qed_info = &qdev->dev_info.common;
 577         struct qed_update_vport_params vport_update_params;
 578         struct qede_tx_queue *txq;
 579         struct qede_fastpath *fp;
 580         dma_addr_t p_phys_table;
 581         int txq_index;
 582         uint16_t page_cnt;
 583         int vlan_removal_en = 1;
 584         int rc, tc, i;
 585
 586         for_each_queue(i) {
 587                 fp = &qdev->fp_array[i];
 588                 if (fp->type & QEDE_FASTPATH_RX) {
 589                         p_phys_table = ecore_chain_get_pbl_phys(&fp->rxq->
 590                                                                 rx_comp_ring);
 591                         page_cnt = ecore_chain_get_page_cnt(&fp->rxq->
 592                                                                 rx_comp_ring);
 593
 594                         ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
 595
 596                         rc = qdev->ops->q_rx_start(edev, i, fp->rxq->queue_id,
 597                                            0,
 598                                            fp->sb_info->igu_sb_id,
 599                                            RX_PI,
 600                                            fp->rxq->rx_buf_size,
 601                                            fp->rxq->rx_bd_ring.p_phys_addr,
 602                                            p_phys_table,
 603                                            page_cnt,
 604                                            &fp->rxq->hw_rxq_prod_addr);
 605                         if (rc) {
 606                                 DP_ERR(edev, "Start rxq #%d failed %d\n",
 607                                        fp->rxq->queue_id, rc);
 608                                 return rc;
 609                         }
 610
 611                         fp->rxq->hw_cons_ptr =
 612                                         &fp->sb_info->sb_virt->pi_array[RX_PI];
 613
 614                         qede_update_rx_prod(qdev, fp->rxq);
 615                 }
 616
 617                 if (!(fp->type & QEDE_FASTPATH_TX))
 618                         continue;
 619                 for (tc = 0; tc < qdev->num_tc; tc++) {
 620                         txq = fp->txqs[tc];
 621                         txq_index = tc * QEDE_RSS_CNT(qdev) + i;
 622
 623                         p_phys_table = ecore_chain_get_pbl_phys(&txq->tx_pbl);
 624                         page_cnt = ecore_chain_get_page_cnt(&txq->tx_pbl);
 625                         rc = qdev->ops->q_tx_start(edev, i, txq->queue_id,
 626                                                    0,
 627                                                    fp->sb_info->igu_sb_id,
 628                                                    TX_PI(tc),
 629                                                    p_phys_table, page_cnt,
 630                                                    &txq->doorbell_addr);
 631                         if (rc) {
 632                                 DP_ERR(edev, "Start txq %u failed %d\n",
 633                                        txq_index, rc);
 634                                 return rc;
 635                         }
 636
 637                         txq->hw_cons_ptr =
 638                             &fp->sb_info->sb_virt->pi_array[TX_PI(tc)];
 639                         SET_FIELD(txq->tx_db.data.params,
 640                                   ETH_DB_DATA_DEST, DB_DEST_XCM);
 641                         SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD,
 642                                   DB_AGG_CMD_SET);
 643                         SET_FIELD(txq->tx_db.data.params,
 644                                   ETH_DB_DATA_AGG_VAL_SEL,
 645                                   DQ_XCM_ETH_TX_BD_PROD_CMD);
 646
 647                         txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD;
 648                 }
 649         }
 650
 651         /* Prepare and send the vport enable */
 652         memset(&vport_update_params, 0, sizeof(vport_update_params));
 653         vport_update_params.vport_id = 0;
 654         vport_update_params.update_vport_active_flg = 1;
 655         vport_update_params.vport_active_flg = 1;
 656
 657         /* @DPDK */
 658         if (qed_info->mf_mode == MF_NPAR && qed_info->tx_switching) {
 659                 /* TBD: Check SRIOV enabled for VF */
 660                 vport_update_params.update_tx_switching_flg = 1;
 661                 vport_update_params.tx_switching_flg = 1;
 662         }
 663
 664         if (!qede_config_rss(eth_dev, rss_params)) {
 665                 vport_update_params.update_rss_flg = 1;
 666
 667                 qdev->rss_enabled = 1;
 668                 DP_INFO(edev, "Updating RSS flag\n");
 669         } else {
 670                 qdev->rss_enabled = 0;
 671                 DP_INFO(edev, "Not Updating RSS flag\n");
 672         }
 673
 674         rte_memcpy(&vport_update_params.rss_params, rss_params,
 675                sizeof(*rss_params));
 676
 677         rc = qdev->ops->vport_update(edev, &vport_update_params);
 678         if (rc) {
 679                 DP_ERR(edev, "Update V-PORT failed %d\n", rc);
 680                 return rc;
 681         }
 682
 683         return 0;
 684 }
 685
 686 #ifdef ENC_SUPPORTED
 687 static bool qede_tunn_exist(uint16_t flag)
 688 {
 689         return !!((PARSING_AND_ERR_FLAGS_TUNNELEXIST_MASK <<
 690                     PARSING_AND_ERR_FLAGS_TUNNELEXIST_SHIFT) & flag);
 691 }
 692
 693 static inline uint8_t qede_check_tunn_csum(uint16_t flag)
 694 {
 695         uint8_t tcsum = 0;
 696         uint16_t csum_flag = 0;
 697
 698         if ((PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_MASK <<
 699              PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMWASCALCULATED_SHIFT) & flag)
 700                 csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_MASK <<
 701                     PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT;
 702
 703         if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK <<
 704              PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) {
 705                 csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK <<
 706                     PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT;
 707                 tcsum = QEDE_TUNN_CSUM_UNNECESSARY;
 708         }
 709
 710         csum_flag |= PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_MASK <<
 711             PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_SHIFT |
 712             PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK <<
 713             PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT;
 714
 715         if (csum_flag & flag)
 716                 return QEDE_CSUM_ERROR;
 717
 718         return QEDE_CSUM_UNNECESSARY | tcsum;
 719 }
 720 #else
 721 static inline uint8_t qede_tunn_exist(uint16_t flag)
 722 {
 723         return 0;
 724 }
 725
 726 static inline uint8_t qede_check_tunn_csum(uint16_t flag)
 727 {
 728         return 0;
 729 }
 730 #endif
 731
 732 static inline uint8_t qede_check_notunn_csum(uint16_t flag)
 733 {
 734         uint8_t csum = 0;
 735         uint16_t csum_flag = 0;
 736
 737         if ((PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK <<
 738              PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & flag) {
 739                 csum_flag |= PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK <<
 740                     PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT;
 741                 csum = QEDE_CSUM_UNNECESSARY;
 742         }
 743
 744         csum_flag |= PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK <<
 745             PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT;
 746
 747         if (csum_flag & flag)
 748                 return QEDE_CSUM_ERROR;
 749
 750         return csum;
 751 }
 752
 753 static inline uint8_t qede_check_csum(uint16_t flag)
 754 {
 755         if (likely(!qede_tunn_exist(flag)))
 756                 return qede_check_notunn_csum(flag);
 757         else
 758                 return qede_check_tunn_csum(flag);
 759 }
 760
 761 static inline void qede_rx_bd_ring_consume(struct qede_rx_queue *rxq)
 762 {
 763         ecore_chain_consume(&rxq->rx_bd_ring);
 764         rxq->sw_rx_cons++;
 765 }
 766
 767 static inline void
 768 qede_reuse_page(struct qede_dev *qdev,
 769                 struct qede_rx_queue *rxq, struct qede_rx_entry *curr_cons)
 770 {
 771         struct eth_rx_bd *rx_bd_prod = ecore_chain_produce(&rxq->rx_bd_ring);
 772         uint16_t idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
 773         struct qede_rx_entry *curr_prod;
 774         dma_addr_t new_mapping;
 775
 776         curr_prod = &rxq->sw_rx_ring[idx];
 777         *curr_prod = *curr_cons;
 778
 779         new_mapping = rte_mbuf_data_dma_addr_default(curr_prod->mbuf) +
 780                       curr_prod->page_offset;
 781
 782         rx_bd_prod->addr.hi = rte_cpu_to_le_32(U64_HI(new_mapping));
 783         rx_bd_prod->addr.lo = rte_cpu_to_le_32(U64_LO(new_mapping));
 784
 785         rxq->sw_rx_prod++;
 786 }
 787
 788 static inline void
 789 qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq,
 790                         struct qede_dev *qdev, uint8_t count)
 791 {
 792         struct qede_rx_entry *curr_cons;
 793
 794         for (; count > 0; count--) {
 795                 curr_cons = &rxq->sw_rx_ring[rxq->sw_rx_cons & NUM_RX_BDS(rxq)];
 796                 qede_reuse_page(qdev, rxq, curr_cons);
 797                 qede_rx_bd_ring_consume(rxq);
 798         }
 799 }
 800
 801 static inline uint32_t qede_rx_cqe_to_pkt_type(uint16_t flags)
 802 {
 803         uint32_t p_type;
 804         /* TBD - L4 indications needed ? */
 805         uint16_t protocol = ((PARSING_AND_ERR_FLAGS_L3TYPE_MASK <<
 806                               PARSING_AND_ERR_FLAGS_L3TYPE_SHIFT) & flags);
 807
 808         /* protocol = 3 means LLC/SNAP over Ethernet */
 809         if (unlikely(protocol == 0 || protocol == 3))
 810                 p_type = RTE_PTYPE_UNKNOWN;
 811         else if (protocol == 1)
 812                 p_type = RTE_PTYPE_L3_IPV4;
 813         else if (protocol == 2)
 814                 p_type = RTE_PTYPE_L3_IPV6;
 815
 816         return RTE_PTYPE_L2_ETHER | p_type;
 817 }
 818
 819 uint16_t
 820 qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 821 {
 822         struct qede_rx_queue *rxq = p_rxq;
 823         struct qede_dev *qdev = rxq->qdev;
 824         struct ecore_dev *edev = &qdev->edev;
 825         struct qede_fastpath *fp = &qdev->fp_array[rxq->queue_id];
 826         uint16_t hw_comp_cons, sw_comp_cons, sw_rx_index;
 827         uint16_t rx_pkt = 0;
 828         union eth_rx_cqe *cqe;
 829         struct eth_fast_path_rx_reg_cqe *fp_cqe;
 830         register struct rte_mbuf *rx_mb = NULL;
 831         enum eth_rx_cqe_type cqe_type;
 832         uint16_t len, pad;
 833         uint16_t preload_idx;
 834         uint8_t csum_flag;
 835         uint16_t parse_flag;
 836         enum rss_hash_type htype;
 837
 838         hw_comp_cons = rte_le_to_cpu_16(*rxq->hw_cons_ptr);
 839         sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
 840
 841         rte_rmb();
 842
 843         if (hw_comp_cons == sw_comp_cons)
 844                 return 0;
 845
 846         while (sw_comp_cons != hw_comp_cons) {
 847                 /* Get the CQE from the completion ring */
 848                 cqe =
 849                     (union eth_rx_cqe *)ecore_chain_consume(&rxq->rx_comp_ring);
 850                 cqe_type = cqe->fast_path_regular.type;
 851
 852                 if (unlikely(cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH)) {
 853                         PMD_RX_LOG(DEBUG, rxq, "Got a slowath CQE\n");
 854
 855                         qdev->ops->eth_cqe_completion(edev, fp->id,
 856                                 (struct eth_slow_path_rx_cqe *)cqe);
 857                         goto next_cqe;
 858                 }
 859
 860                 /* Get the data from the SW ring */
 861                 sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
 862                 rx_mb = rxq->sw_rx_ring[sw_rx_index].mbuf;
 863                 assert(rx_mb != NULL);
 864
 865                 /* non GRO */
 866                 fp_cqe = &cqe->fast_path_regular;
 867
 868                 len = rte_le_to_cpu_16(fp_cqe->len_on_first_bd);
 869                 pad = fp_cqe->placement_offset;
 870                 assert((len + pad) <= rx_mb->buf_len);
 871
 872                 PMD_RX_LOG(DEBUG, rxq,
 873                            "CQE type = 0x%x, flags = 0x%x, vlan = 0x%x"
 874                            " len = %u, parsing_flags = %d\n",
 875                            cqe_type, fp_cqe->bitfields,
 876                            rte_le_to_cpu_16(fp_cqe->vlan_tag),
 877                            len, rte_le_to_cpu_16(fp_cqe->pars_flags.flags));
 878
 879                 /* If this is an error packet then drop it */
 880                 parse_flag =
 881                     rte_le_to_cpu_16(cqe->fast_path_regular.pars_flags.flags);
 882                 csum_flag = qede_check_csum(parse_flag);
 883                 if (unlikely(csum_flag == QEDE_CSUM_ERROR)) {
 884                         PMD_RX_LOG(ERR, rxq,
 885                                    "CQE in CONS = %u has error, flags = 0x%x "
 886                                    "dropping incoming packet\n",
 887                                    sw_comp_cons, parse_flag);
 888                         rxq->rx_hw_errors++;
 889                         qede_recycle_rx_bd_ring(rxq, qdev, fp_cqe->bd_num);
 890                         goto next_cqe;
 891                 }
 892
 893                 if (unlikely(qede_alloc_rx_buffer(rxq) != 0)) {
 894                         PMD_RX_LOG(ERR, rxq,
 895                                    "New buffer allocation failed,"
 896                                    "dropping incoming packet\n");
 897                         qede_recycle_rx_bd_ring(rxq, qdev, fp_cqe->bd_num);
 898                         rte_eth_devices[rxq->port_id].
 899                             data->rx_mbuf_alloc_failed++;
 900                         rxq->rx_alloc_errors++;
 901                         break;
 902                 }
 903
 904                 qede_rx_bd_ring_consume(rxq);
 905
 906                 /* Prefetch next mbuf while processing current one. */
 907                 preload_idx = rxq->sw_rx_cons & NUM_RX_BDS(rxq);
 908                 rte_prefetch0(rxq->sw_rx_ring[preload_idx].mbuf);
 909
 910                 if (fp_cqe->bd_num != 1)
 911                         PMD_RX_LOG(DEBUG, rxq,
 912                                    "Jumbo-over-BD packet not supported\n");
 913
 914                 /* Update MBUF fields */
 915                 rx_mb->ol_flags = 0;
 916                 rx_mb->data_off = pad + RTE_PKTMBUF_HEADROOM;
 917                 rx_mb->nb_segs = 1;
 918                 rx_mb->data_len = len;
 919                 rx_mb->pkt_len = len;
 920                 rx_mb->port = rxq->port_id;
 921                 rx_mb->packet_type = qede_rx_cqe_to_pkt_type(parse_flag);
 922
 923                 htype = (uint8_t)GET_FIELD(fp_cqe->bitfields,
 924                                 ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE);
 925                 if (qdev->rss_enabled && htype) {
 926                         rx_mb->ol_flags |= PKT_RX_RSS_HASH;
 927                         rx_mb->hash.rss = rte_le_to_cpu_32(fp_cqe->rss_hash);
 928                         PMD_RX_LOG(DEBUG, rxq, "Hash result 0x%x\n",
 929                                    rx_mb->hash.rss);
 930                 }
 931
 932                 rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
 933
 934                 if (CQE_HAS_VLAN(parse_flag)) {
 935                         rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag);
 936                         rx_mb->ol_flags |= PKT_RX_VLAN_PKT;
 937                 }
 938
 939                 if (CQE_HAS_OUTER_VLAN(parse_flag)) {
 940                         /* FW does not provide indication of Outer VLAN tag,
 941                          * which is always stripped, so vlan_tci_outer is set
 942                          * to 0. Here vlan_tag represents inner VLAN tag.
 943                          */
 944                         rx_mb->vlan_tci = rte_le_to_cpu_16(fp_cqe->vlan_tag);
 945                         rx_mb->ol_flags |= PKT_RX_QINQ_PKT;
 946                 }
 947
 948                 rx_pkts[rx_pkt] = rx_mb;
 949                 rx_pkt++;
 950 next_cqe:
 951                 ecore_chain_recycle_consumed(&rxq->rx_comp_ring);
 952                 sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring);
 953                 if (rx_pkt == nb_pkts) {
 954                         PMD_RX_LOG(DEBUG, rxq,
 955                                    "Budget reached nb_pkts=%u received=%u\n",
 956                                    rx_pkt, nb_pkts);
 957                         break;
 958                 }
 959         }
 960
 961         qede_update_rx_prod(qdev, rxq);
 962
 963         PMD_RX_LOG(DEBUG, rxq, "rx_pkts=%u core=%d\n", rx_pkt, rte_lcore_id());
 964
 965         return rx_pkt;
 966 }
 967
 968 static inline int
 969 qede_free_tx_pkt(struct ecore_dev *edev, struct qede_tx_queue *txq)
 970 {
 971         uint16_t idx = TX_CONS(txq);
 972         struct eth_tx_bd *tx_data_bd;
 973         struct rte_mbuf *mbuf = txq->sw_tx_ring[idx].mbuf;
 974
 975         if (unlikely(!mbuf)) {
 976                 PMD_TX_LOG(ERR, txq,
 977                            "null mbuf nb_tx_desc %u nb_tx_avail %u "
 978                            "sw_tx_cons %u sw_tx_prod %u\n",
 979                            txq->nb_tx_desc, txq->nb_tx_avail, idx,
 980                            TX_PROD(txq));
 981                 return -1;
 982         }
 983
 984         /* Free now */
 985         rte_pktmbuf_free_seg(mbuf);
 986         txq->sw_tx_ring[idx].mbuf = NULL;
 987         ecore_chain_consume(&txq->tx_pbl);
 988         txq->nb_tx_avail++;
 989
 990         return 0;
 991 }
 992
 993 static inline uint16_t
 994 qede_process_tx_compl(struct ecore_dev *edev, struct qede_tx_queue *txq)
 995 {
 996         uint16_t tx_compl = 0;
 997         uint16_t hw_bd_cons;
 998         int rc;
 999
1000         hw_bd_cons = rte_le_to_cpu_16(*txq->hw_cons_ptr);
1001         rte_compiler_barrier();
1002
1003         while (hw_bd_cons != ecore_chain_get_cons_idx(&txq->tx_pbl)) {
1004                 rc = qede_free_tx_pkt(edev, txq);
1005                 if (rc) {
1006                         DP_NOTICE(edev, false,
1007                                   "hw_bd_cons = %d, chain_cons=%d\n",
1008                                   hw_bd_cons,
1009                                   ecore_chain_get_cons_idx(&txq->tx_pbl));
1010                         break;
1011                 }
1012                 txq->sw_tx_cons++;      /* Making TXD available */
1013                 tx_compl++;
1014         }
1015
1016         PMD_TX_LOG(DEBUG, txq, "Tx compl %u sw_tx_cons %u avail %u\n",
1017                    tx_compl, txq->sw_tx_cons, txq->nb_tx_avail);
1018         return tx_compl;
1019 }
1020
1021 uint16_t
1022 qede_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1023 {
1024         struct qede_tx_queue *txq = p_txq;
1025         struct qede_dev *qdev = txq->qdev;
1026         struct ecore_dev *edev = &qdev->edev;
1027         struct qede_fastpath *fp;
1028         struct eth_tx_1st_bd *first_bd;
1029         uint16_t nb_tx_pkts;
1030         uint16_t nb_pkt_sent = 0;
1031         uint16_t bd_prod;
1032         uint16_t idx;
1033         uint16_t tx_count;
1034
1035         fp = &qdev->fp_array[QEDE_RSS_COUNT(qdev) + txq->queue_id];
1036
1037         if (unlikely(txq->nb_tx_avail < txq->tx_free_thresh)) {
1038                 PMD_TX_LOG(DEBUG, txq, "send=%u avail=%u free_thresh=%u\n",
1039                            nb_pkts, txq->nb_tx_avail, txq->tx_free_thresh);
1040                 (void)qede_process_tx_compl(edev, txq);
1041         }
1042
1043         nb_tx_pkts = RTE_MIN(nb_pkts, (txq->nb_tx_avail / MAX_NUM_TX_BDS));
1044         if (unlikely(nb_tx_pkts == 0)) {
1045                 PMD_TX_LOG(DEBUG, txq, "Out of BDs nb_pkts=%u avail=%u\n",
1046                            nb_pkts, txq->nb_tx_avail);
1047                 return 0;
1048         }
1049
1050         tx_count = nb_tx_pkts;
1051         while (nb_tx_pkts--) {
1052                 /* Fill the entry in the SW ring and the BDs in the FW ring */
1053                 idx = TX_PROD(txq);
1054                 struct rte_mbuf *mbuf = *tx_pkts++;
1055                 txq->sw_tx_ring[idx].mbuf = mbuf;
1056                 first_bd = (struct eth_tx_1st_bd *)
1057                     ecore_chain_produce(&txq->tx_pbl);
1058                 first_bd->data.bd_flags.bitfields =
1059                     1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT;
1060                 /* Map MBUF linear data for DMA and set in the first BD */
1061                 QEDE_BD_SET_ADDR_LEN(first_bd, rte_mbuf_data_dma_addr(mbuf),
1062                                      mbuf->data_len);
1063
1064                 /* Descriptor based VLAN insertion */
1065                 if (mbuf->ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) {
1066                         first_bd->data.vlan = rte_cpu_to_le_16(mbuf->vlan_tci);
1067                         first_bd->data.bd_flags.bitfields |=
1068                             1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT;
1069                 }
1070
1071                 /* Offload the IP checksum in the hardware */
1072                 if (mbuf->ol_flags & PKT_TX_IP_CKSUM) {
1073                         first_bd->data.bd_flags.bitfields |=
1074                             1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT;
1075                 }
1076
1077                 /* L4 checksum offload (tcp or udp) */
1078                 if (mbuf->ol_flags & (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
1079                         first_bd->data.bd_flags.bitfields |=
1080                             1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT;
1081                         /* IPv6 + extn. -> later */
1082                 }
1083                 first_bd->data.nbds = MAX_NUM_TX_BDS;
1084                 txq->sw_tx_prod++;
1085                 rte_prefetch0(txq->sw_tx_ring[TX_PROD(txq)].mbuf);
1086                 txq->nb_tx_avail--;
1087                 bd_prod =
1088                     rte_cpu_to_le_16(ecore_chain_get_prod_idx(&txq->tx_pbl));
1089                 nb_pkt_sent++;
1090         }
1091
1092         /* Write value of prod idx into bd_prod */
1093         txq->tx_db.data.bd_prod = bd_prod;
1094         rte_wmb();
1095         rte_compiler_barrier();
1096         DIRECT_REG_WR(edev, txq->doorbell_addr, txq->tx_db.raw);
1097         rte_wmb();
1098
1099         /* Check again for Tx completions */
1100         (void)qede_process_tx_compl(edev, txq);
1101
1102         PMD_TX_LOG(DEBUG, txq, "to_send=%u can_send=%u sent=%u core=%d\n",
1103                    nb_pkts, tx_count, nb_pkt_sent, rte_lcore_id());
1104
1105         return nb_pkt_sent;
1106 }
1107
1108 static void qede_init_fp_queue(struct rte_eth_dev *eth_dev)
1109 {
1110         struct qede_dev *qdev = eth_dev->data->dev_private;
1111         struct qede_fastpath *fp;
1112         uint8_t i, rss_id, txq_index, tc;
1113         int rxq = 0, txq = 0;
1114
1115         for_each_queue(i) {
1116                 fp = &qdev->fp_array[i];
1117                 if (fp->type & QEDE_FASTPATH_RX) {
1118                         fp->rxq = eth_dev->data->rx_queues[i];
1119                         fp->rxq->queue_id = rxq++;
1120                 }
1121
1122                 if (fp->type & QEDE_FASTPATH_TX) {
1123                         for (tc = 0; tc < qdev->num_tc; tc++) {
1124                                 txq_index = tc * QEDE_TSS_CNT(qdev) + txq;
1125                                 fp->txqs[tc] =
1126                                         eth_dev->data->tx_queues[txq_index];
1127                                 fp->txqs[tc]->queue_id = txq_index;
1128                         }
1129                         txq++;
1130                 }
1131         }
1132 }
1133
1134 int qede_dev_start(struct rte_eth_dev *eth_dev)
1135 {
1136         struct qede_dev *qdev = eth_dev->data->dev_private;
1137         struct ecore_dev *edev = &qdev->edev;
1138         struct qed_link_output link_output;
1139         struct qede_fastpath *fp;
1140         int rc, i;
1141
1142         DP_INFO(edev, "Device state is %d\n", qdev->state);
1143
1144         if (qdev->state == QEDE_DEV_START) {
1145                 DP_INFO(edev, "Port is already started\n");
1146                 return 0;
1147         }
1148
1149         if (qdev->state == QEDE_DEV_CONFIG)
1150                 qede_init_fp_queue(eth_dev);
1151
1152         rc = qede_start_queues(eth_dev, true);
1153         if (rc) {
1154                 DP_ERR(edev, "Failed to start queues\n");
1155                 /* TBD: free */
1156                 return rc;
1157         }
1158
1159         /* Bring-up the link */
1160         qede_dev_set_link_state(eth_dev, true);
1161
1162         /* Reset ring */
1163         if (qede_reset_fp_rings(qdev))
1164                 return -ENOMEM;
1165
1166         /* Start/resume traffic */
1167         qdev->ops->fastpath_start(edev);
1168
1169         qdev->state = QEDE_DEV_START;
1170
1171         DP_INFO(edev, "dev_state is QEDE_DEV_START\n");
1172
1173         return 0;
1174 }
1175
1176 static int qede_drain_txq(struct qede_dev *qdev,
1177                           struct qede_tx_queue *txq, bool allow_drain)
1178 {
1179         struct ecore_dev *edev = &qdev->edev;
1180         int rc, cnt = 1000;
1181
1182         while (txq->sw_tx_cons != txq->sw_tx_prod) {
1183                 qede_process_tx_compl(edev, txq);
1184                 if (!cnt) {
1185                         if (allow_drain) {
1186                                 DP_NOTICE(edev, false,
1187                                           "Tx queue[%u] is stuck,"
1188                                           "requesting MCP to drain\n",
1189                                           txq->queue_id);
1190                                 rc = qdev->ops->common->drain(edev);
1191                                 if (rc)
1192                                         return rc;
1193                                 return qede_drain_txq(qdev, txq, false);
1194                         }
1195
1196                         DP_NOTICE(edev, false,
1197                                   "Timeout waiting for tx queue[%d]:"
1198                                   "PROD=%d, CONS=%d\n",
1199                                   txq->queue_id, txq->sw_tx_prod,
1200                                   txq->sw_tx_cons);
1201                         return -ENODEV;
1202                 }
1203                 cnt--;
1204                 DELAY(1000);
1205                 rte_compiler_barrier();
1206         }
1207
1208         /* FW finished processing, wait for HW to transmit all tx packets */
1209         DELAY(2000);
1210
1211         return 0;
1212 }
1213
1214 static int qede_stop_queues(struct qede_dev *qdev)
1215 {
1216         struct qed_update_vport_params vport_update_params;
1217         struct ecore_dev *edev = &qdev->edev;
1218         int rc, tc, i;
1219
1220         /* Disable the vport */
1221         memset(&vport_update_params, 0, sizeof(vport_update_params));
1222         vport_update_params.vport_id = 0;
1223         vport_update_params.update_vport_active_flg = 1;
1224         vport_update_params.vport_active_flg = 0;
1225         vport_update_params.update_rss_flg = 0;
1226
1227         DP_INFO(edev, "Deactivate vport\n");
1228
1229         rc = qdev->ops->vport_update(edev, &vport_update_params);
1230         if (rc) {
1231                 DP_ERR(edev, "Failed to update vport\n");
1232                 return rc;
1233         }
1234
1235         DP_INFO(edev, "Flushing tx queues\n");
1236
1237         /* Flush Tx queues. If needed, request drain from MCP */
1238         for_each_queue(i) {
1239                 struct qede_fastpath *fp = &qdev->fp_array[i];
1240
1241                 if (fp->type & QEDE_FASTPATH_TX) {
1242                         for (tc = 0; tc < qdev->num_tc; tc++) {
1243                                 struct qede_tx_queue *txq = fp->txqs[tc];
1244
1245                                 rc = qede_drain_txq(qdev, txq, true);
1246                                 if (rc)
1247                                         return rc;
1248                         }
1249                 }
1250         }
1251
1252         /* Stop all Queues in reverse order */
1253         for (i = QEDE_QUEUE_CNT(qdev) - 1; i >= 0; i--) {
1254                 struct qed_stop_rxq_params rx_params;
1255
1256                 /* Stop the Tx Queue(s) */
1257                 if (qdev->fp_array[i].type & QEDE_FASTPATH_TX) {
1258                         for (tc = 0; tc < qdev->num_tc; tc++) {
1259                                 struct qed_stop_txq_params tx_params;
1260                                 u8 val;
1261
1262                                 tx_params.rss_id = i;
1263                                 val = qdev->fp_array[i].txqs[tc]->queue_id;
1264                                 tx_params.tx_queue_id = val;
1265
1266                                 DP_INFO(edev, "Stopping tx queues\n");
1267                                 rc = qdev->ops->q_tx_stop(edev, &tx_params);
1268                                 if (rc) {
1269                                         DP_ERR(edev, "Failed to stop TXQ #%d\n",
1270                                                tx_params.tx_queue_id);
1271                                         return rc;
1272                                 }
1273                         }
1274                 }
1275
1276                 /* Stop the Rx Queue */
1277                 if (qdev->fp_array[i].type & QEDE_FASTPATH_RX) {
1278                         memset(&rx_params, 0, sizeof(rx_params));
1279                         rx_params.rss_id = i;
1280                         rx_params.rx_queue_id = qdev->fp_array[i].rxq->queue_id;
1281                         rx_params.eq_completion_only = 1;
1282
1283                         DP_INFO(edev, "Stopping rx queues\n");
1284
1285                         rc = qdev->ops->q_rx_stop(edev, &rx_params);
1286                         if (rc) {
1287                                 DP_ERR(edev, "Failed to stop RXQ #%d\n", i);
1288                                 return rc;
1289                         }
1290                 }
1291         }
1292
1293         return 0;
1294 }
1295
1296 int qede_reset_fp_rings(struct qede_dev *qdev)
1297 {
1298         struct qede_fastpath *fp;
1299         struct qede_tx_queue *txq;
1300         uint8_t tc;
1301         uint16_t id, i;
1302
1303         for_each_queue(id) {
1304                 fp = &qdev->fp_array[id];
1305
1306                 if (fp->type & QEDE_FASTPATH_RX) {
1307                         DP_INFO(&qdev->edev,
1308                                 "Reset FP chain for RSS %u\n", id);
1309                         qede_rx_queue_release_mbufs(fp->rxq);
1310                         ecore_chain_reset(&fp->rxq->rx_bd_ring);
1311                         ecore_chain_reset(&fp->rxq->rx_comp_ring);
1312                         fp->rxq->sw_rx_prod = 0;
1313                         fp->rxq->sw_rx_cons = 0;
1314                         *fp->rxq->hw_cons_ptr = 0;
1315                         for (i = 0; i < fp->rxq->nb_rx_desc; i++) {
1316                                 if (qede_alloc_rx_buffer(fp->rxq)) {
1317                                         DP_ERR(&qdev->edev,
1318                                                "RX buffer allocation failed\n");
1319                                         return -ENOMEM;
1320                                 }
1321                         }
1322                 }
1323                 if (fp->type & QEDE_FASTPATH_TX) {
1324                         for (tc = 0; tc < qdev->num_tc; tc++) {
1325                                 txq = fp->txqs[tc];
1326                                 ecore_chain_reset(&txq->tx_pbl);
1327                                 txq->sw_tx_cons = 0;
1328                                 txq->sw_tx_prod = 0;
1329                                 *txq->hw_cons_ptr = 0;
1330                         }
1331                 }
1332         }
1333
1334         return 0;
1335 }
1336
1337 /* This function frees all memory of a single fp */
1338 static void qede_free_mem_fp(struct rte_eth_dev *eth_dev,
1339                              struct qede_fastpath *fp)
1340 {
1341         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
1342         uint8_t tc;
1343
1344         qede_rx_queue_release(fp->rxq);
1345         for (tc = 0; tc < qdev->num_tc; tc++) {
1346                 qede_tx_queue_release(fp->txqs[tc]);
1347                 eth_dev->data->tx_queues[tc] = NULL;
1348         }
1349 }
1350
1351 void qede_free_mem_load(struct rte_eth_dev *eth_dev)
1352 {
1353         struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev);
1354         struct qede_fastpath *fp;
1355         uint8_t rss_id;
1356
1357         for_each_queue(rss_id) {
1358                 fp = &qdev->fp_array[rss_id];
1359                 qede_free_mem_fp(eth_dev, fp);
1360                 eth_dev->data->rx_queues[rss_id] = NULL;
1361         }
1362 }
1363
1364 void qede_dev_stop(struct rte_eth_dev *eth_dev)
1365 {
1366         struct qede_dev *qdev = eth_dev->data->dev_private;
1367         struct ecore_dev *edev = &qdev->edev;
1368
1369         DP_INFO(edev, "port %u\n", eth_dev->data->port_id);
1370
1371         if (qdev->state != QEDE_DEV_START) {
1372                 DP_INFO(edev, "Device not yet started\n");
1373                 return;
1374         }
1375
1376         if (qede_stop_queues(qdev))
1377                 DP_ERR(edev, "Didn't succeed to close queues\n");
1378
1379         DP_INFO(edev, "Stopped queues\n");
1380
1381         qdev->ops->fastpath_stop(edev);
1382
1383         /* Bring the link down */
1384         qede_dev_set_link_state(eth_dev, false);
1385
1386         qdev->state = QEDE_DEV_STOP;
1387
1388         DP_INFO(edev, "dev_state is QEDE_DEV_STOP\n");
1389 }