drivers/net/cxgbe/cxgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2014-2015 Chelsio Communications.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Chelsio Communications nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43
  44 #include <rte_byteorder.h>
  45 #include <rte_common.h>
  46 #include <rte_cycles.h>
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_atomic.h>
  52 #include <rte_branch_prediction.h>
  53 #include <rte_memory.h>
  54 #include <rte_memzone.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_alarm.h>
  58 #include <rte_ether.h>
  59 #include <rte_ethdev.h>
  60 #include <rte_atomic.h>
  61 #include <rte_malloc.h>
  62 #include <rte_random.h>
  63 #include <rte_dev.h>
  64
  65 #include "cxgbe.h"
  66
  67 /*
  68  * Macros needed to support the PCI Device ID Table ...
  69  */
  70 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
  71         static struct rte_pci_id cxgb4_pci_tbl[] = {
  72 #define CH_PCI_DEVICE_ID_FUNCTION 0x4
  73
  74 #define PCI_VENDOR_ID_CHELSIO 0x1425
  75
  76 #define CH_PCI_ID_TABLE_ENTRY(devid) \
  77                 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CHELSIO, (devid)) }
  78
  79 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
  80                 { .vendor_id = 0, } \
  81         }
  82
  83 /*
  84  *... and the PCI ID Table itself ...
  85  */
  86 #include "t4_pci_id_tbl.h"
  87
  88 static uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
  89                                 uint16_t nb_pkts)
  90 {
  91         struct sge_eth_txq *txq = (struct sge_eth_txq *)tx_queue;
  92         uint16_t pkts_sent, pkts_remain;
  93         uint16_t total_sent = 0;
  94         int ret = 0;
  95
  96         CXGBE_DEBUG_TX(adapter, "%s: txq = %p; tx_pkts = %p; nb_pkts = %d\n",
  97                        __func__, txq, tx_pkts, nb_pkts);
  98
  99         t4_os_lock(&txq->txq_lock);
 100         /* free up desc from already completed tx */
 101         reclaim_completed_tx(&txq->q);
 102         while (total_sent < nb_pkts) {
 103                 pkts_remain = nb_pkts - total_sent;
 104
 105                 for (pkts_sent = 0; pkts_sent < pkts_remain; pkts_sent++) {
 106                         ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent]);
 107                         if (ret < 0)
 108                                 break;
 109                 }
 110                 if (!pkts_sent)
 111                         break;
 112                 total_sent += pkts_sent;
 113                 /* reclaim as much as possible */
 114                 reclaim_completed_tx(&txq->q);
 115         }
 116
 117         t4_os_unlock(&txq->txq_lock);
 118         return total_sent;
 119 }
 120
 121 static uint16_t cxgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 122                                 uint16_t nb_pkts)
 123 {
 124         struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)rx_queue;
 125         unsigned int work_done;
 126
 127         CXGBE_DEBUG_RX(adapter, "%s: rxq->rspq.cntxt_id = %u; nb_pkts = %d\n",
 128                        __func__, rxq->rspq.cntxt_id, nb_pkts);
 129
 130         if (cxgbe_poll(&rxq->rspq, rx_pkts, (unsigned int)nb_pkts, &work_done))
 131                 dev_err(adapter, "error in cxgbe poll\n");
 132
 133         CXGBE_DEBUG_RX(adapter, "%s: work_done = %u\n", __func__, work_done);
 134         return work_done;
 135 }
 136
 137 static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev,
 138                                struct rte_eth_dev_info *device_info)
 139 {
 140         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 141         struct adapter *adapter = pi->adapter;
 142         int max_queues = adapter->sge.max_ethqsets / adapter->params.nports;
 143
 144         device_info->min_rx_bufsize = CXGBE_MIN_RX_BUFSIZE;
 145         device_info->max_rx_pktlen = CXGBE_MAX_RX_PKTLEN;
 146         device_info->max_rx_queues = max_queues;
 147         device_info->max_tx_queues = max_queues;
 148         device_info->max_mac_addrs = 1;
 149         /* XXX: For now we support one MAC/port */
 150         device_info->max_vfs = adapter->params.arch.vfcount;
 151         device_info->max_vmdq_pools = 0; /* XXX: For now no support for VMDQ */
 152
 153         device_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
 154                                        DEV_RX_OFFLOAD_IPV4_CKSUM |
 155                                        DEV_RX_OFFLOAD_UDP_CKSUM |
 156                                        DEV_RX_OFFLOAD_TCP_CKSUM;
 157
 158         device_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
 159                                        DEV_TX_OFFLOAD_IPV4_CKSUM |
 160                                        DEV_TX_OFFLOAD_UDP_CKSUM |
 161                                        DEV_TX_OFFLOAD_TCP_CKSUM |
 162                                        DEV_TX_OFFLOAD_TCP_TSO;
 163
 164         device_info->reta_size = pi->rss_size;
 165 }
 166
 167 static void cxgbe_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 168 {
 169         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 170         struct adapter *adapter = pi->adapter;
 171
 172         t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
 173                       1, -1, 1, -1, false);
 174 }
 175
 176 static void cxgbe_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 177 {
 178         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 179         struct adapter *adapter = pi->adapter;
 180
 181         t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
 182                       0, -1, 1, -1, false);
 183 }
 184
 185 static void cxgbe_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 186 {
 187         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 188         struct adapter *adapter = pi->adapter;
 189
 190         /* TODO: address filters ?? */
 191
 192         t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
 193                       -1, 1, 1, -1, false);
 194 }
 195
 196 static void cxgbe_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 197 {
 198         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 199         struct adapter *adapter = pi->adapter;
 200
 201         /* TODO: address filters ?? */
 202
 203         t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
 204                       -1, 0, 1, -1, false);
 205 }
 206
 207 static int cxgbe_dev_link_update(struct rte_eth_dev *eth_dev,
 208                                  __rte_unused int wait_to_complete)
 209 {
 210         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 211         struct adapter *adapter = pi->adapter;
 212         struct sge *s = &adapter->sge;
 213         struct rte_eth_link *old_link = &eth_dev->data->dev_link;
 214         unsigned int work_done, budget = 4;
 215
 216         cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done);
 217         if (old_link->link_status == pi->link_cfg.link_ok)
 218                 return -1;  /* link not changed */
 219
 220         eth_dev->data->dev_link.link_status = pi->link_cfg.link_ok;
 221         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 222         eth_dev->data->dev_link.link_speed = pi->link_cfg.speed;
 223
 224         /* link has changed */
 225         return 0;
 226 }
 227
 228 static int cxgbe_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu)
 229 {
 230         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 231         struct adapter *adapter = pi->adapter;
 232         struct rte_eth_dev_info dev_info;
 233         int err;
 234         uint16_t new_mtu = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 235
 236         cxgbe_dev_info_get(eth_dev, &dev_info);
 237
 238         /* Must accommodate at least ETHER_MIN_MTU */
 239         if ((new_mtu < ETHER_MIN_MTU) || (new_mtu > dev_info.max_rx_pktlen))
 240                 return -EINVAL;
 241
 242         /* set to jumbo mode if needed */
 243         if (new_mtu > ETHER_MAX_LEN)
 244                 eth_dev->data->dev_conf.rxmode.jumbo_frame = 1;
 245         else
 246                 eth_dev->data->dev_conf.rxmode.jumbo_frame = 0;
 247
 248         err = t4_set_rxmode(adapter, adapter->mbox, pi->viid, new_mtu, -1, -1,
 249                             -1, -1, true);
 250         if (!err)
 251                 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = new_mtu;
 252
 253         return err;
 254 }
 255
 256 static int cxgbe_dev_tx_queue_start(struct rte_eth_dev *eth_dev,
 257                                     uint16_t tx_queue_id);
 258 static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
 259                                     uint16_t tx_queue_id);
 260 static void cxgbe_dev_tx_queue_release(void *q);
 261 static void cxgbe_dev_rx_queue_release(void *q);
 262
 263 /*
 264  * Stop device.
 265  */
 266 static void cxgbe_dev_close(struct rte_eth_dev *eth_dev)
 267 {
 268         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 269         struct adapter *adapter = pi->adapter;
 270         int i, dev_down = 0;
 271
 272         CXGBE_FUNC_TRACE();
 273
 274         if (!(adapter->flags & FULL_INIT_DONE))
 275                 return;
 276
 277         cxgbe_down(pi);
 278
 279         /*
 280          *  We clear queues only if both tx and rx path of the port
 281          *  have been disabled
 282          */
 283         t4_sge_eth_clear_queues(pi);
 284
 285         /*  See if all ports are down */
 286         for_each_port(adapter, i) {
 287                 pi = adap2pinfo(adapter, i);
 288                 /*
 289                  * Skip first port of the adapter since it will be closed
 290                  * by DPDK
 291                  */
 292                 if (i == 0)
 293                         continue;
 294                 dev_down += (pi->eth_dev->data->dev_started == 0) ? 1 : 0;
 295         }
 296
 297         /* If rest of the ports are stopped, then free up resources */
 298         if (dev_down == (adapter->params.nports - 1))
 299                 cxgbe_close(adapter);
 300 }
 301
 302 /* Start the device.
 303  * It returns 0 on success.
 304  */
 305 static int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
 306 {
 307         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 308         struct adapter *adapter = pi->adapter;
 309         int err = 0, i;
 310
 311         CXGBE_FUNC_TRACE();
 312
 313         /*
 314          * If we don't have a connection to the firmware there's nothing we
 315          * can do.
 316          */
 317         if (!(adapter->flags & FW_OK)) {
 318                 err = -ENXIO;
 319                 goto out;
 320         }
 321
 322         if (!(adapter->flags & FULL_INIT_DONE)) {
 323                 err = cxgbe_up(adapter);
 324                 if (err < 0)
 325                         goto out;
 326         }
 327
 328         err = setup_rss(pi);
 329         if (err)
 330                 goto out;
 331
 332         for (i = 0; i < pi->n_tx_qsets; i++) {
 333                 err = cxgbe_dev_tx_queue_start(eth_dev, i);
 334                 if (err)
 335                         goto out;
 336         }
 337
 338         for (i = 0; i < pi->n_rx_qsets; i++) {
 339                 err = cxgbe_dev_rx_queue_start(eth_dev, i);
 340                 if (err)
 341                         goto out;
 342         }
 343
 344         err = link_start(pi);
 345         if (err)
 346                 goto out;
 347
 348 out:
 349         return err;
 350 }
 351
 352 /*
 353  * Stop device: disable rx and tx functions to allow for reconfiguring.
 354  */
 355 static void cxgbe_dev_stop(struct rte_eth_dev *eth_dev)
 356 {
 357         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 358         struct adapter *adapter = pi->adapter;
 359
 360         CXGBE_FUNC_TRACE();
 361
 362         if (!(adapter->flags & FULL_INIT_DONE))
 363                 return;
 364
 365         cxgbe_down(pi);
 366
 367         /*
 368          *  We clear queues only if both tx and rx path of the port
 369          *  have been disabled
 370          */
 371         t4_sge_eth_clear_queues(pi);
 372 }
 373
 374 static int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
 375 {
 376         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 377         struct adapter *adapter = pi->adapter;
 378         int err;
 379
 380         CXGBE_FUNC_TRACE();
 381
 382         if (!(adapter->flags & FW_QUEUE_BOUND)) {
 383                 err = setup_sge_fwevtq(adapter);
 384                 if (err)
 385                         return err;
 386                 adapter->flags |= FW_QUEUE_BOUND;
 387         }
 388
 389         err = cfg_queue_count(eth_dev);
 390         if (err)
 391                 return err;
 392
 393         return 0;
 394 }
 395
 396 static int cxgbe_dev_tx_queue_start(struct rte_eth_dev *eth_dev,
 397                                     uint16_t tx_queue_id)
 398 {
 399         struct sge_eth_txq *txq = (struct sge_eth_txq *)
 400                                   (eth_dev->data->tx_queues[tx_queue_id]);
 401
 402         dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id);
 403
 404         return t4_sge_eth_txq_start(txq);
 405 }
 406
 407 static int cxgbe_dev_tx_queue_stop(struct rte_eth_dev *eth_dev,
 408                                    uint16_t tx_queue_id)
 409 {
 410         struct sge_eth_txq *txq = (struct sge_eth_txq *)
 411                                   (eth_dev->data->tx_queues[tx_queue_id]);
 412
 413         dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id);
 414
 415         return t4_sge_eth_txq_stop(txq);
 416 }
 417
 418 static int cxgbe_dev_tx_queue_setup(struct rte_eth_dev *eth_dev,
 419                                     uint16_t queue_idx, uint16_t nb_desc,
 420                                     unsigned int socket_id,
 421                                     const struct rte_eth_txconf *tx_conf)
 422 {
 423         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 424         struct adapter *adapter = pi->adapter;
 425         struct sge *s = &adapter->sge;
 426         struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset + queue_idx];
 427         int err = 0;
 428         unsigned int temp_nb_desc;
 429
 430         RTE_SET_USED(tx_conf);
 431
 432         dev_debug(adapter, "%s: eth_dev->data->nb_tx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; pi->first_qset = %u\n",
 433                   __func__, eth_dev->data->nb_tx_queues, queue_idx, nb_desc,
 434                   socket_id, pi->first_qset);
 435
 436         /*  Free up the existing queue  */
 437         if (eth_dev->data->tx_queues[queue_idx]) {
 438                 cxgbe_dev_tx_queue_release(eth_dev->data->tx_queues[queue_idx]);
 439                 eth_dev->data->tx_queues[queue_idx] = NULL;
 440         }
 441
 442         eth_dev->data->tx_queues[queue_idx] = (void *)txq;
 443
 444         /* Sanity Checking
 445          *
 446          * nb_desc should be > 1023 and <= CXGBE_MAX_RING_DESC_SIZE
 447          */
 448         temp_nb_desc = nb_desc;
 449         if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) {
 450                 dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n",
 451                          __func__, CXGBE_MIN_RING_DESC_SIZE,
 452                          CXGBE_DEFAULT_TX_DESC_SIZE);
 453                 temp_nb_desc = CXGBE_DEFAULT_TX_DESC_SIZE;
 454         } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) {
 455                 dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n",
 456                         __func__, CXGBE_MIN_RING_DESC_SIZE,
 457                         CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_TX_DESC_SIZE);
 458                 return -(EINVAL);
 459         }
 460
 461         txq->q.size = temp_nb_desc;
 462
 463         err = t4_sge_alloc_eth_txq(adapter, txq, eth_dev, queue_idx,
 464                                    s->fw_evtq.cntxt_id, socket_id);
 465
 466         dev_debug(adapter, "%s: txq->q.cntxt_id= %d err = %d\n",
 467                   __func__, txq->q.cntxt_id, err);
 468
 469         return err;
 470 }
 471
 472 static void cxgbe_dev_tx_queue_release(void *q)
 473 {
 474         struct sge_eth_txq *txq = (struct sge_eth_txq *)q;
 475
 476         if (txq) {
 477                 struct port_info *pi = (struct port_info *)
 478                                        (txq->eth_dev->data->dev_private);
 479                 struct adapter *adap = pi->adapter;
 480
 481                 dev_debug(adapter, "%s: pi->port_id = %d; tx_queue_id = %d\n",
 482                           __func__, pi->port_id, txq->q.cntxt_id);
 483
 484                 t4_sge_eth_txq_release(adap, txq);
 485         }
 486 }
 487
 488 static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
 489                                     uint16_t rx_queue_id)
 490 {
 491         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 492         struct adapter *adap = pi->adapter;
 493         struct sge_rspq *q;
 494
 495         dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
 496                   __func__, pi->port_id, rx_queue_id);
 497
 498         q = eth_dev->data->rx_queues[rx_queue_id];
 499         return t4_sge_eth_rxq_start(adap, q);
 500 }
 501
 502 static int cxgbe_dev_rx_queue_stop(struct rte_eth_dev *eth_dev,
 503                                    uint16_t rx_queue_id)
 504 {
 505         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 506         struct adapter *adap = pi->adapter;
 507         struct sge_rspq *q;
 508
 509         dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
 510                   __func__, pi->port_id, rx_queue_id);
 511
 512         q = eth_dev->data->rx_queues[rx_queue_id];
 513         return t4_sge_eth_rxq_stop(adap, q);
 514 }
 515
 516 static int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 517                                     uint16_t queue_idx, uint16_t nb_desc,
 518                                     unsigned int socket_id,
 519                                     const struct rte_eth_rxconf *rx_conf,
 520                                     struct rte_mempool *mp)
 521 {
 522         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 523         struct adapter *adapter = pi->adapter;
 524         struct sge *s = &adapter->sge;
 525         struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset + queue_idx];
 526         int err = 0;
 527         int msi_idx = 0;
 528         unsigned int temp_nb_desc;
 529         struct rte_eth_dev_info dev_info;
 530         unsigned int pkt_len = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 531
 532         RTE_SET_USED(rx_conf);
 533
 534         dev_debug(adapter, "%s: eth_dev->data->nb_rx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; mp = %p\n",
 535                   __func__, eth_dev->data->nb_rx_queues, queue_idx, nb_desc,
 536                   socket_id, mp);
 537
 538         cxgbe_dev_info_get(eth_dev, &dev_info);
 539
 540         /* Must accommodate at least ETHER_MIN_MTU */
 541         if ((pkt_len < dev_info.min_rx_bufsize) ||
 542             (pkt_len > dev_info.max_rx_pktlen)) {
 543                 dev_err(adap, "%s: max pkt len must be > %d and <= %d\n",
 544                         __func__, dev_info.min_rx_bufsize,
 545                         dev_info.max_rx_pktlen);
 546                 return -EINVAL;
 547         }
 548
 549         /*  Free up the existing queue  */
 550         if (eth_dev->data->rx_queues[queue_idx]) {
 551                 cxgbe_dev_rx_queue_release(eth_dev->data->rx_queues[queue_idx]);
 552                 eth_dev->data->rx_queues[queue_idx] = NULL;
 553         }
 554
 555         eth_dev->data->rx_queues[queue_idx] = (void *)rxq;
 556
 557         /* Sanity Checking
 558          *
 559          * nb_desc should be > 0 and <= CXGBE_MAX_RING_DESC_SIZE
 560          */
 561         temp_nb_desc = nb_desc;
 562         if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) {
 563                 dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n",
 564                          __func__, CXGBE_MIN_RING_DESC_SIZE,
 565                          CXGBE_DEFAULT_RX_DESC_SIZE);
 566                 temp_nb_desc = CXGBE_DEFAULT_RX_DESC_SIZE;
 567         } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) {
 568                 dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n",
 569                         __func__, CXGBE_MIN_RING_DESC_SIZE,
 570                         CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_RX_DESC_SIZE);
 571                 return -(EINVAL);
 572         }
 573
 574         rxq->rspq.size = temp_nb_desc;
 575         if ((&rxq->fl) != NULL)
 576                 rxq->fl.size = temp_nb_desc;
 577
 578         /* Set to jumbo mode if necessary */
 579         if (pkt_len > ETHER_MAX_LEN)
 580                 eth_dev->data->dev_conf.rxmode.jumbo_frame = 1;
 581         else
 582                 eth_dev->data->dev_conf.rxmode.jumbo_frame = 0;
 583
 584         err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx,
 585                                &rxq->fl, t4_ethrx_handler,
 586                                t4_get_mps_bg_map(adapter, pi->tx_chan), mp,
 587                                queue_idx, socket_id);
 588
 589         dev_debug(adapter, "%s: err = %d; port_id = %d; cntxt_id = %u\n",
 590                   __func__, err, pi->port_id, rxq->rspq.cntxt_id);
 591         return err;
 592 }
 593
 594 static void cxgbe_dev_rx_queue_release(void *q)
 595 {
 596         struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)q;
 597         struct sge_rspq *rq = &rxq->rspq;
 598
 599         if (rq) {
 600                 struct port_info *pi = (struct port_info *)
 601                                        (rq->eth_dev->data->dev_private);
 602                 struct adapter *adap = pi->adapter;
 603
 604                 dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
 605                           __func__, pi->port_id, rxq->rspq.cntxt_id);
 606
 607                 t4_sge_eth_rxq_release(adap, rxq);
 608         }
 609 }
 610
 611 /*
 612  * Get port statistics.
 613  */
 614 static void cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev,
 615                                 struct rte_eth_stats *eth_stats)
 616 {
 617         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 618         struct adapter *adapter = pi->adapter;
 619         struct sge *s = &adapter->sge;
 620         struct port_stats ps;
 621         unsigned int i;
 622
 623         cxgbe_stats_get(pi, &ps);
 624
 625         /* RX Stats */
 626         eth_stats->ipackets = ps.rx_frames;
 627         eth_stats->ibytes   = ps.rx_octets;
 628         eth_stats->imcasts  = ps.rx_mcast_frames;
 629         eth_stats->imissed  = ps.rx_ovflow0 + ps.rx_ovflow1 +
 630                               ps.rx_ovflow2 + ps.rx_ovflow3 +
 631                               ps.rx_trunc0 + ps.rx_trunc1 +
 632                               ps.rx_trunc2 + ps.rx_trunc3;
 633         eth_stats->ibadcrc  = ps.rx_fcs_err;
 634         eth_stats->ibadlen  = ps.rx_jabber + ps.rx_too_long + ps.rx_runt;
 635         eth_stats->ierrors  = ps.rx_symbol_err + eth_stats->ibadcrc +
 636                               eth_stats->ibadlen + ps.rx_len_err +
 637                               eth_stats->imissed;
 638         eth_stats->rx_pause_xon  = ps.rx_pause;
 639
 640         /* TX Stats */
 641         eth_stats->opackets = ps.tx_frames;
 642         eth_stats->obytes   = ps.tx_octets;
 643         eth_stats->oerrors  = ps.tx_error_frames;
 644         eth_stats->tx_pause_xon  = ps.tx_pause;
 645
 646         for (i = 0; i < pi->n_rx_qsets; i++) {
 647                 struct sge_eth_rxq *rxq =
 648                         &s->ethrxq[pi->first_qset + i];
 649
 650                 eth_stats->q_ipackets[i] = rxq->stats.pkts;
 651                 eth_stats->q_ibytes[i] = rxq->stats.rx_bytes;
 652         }
 653
 654         for (i = 0; i < pi->n_tx_qsets; i++) {
 655                 struct sge_eth_txq *txq =
 656                         &s->ethtxq[pi->first_qset + i];
 657
 658                 eth_stats->q_opackets[i] = txq->stats.pkts;
 659                 eth_stats->q_obytes[i] = txq->stats.tx_bytes;
 660                 eth_stats->q_errors[i] = txq->stats.mapping_err;
 661         }
 662 }
 663
 664 /*
 665  * Reset port statistics.
 666  */
 667 static void cxgbe_dev_stats_reset(struct rte_eth_dev *eth_dev)
 668 {
 669         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 670         struct adapter *adapter = pi->adapter;
 671         struct sge *s = &adapter->sge;
 672         unsigned int i;
 673
 674         cxgbe_stats_reset(pi);
 675         for (i = 0; i < pi->n_rx_qsets; i++) {
 676                 struct sge_eth_rxq *rxq =
 677                         &s->ethrxq[pi->first_qset + i];
 678
 679                 rxq->stats.pkts = 0;
 680                 rxq->stats.rx_bytes = 0;
 681         }
 682         for (i = 0; i < pi->n_tx_qsets; i++) {
 683                 struct sge_eth_txq *txq =
 684                         &s->ethtxq[pi->first_qset + i];
 685
 686                 txq->stats.pkts = 0;
 687                 txq->stats.tx_bytes = 0;
 688                 txq->stats.mapping_err = 0;
 689         }
 690 }
 691
 692 static int cxgbe_flow_ctrl_get(struct rte_eth_dev *eth_dev,
 693                                struct rte_eth_fc_conf *fc_conf)
 694 {
 695         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 696         struct link_config *lc = &pi->link_cfg;
 697         int rx_pause, tx_pause;
 698
 699         fc_conf->autoneg = lc->fc & PAUSE_AUTONEG;
 700         rx_pause = lc->fc & PAUSE_RX;
 701         tx_pause = lc->fc & PAUSE_TX;
 702
 703         if (rx_pause && tx_pause)
 704                 fc_conf->mode = RTE_FC_FULL;
 705         else if (rx_pause)
 706                 fc_conf->mode = RTE_FC_RX_PAUSE;
 707         else if (tx_pause)
 708                 fc_conf->mode = RTE_FC_TX_PAUSE;
 709         else
 710                 fc_conf->mode = RTE_FC_NONE;
 711         return 0;
 712 }
 713
 714 static int cxgbe_flow_ctrl_set(struct rte_eth_dev *eth_dev,
 715                                struct rte_eth_fc_conf *fc_conf)
 716 {
 717         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 718         struct adapter *adapter = pi->adapter;
 719         struct link_config *lc = &pi->link_cfg;
 720
 721         if (lc->supported & FW_PORT_CAP_ANEG) {
 722                 if (fc_conf->autoneg)
 723                         lc->requested_fc |= PAUSE_AUTONEG;
 724                 else
 725                         lc->requested_fc &= ~PAUSE_AUTONEG;
 726         }
 727
 728         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 729             (fc_conf->mode & RTE_FC_RX_PAUSE))
 730                 lc->requested_fc |= PAUSE_RX;
 731         else
 732                 lc->requested_fc &= ~PAUSE_RX;
 733
 734         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 735             (fc_conf->mode & RTE_FC_TX_PAUSE))
 736                 lc->requested_fc |= PAUSE_TX;
 737         else
 738                 lc->requested_fc &= ~PAUSE_TX;
 739
 740         return t4_link_l1cfg(adapter, adapter->mbox, pi->tx_chan,
 741                              &pi->link_cfg);
 742 }
 743
 744 static struct eth_dev_ops cxgbe_eth_dev_ops = {
 745         .dev_start              = cxgbe_dev_start,
 746         .dev_stop               = cxgbe_dev_stop,
 747         .dev_close              = cxgbe_dev_close,
 748         .promiscuous_enable     = cxgbe_dev_promiscuous_enable,
 749         .promiscuous_disable    = cxgbe_dev_promiscuous_disable,
 750         .allmulticast_enable    = cxgbe_dev_allmulticast_enable,
 751         .allmulticast_disable   = cxgbe_dev_allmulticast_disable,
 752         .dev_configure          = cxgbe_dev_configure,
 753         .dev_infos_get          = cxgbe_dev_info_get,
 754         .link_update            = cxgbe_dev_link_update,
 755         .mtu_set                = cxgbe_dev_mtu_set,
 756         .tx_queue_setup         = cxgbe_dev_tx_queue_setup,
 757         .tx_queue_start         = cxgbe_dev_tx_queue_start,
 758         .tx_queue_stop          = cxgbe_dev_tx_queue_stop,
 759         .tx_queue_release       = cxgbe_dev_tx_queue_release,
 760         .rx_queue_setup         = cxgbe_dev_rx_queue_setup,
 761         .rx_queue_start         = cxgbe_dev_rx_queue_start,
 762         .rx_queue_stop          = cxgbe_dev_rx_queue_stop,
 763         .rx_queue_release       = cxgbe_dev_rx_queue_release,
 764         .stats_get              = cxgbe_dev_stats_get,
 765         .stats_reset            = cxgbe_dev_stats_reset,
 766         .flow_ctrl_get          = cxgbe_flow_ctrl_get,
 767         .flow_ctrl_set          = cxgbe_flow_ctrl_set,
 768 };
 769
 770 /*
 771  * Initialize driver
 772  * It returns 0 on success.
 773  */
 774 static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
 775 {
 776         struct rte_pci_device *pci_dev;
 777         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 778         struct adapter *adapter = NULL;
 779         char name[RTE_ETH_NAME_MAX_LEN];
 780         int err = 0;
 781
 782         CXGBE_FUNC_TRACE();
 783
 784         eth_dev->dev_ops = &cxgbe_eth_dev_ops;
 785         eth_dev->rx_pkt_burst = &cxgbe_recv_pkts;
 786         eth_dev->tx_pkt_burst = &cxgbe_xmit_pkts;
 787
 788         /* for secondary processes, we don't initialise any further as primary
 789          * has already done this work.
 790          */
 791         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 792                 return 0;
 793
 794         pci_dev = eth_dev->pci_dev;
 795         snprintf(name, sizeof(name), "cxgbeadapter%d", eth_dev->data->port_id);
 796         adapter = rte_zmalloc(name, sizeof(*adapter), 0);
 797         if (!adapter)
 798                 return -1;
 799
 800         adapter->use_unpacked_mode = 1;
 801         adapter->regs = (void *)pci_dev->mem_resource[0].addr;
 802         if (!adapter->regs) {
 803                 dev_err(adapter, "%s: cannot map device registers\n", __func__);
 804                 err = -ENOMEM;
 805                 goto out_free_adapter;
 806         }
 807         adapter->pdev = pci_dev;
 808         adapter->eth_dev = eth_dev;
 809         pi->adapter = adapter;
 810
 811         err = cxgbe_probe(adapter);
 812         if (err)
 813                 dev_err(adapter, "%s: cxgbe probe failed with err %d\n",
 814                         __func__, err);
 815
 816 out_free_adapter:
 817         return err;
 818 }
 819
 820 static struct eth_driver rte_cxgbe_pmd = {
 821         {
 822                 .name = "rte_cxgbe_pmd",
 823                 .id_table = cxgb4_pci_tbl,
 824                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 825         },
 826         .eth_dev_init = eth_cxgbe_dev_init,
 827         .dev_private_size = sizeof(struct port_info),
 828 };
 829
 830 /*
 831  * Driver initialization routine.
 832  * Invoked once at EAL init time.
 833  * Register itself as the [Poll Mode] Driver of PCI CXGBE devices.
 834  */
 835 static int rte_cxgbe_pmd_init(const char *name __rte_unused,
 836                               const char *params __rte_unused)
 837 {
 838         CXGBE_FUNC_TRACE();
 839
 840         rte_eth_driver_register(&rte_cxgbe_pmd);
 841         return 0;
 842 }
 843
 844 static struct rte_driver rte_cxgbe_driver = {
 845         .name = "cxgbe_driver",
 846         .type = PMD_PDEV,
 847         .init = rte_cxgbe_pmd_init,
 848 };
 849
 850 PMD_REGISTER_DRIVER(rte_cxgbe_driver);