drivers/net/cxgbe/cxgbe_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2014-2017 Chelsio Communications.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Chelsio Communications nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <string.h>
  39 #include <unistd.h>
  40 #include <stdarg.h>
  41 #include <inttypes.h>
  42 #include <netinet/in.h>
  43
  44 #include <rte_byteorder.h>
  45 #include <rte_common.h>
  46 #include <rte_cycles.h>
  47 #include <rte_interrupts.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_pci.h>
  51 #include <rte_bus_pci.h>
  52 #include <rte_atomic.h>
  53 #include <rte_branch_prediction.h>
  54 #include <rte_memory.h>
  55 #include <rte_memzone.h>
  56 #include <rte_tailq.h>
  57 #include <rte_eal.h>
  58 #include <rte_alarm.h>
  59 #include <rte_ether.h>
  60 #include <rte_ethdev.h>
  61 #include <rte_ethdev_pci.h>
  62 #include <rte_malloc.h>
  63 #include <rte_random.h>
  64 #include <rte_dev.h>
  65
  66 #include "cxgbe.h"
  67
  68 /*
  69  * Macros needed to support the PCI Device ID Table ...
  70  */
  71 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
  72         static const struct rte_pci_id cxgb4_pci_tbl[] = {
  73 #define CH_PCI_DEVICE_ID_FUNCTION 0x4
  74
  75 #define PCI_VENDOR_ID_CHELSIO 0x1425
  76
  77 #define CH_PCI_ID_TABLE_ENTRY(devid) \
  78                 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CHELSIO, (devid)) }
  79
  80 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
  81                 { .vendor_id = 0, } \
  82         }
  83
  84 /*
  85  *... and the PCI ID Table itself ...
  86  */
  87 #include "t4_pci_id_tbl.h"
  88
  89 static uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
  90                                 uint16_t nb_pkts)
  91 {
  92         struct sge_eth_txq *txq = (struct sge_eth_txq *)tx_queue;
  93         uint16_t pkts_sent, pkts_remain;
  94         uint16_t total_sent = 0;
  95         int ret = 0;
  96
  97         CXGBE_DEBUG_TX(adapter, "%s: txq = %p; tx_pkts = %p; nb_pkts = %d\n",
  98                        __func__, txq, tx_pkts, nb_pkts);
  99
 100         t4_os_lock(&txq->txq_lock);
 101         /* free up desc from already completed tx */
 102         reclaim_completed_tx(&txq->q);
 103         while (total_sent < nb_pkts) {
 104                 pkts_remain = nb_pkts - total_sent;
 105
 106                 for (pkts_sent = 0; pkts_sent < pkts_remain; pkts_sent++) {
 107                         ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent],
 108                                           nb_pkts);
 109                         if (ret < 0)
 110                                 break;
 111                 }
 112                 if (!pkts_sent)
 113                         break;
 114                 total_sent += pkts_sent;
 115                 /* reclaim as much as possible */
 116                 reclaim_completed_tx(&txq->q);
 117         }
 118
 119         t4_os_unlock(&txq->txq_lock);
 120         return total_sent;
 121 }
 122
 123 static uint16_t cxgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 124                                 uint16_t nb_pkts)
 125 {
 126         struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)rx_queue;
 127         unsigned int work_done;
 128
 129         CXGBE_DEBUG_RX(adapter, "%s: rxq->rspq.cntxt_id = %u; nb_pkts = %d\n",
 130                        __func__, rxq->rspq.cntxt_id, nb_pkts);
 131
 132         if (cxgbe_poll(&rxq->rspq, rx_pkts, (unsigned int)nb_pkts, &work_done))
 133                 dev_err(adapter, "error in cxgbe poll\n");
 134
 135         CXGBE_DEBUG_RX(adapter, "%s: work_done = %u\n", __func__, work_done);
 136         return work_done;
 137 }
 138
 139 static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev,
 140                                struct rte_eth_dev_info *device_info)
 141 {
 142         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 143         struct adapter *adapter = pi->adapter;
 144         int max_queues = adapter->sge.max_ethqsets / adapter->params.nports;
 145
 146         static const struct rte_eth_desc_lim cxgbe_desc_lim = {
 147                 .nb_max = CXGBE_MAX_RING_DESC_SIZE,
 148                 .nb_min = CXGBE_MIN_RING_DESC_SIZE,
 149                 .nb_align = 1,
 150         };
 151
 152         device_info->pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 153
 154         device_info->min_rx_bufsize = CXGBE_MIN_RX_BUFSIZE;
 155         device_info->max_rx_pktlen = CXGBE_MAX_RX_PKTLEN;
 156         device_info->max_rx_queues = max_queues;
 157         device_info->max_tx_queues = max_queues;
 158         device_info->max_mac_addrs = 1;
 159         /* XXX: For now we support one MAC/port */
 160         device_info->max_vfs = adapter->params.arch.vfcount;
 161         device_info->max_vmdq_pools = 0; /* XXX: For now no support for VMDQ */
 162
 163         device_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
 164                                        DEV_RX_OFFLOAD_IPV4_CKSUM |
 165                                        DEV_RX_OFFLOAD_UDP_CKSUM |
 166                                        DEV_RX_OFFLOAD_TCP_CKSUM;
 167
 168         device_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
 169                                        DEV_TX_OFFLOAD_IPV4_CKSUM |
 170                                        DEV_TX_OFFLOAD_UDP_CKSUM |
 171                                        DEV_TX_OFFLOAD_TCP_CKSUM |
 172                                        DEV_TX_OFFLOAD_TCP_TSO;
 173
 174         device_info->reta_size = pi->rss_size;
 175
 176         device_info->rx_desc_lim = cxgbe_desc_lim;
 177         device_info->tx_desc_lim = cxgbe_desc_lim;
 178         cxgbe_get_speed_caps(pi, &device_info->speed_capa);
 179 }
 180
 181 static void cxgbe_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 182 {
 183         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 184         struct adapter *adapter = pi->adapter;
 185
 186         t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
 187                       1, -1, 1, -1, false);
 188 }
 189
 190 static void cxgbe_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 191 {
 192         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 193         struct adapter *adapter = pi->adapter;
 194
 195         t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
 196                       0, -1, 1, -1, false);
 197 }
 198
 199 static void cxgbe_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 200 {
 201         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 202         struct adapter *adapter = pi->adapter;
 203
 204         /* TODO: address filters ?? */
 205
 206         t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
 207                       -1, 1, 1, -1, false);
 208 }
 209
 210 static void cxgbe_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 211 {
 212         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 213         struct adapter *adapter = pi->adapter;
 214
 215         /* TODO: address filters ?? */
 216
 217         t4_set_rxmode(adapter, adapter->mbox, pi->viid, -1,
 218                       -1, 0, 1, -1, false);
 219 }
 220
 221 static int cxgbe_dev_link_update(struct rte_eth_dev *eth_dev,
 222                                  __rte_unused int wait_to_complete)
 223 {
 224         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 225         struct adapter *adapter = pi->adapter;
 226         struct sge *s = &adapter->sge;
 227         struct rte_eth_link *old_link = &eth_dev->data->dev_link;
 228         unsigned int work_done, budget = 4;
 229
 230         cxgbe_poll(&s->fw_evtq, NULL, budget, &work_done);
 231         if (old_link->link_status == pi->link_cfg.link_ok)
 232                 return -1;  /* link not changed */
 233
 234         eth_dev->data->dev_link.link_status = pi->link_cfg.link_ok;
 235         eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
 236         eth_dev->data->dev_link.link_speed = pi->link_cfg.speed;
 237
 238         /* link has changed */
 239         return 0;
 240 }
 241
 242 static int cxgbe_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu)
 243 {
 244         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 245         struct adapter *adapter = pi->adapter;
 246         struct rte_eth_dev_info dev_info;
 247         int err;
 248         uint16_t new_mtu = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 249
 250         cxgbe_dev_info_get(eth_dev, &dev_info);
 251
 252         /* Must accommodate at least ETHER_MIN_MTU */
 253         if ((new_mtu < ETHER_MIN_MTU) || (new_mtu > dev_info.max_rx_pktlen))
 254                 return -EINVAL;
 255
 256         /* set to jumbo mode if needed */
 257         if (new_mtu > ETHER_MAX_LEN)
 258                 eth_dev->data->dev_conf.rxmode.jumbo_frame = 1;
 259         else
 260                 eth_dev->data->dev_conf.rxmode.jumbo_frame = 0;
 261
 262         err = t4_set_rxmode(adapter, adapter->mbox, pi->viid, new_mtu, -1, -1,
 263                             -1, -1, true);
 264         if (!err)
 265                 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = new_mtu;
 266
 267         return err;
 268 }
 269
 270 static int cxgbe_dev_tx_queue_start(struct rte_eth_dev *eth_dev,
 271                                     uint16_t tx_queue_id);
 272 static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
 273                                     uint16_t tx_queue_id);
 274 static void cxgbe_dev_tx_queue_release(void *q);
 275 static void cxgbe_dev_rx_queue_release(void *q);
 276
 277 /*
 278  * Stop device.
 279  */
 280 static void cxgbe_dev_close(struct rte_eth_dev *eth_dev)
 281 {
 282         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 283         struct adapter *adapter = pi->adapter;
 284         int i, dev_down = 0;
 285
 286         CXGBE_FUNC_TRACE();
 287
 288         if (!(adapter->flags & FULL_INIT_DONE))
 289                 return;
 290
 291         cxgbe_down(pi);
 292
 293         /*
 294          *  We clear queues only if both tx and rx path of the port
 295          *  have been disabled
 296          */
 297         t4_sge_eth_clear_queues(pi);
 298
 299         /*  See if all ports are down */
 300         for_each_port(adapter, i) {
 301                 pi = adap2pinfo(adapter, i);
 302                 /*
 303                  * Skip first port of the adapter since it will be closed
 304                  * by DPDK
 305                  */
 306                 if (i == 0)
 307                         continue;
 308                 dev_down += (pi->eth_dev->data->dev_started == 0) ? 1 : 0;
 309         }
 310
 311         /* If rest of the ports are stopped, then free up resources */
 312         if (dev_down == (adapter->params.nports - 1))
 313                 cxgbe_close(adapter);
 314 }
 315
 316 /* Start the device.
 317  * It returns 0 on success.
 318  */
 319 static int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
 320 {
 321         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 322         struct adapter *adapter = pi->adapter;
 323         int err = 0, i;
 324
 325         CXGBE_FUNC_TRACE();
 326
 327         /*
 328          * If we don't have a connection to the firmware there's nothing we
 329          * can do.
 330          */
 331         if (!(adapter->flags & FW_OK)) {
 332                 err = -ENXIO;
 333                 goto out;
 334         }
 335
 336         if (!(adapter->flags & FULL_INIT_DONE)) {
 337                 err = cxgbe_up(adapter);
 338                 if (err < 0)
 339                         goto out;
 340         }
 341
 342         cxgbe_enable_rx_queues(pi);
 343
 344         err = setup_rss(pi);
 345         if (err)
 346                 goto out;
 347
 348         for (i = 0; i < pi->n_tx_qsets; i++) {
 349                 err = cxgbe_dev_tx_queue_start(eth_dev, i);
 350                 if (err)
 351                         goto out;
 352         }
 353
 354         for (i = 0; i < pi->n_rx_qsets; i++) {
 355                 err = cxgbe_dev_rx_queue_start(eth_dev, i);
 356                 if (err)
 357                         goto out;
 358         }
 359
 360         err = link_start(pi);
 361         if (err)
 362                 goto out;
 363
 364 out:
 365         return err;
 366 }
 367
 368 /*
 369  * Stop device: disable rx and tx functions to allow for reconfiguring.
 370  */
 371 static void cxgbe_dev_stop(struct rte_eth_dev *eth_dev)
 372 {
 373         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 374         struct adapter *adapter = pi->adapter;
 375
 376         CXGBE_FUNC_TRACE();
 377
 378         if (!(adapter->flags & FULL_INIT_DONE))
 379                 return;
 380
 381         cxgbe_down(pi);
 382
 383         /*
 384          *  We clear queues only if both tx and rx path of the port
 385          *  have been disabled
 386          */
 387         t4_sge_eth_clear_queues(pi);
 388 }
 389
 390 static int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
 391 {
 392         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 393         struct adapter *adapter = pi->adapter;
 394         int err;
 395
 396         CXGBE_FUNC_TRACE();
 397
 398         if (!(adapter->flags & FW_QUEUE_BOUND)) {
 399                 err = setup_sge_fwevtq(adapter);
 400                 if (err)
 401                         return err;
 402                 adapter->flags |= FW_QUEUE_BOUND;
 403         }
 404
 405         err = cfg_queue_count(eth_dev);
 406         if (err)
 407                 return err;
 408
 409         return 0;
 410 }
 411
 412 static int cxgbe_dev_tx_queue_start(struct rte_eth_dev *eth_dev,
 413                                     uint16_t tx_queue_id)
 414 {
 415         int ret;
 416         struct sge_eth_txq *txq = (struct sge_eth_txq *)
 417                                   (eth_dev->data->tx_queues[tx_queue_id]);
 418
 419         dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id);
 420
 421         ret = t4_sge_eth_txq_start(txq);
 422         if (ret == 0)
 423                 eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 424
 425         return ret;
 426 }
 427
 428 static int cxgbe_dev_tx_queue_stop(struct rte_eth_dev *eth_dev,
 429                                    uint16_t tx_queue_id)
 430 {
 431         int ret;
 432         struct sge_eth_txq *txq = (struct sge_eth_txq *)
 433                                   (eth_dev->data->tx_queues[tx_queue_id]);
 434
 435         dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id);
 436
 437         ret = t4_sge_eth_txq_stop(txq);
 438         if (ret == 0)
 439                 eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 440
 441         return ret;
 442 }
 443
 444 static int cxgbe_dev_tx_queue_setup(struct rte_eth_dev *eth_dev,
 445                                     uint16_t queue_idx, uint16_t nb_desc,
 446                                     unsigned int socket_id,
 447                                     const struct rte_eth_txconf *tx_conf)
 448 {
 449         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 450         struct adapter *adapter = pi->adapter;
 451         struct sge *s = &adapter->sge;
 452         struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset + queue_idx];
 453         int err = 0;
 454         unsigned int temp_nb_desc;
 455
 456         RTE_SET_USED(tx_conf);
 457
 458         dev_debug(adapter, "%s: eth_dev->data->nb_tx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; pi->first_qset = %u\n",
 459                   __func__, eth_dev->data->nb_tx_queues, queue_idx, nb_desc,
 460                   socket_id, pi->first_qset);
 461
 462         /*  Free up the existing queue  */
 463         if (eth_dev->data->tx_queues[queue_idx]) {
 464                 cxgbe_dev_tx_queue_release(eth_dev->data->tx_queues[queue_idx]);
 465                 eth_dev->data->tx_queues[queue_idx] = NULL;
 466         }
 467
 468         eth_dev->data->tx_queues[queue_idx] = (void *)txq;
 469
 470         /* Sanity Checking
 471          *
 472          * nb_desc should be > 1023 and <= CXGBE_MAX_RING_DESC_SIZE
 473          */
 474         temp_nb_desc = nb_desc;
 475         if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) {
 476                 dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n",
 477                          __func__, CXGBE_MIN_RING_DESC_SIZE,
 478                          CXGBE_DEFAULT_TX_DESC_SIZE);
 479                 temp_nb_desc = CXGBE_DEFAULT_TX_DESC_SIZE;
 480         } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) {
 481                 dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n",
 482                         __func__, CXGBE_MIN_RING_DESC_SIZE,
 483                         CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_TX_DESC_SIZE);
 484                 return -(EINVAL);
 485         }
 486
 487         txq->q.size = temp_nb_desc;
 488
 489         err = t4_sge_alloc_eth_txq(adapter, txq, eth_dev, queue_idx,
 490                                    s->fw_evtq.cntxt_id, socket_id);
 491
 492         dev_debug(adapter, "%s: txq->q.cntxt_id= %d err = %d\n",
 493                   __func__, txq->q.cntxt_id, err);
 494
 495         return err;
 496 }
 497
 498 static void cxgbe_dev_tx_queue_release(void *q)
 499 {
 500         struct sge_eth_txq *txq = (struct sge_eth_txq *)q;
 501
 502         if (txq) {
 503                 struct port_info *pi = (struct port_info *)
 504                                        (txq->eth_dev->data->dev_private);
 505                 struct adapter *adap = pi->adapter;
 506
 507                 dev_debug(adapter, "%s: pi->port_id = %d; tx_queue_id = %d\n",
 508                           __func__, pi->port_id, txq->q.cntxt_id);
 509
 510                 t4_sge_eth_txq_release(adap, txq);
 511         }
 512 }
 513
 514 static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
 515                                     uint16_t rx_queue_id)
 516 {
 517         int ret;
 518         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 519         struct adapter *adap = pi->adapter;
 520         struct sge_rspq *q;
 521
 522         dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
 523                   __func__, pi->port_id, rx_queue_id);
 524
 525         q = eth_dev->data->rx_queues[rx_queue_id];
 526
 527         ret = t4_sge_eth_rxq_start(adap, q);
 528         if (ret == 0)
 529                 eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 530
 531         return ret;
 532 }
 533
 534 static int cxgbe_dev_rx_queue_stop(struct rte_eth_dev *eth_dev,
 535                                    uint16_t rx_queue_id)
 536 {
 537         int ret;
 538         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 539         struct adapter *adap = pi->adapter;
 540         struct sge_rspq *q;
 541
 542         dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
 543                   __func__, pi->port_id, rx_queue_id);
 544
 545         q = eth_dev->data->rx_queues[rx_queue_id];
 546         ret = t4_sge_eth_rxq_stop(adap, q);
 547         if (ret == 0)
 548                 eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 549
 550         return ret;
 551 }
 552
 553 static int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 554                                     uint16_t queue_idx, uint16_t nb_desc,
 555                                     unsigned int socket_id,
 556                                     const struct rte_eth_rxconf *rx_conf,
 557                                     struct rte_mempool *mp)
 558 {
 559         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 560         struct adapter *adapter = pi->adapter;
 561         struct sge *s = &adapter->sge;
 562         struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset + queue_idx];
 563         int err = 0;
 564         int msi_idx = 0;
 565         unsigned int temp_nb_desc;
 566         struct rte_eth_dev_info dev_info;
 567         unsigned int pkt_len = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 568
 569         RTE_SET_USED(rx_conf);
 570
 571         dev_debug(adapter, "%s: eth_dev->data->nb_rx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; mp = %p\n",
 572                   __func__, eth_dev->data->nb_rx_queues, queue_idx, nb_desc,
 573                   socket_id, mp);
 574
 575         cxgbe_dev_info_get(eth_dev, &dev_info);
 576
 577         /* Must accommodate at least ETHER_MIN_MTU */
 578         if ((pkt_len < dev_info.min_rx_bufsize) ||
 579             (pkt_len > dev_info.max_rx_pktlen)) {
 580                 dev_err(adap, "%s: max pkt len must be > %d and <= %d\n",
 581                         __func__, dev_info.min_rx_bufsize,
 582                         dev_info.max_rx_pktlen);
 583                 return -EINVAL;
 584         }
 585
 586         /*  Free up the existing queue  */
 587         if (eth_dev->data->rx_queues[queue_idx]) {
 588                 cxgbe_dev_rx_queue_release(eth_dev->data->rx_queues[queue_idx]);
 589                 eth_dev->data->rx_queues[queue_idx] = NULL;
 590         }
 591
 592         eth_dev->data->rx_queues[queue_idx] = (void *)rxq;
 593
 594         /* Sanity Checking
 595          *
 596          * nb_desc should be > 0 and <= CXGBE_MAX_RING_DESC_SIZE
 597          */
 598         temp_nb_desc = nb_desc;
 599         if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) {
 600                 dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n",
 601                          __func__, CXGBE_MIN_RING_DESC_SIZE,
 602                          CXGBE_DEFAULT_RX_DESC_SIZE);
 603                 temp_nb_desc = CXGBE_DEFAULT_RX_DESC_SIZE;
 604         } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) {
 605                 dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n",
 606                         __func__, CXGBE_MIN_RING_DESC_SIZE,
 607                         CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_RX_DESC_SIZE);
 608                 return -(EINVAL);
 609         }
 610
 611         rxq->rspq.size = temp_nb_desc;
 612         if ((&rxq->fl) != NULL)
 613                 rxq->fl.size = temp_nb_desc;
 614
 615         /* Set to jumbo mode if necessary */
 616         if (pkt_len > ETHER_MAX_LEN)
 617                 eth_dev->data->dev_conf.rxmode.jumbo_frame = 1;
 618         else
 619                 eth_dev->data->dev_conf.rxmode.jumbo_frame = 0;
 620
 621         err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx,
 622                                &rxq->fl, t4_ethrx_handler,
 623                                t4_get_tp_ch_map(adapter, pi->tx_chan), mp,
 624                                queue_idx, socket_id);
 625
 626         dev_debug(adapter, "%s: err = %d; port_id = %d; cntxt_id = %u\n",
 627                   __func__, err, pi->port_id, rxq->rspq.cntxt_id);
 628         return err;
 629 }
 630
 631 static void cxgbe_dev_rx_queue_release(void *q)
 632 {
 633         struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)q;
 634         struct sge_rspq *rq = &rxq->rspq;
 635
 636         if (rq) {
 637                 struct port_info *pi = (struct port_info *)
 638                                        (rq->eth_dev->data->dev_private);
 639                 struct adapter *adap = pi->adapter;
 640
 641                 dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
 642                           __func__, pi->port_id, rxq->rspq.cntxt_id);
 643
 644                 t4_sge_eth_rxq_release(adap, rxq);
 645         }
 646 }
 647
 648 /*
 649  * Get port statistics.
 650  */
 651 static int cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev,
 652                                 struct rte_eth_stats *eth_stats)
 653 {
 654         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 655         struct adapter *adapter = pi->adapter;
 656         struct sge *s = &adapter->sge;
 657         struct port_stats ps;
 658         unsigned int i;
 659
 660         cxgbe_stats_get(pi, &ps);
 661
 662         /* RX Stats */
 663         eth_stats->imissed  = ps.rx_ovflow0 + ps.rx_ovflow1 +
 664                               ps.rx_ovflow2 + ps.rx_ovflow3 +
 665                               ps.rx_trunc0 + ps.rx_trunc1 +
 666                               ps.rx_trunc2 + ps.rx_trunc3;
 667         eth_stats->ierrors  = ps.rx_symbol_err + ps.rx_fcs_err +
 668                               ps.rx_jabber + ps.rx_too_long + ps.rx_runt +
 669                               ps.rx_len_err;
 670
 671         /* TX Stats */
 672         eth_stats->opackets = ps.tx_frames;
 673         eth_stats->obytes   = ps.tx_octets;
 674         eth_stats->oerrors  = ps.tx_error_frames;
 675
 676         for (i = 0; i < pi->n_rx_qsets; i++) {
 677                 struct sge_eth_rxq *rxq =
 678                         &s->ethrxq[pi->first_qset + i];
 679
 680                 eth_stats->q_ipackets[i] = rxq->stats.pkts;
 681                 eth_stats->q_ibytes[i] = rxq->stats.rx_bytes;
 682                 eth_stats->ipackets += eth_stats->q_ipackets[i];
 683                 eth_stats->ibytes += eth_stats->q_ibytes[i];
 684         }
 685
 686         for (i = 0; i < pi->n_tx_qsets; i++) {
 687                 struct sge_eth_txq *txq =
 688                         &s->ethtxq[pi->first_qset + i];
 689
 690                 eth_stats->q_opackets[i] = txq->stats.pkts;
 691                 eth_stats->q_obytes[i] = txq->stats.tx_bytes;
 692                 eth_stats->q_errors[i] = txq->stats.mapping_err;
 693         }
 694         return 0;
 695 }
 696
 697 /*
 698  * Reset port statistics.
 699  */
 700 static void cxgbe_dev_stats_reset(struct rte_eth_dev *eth_dev)
 701 {
 702         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 703         struct adapter *adapter = pi->adapter;
 704         struct sge *s = &adapter->sge;
 705         unsigned int i;
 706
 707         cxgbe_stats_reset(pi);
 708         for (i = 0; i < pi->n_rx_qsets; i++) {
 709                 struct sge_eth_rxq *rxq =
 710                         &s->ethrxq[pi->first_qset + i];
 711
 712                 rxq->stats.pkts = 0;
 713                 rxq->stats.rx_bytes = 0;
 714         }
 715         for (i = 0; i < pi->n_tx_qsets; i++) {
 716                 struct sge_eth_txq *txq =
 717                         &s->ethtxq[pi->first_qset + i];
 718
 719                 txq->stats.pkts = 0;
 720                 txq->stats.tx_bytes = 0;
 721                 txq->stats.mapping_err = 0;
 722         }
 723 }
 724
 725 static int cxgbe_flow_ctrl_get(struct rte_eth_dev *eth_dev,
 726                                struct rte_eth_fc_conf *fc_conf)
 727 {
 728         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 729         struct link_config *lc = &pi->link_cfg;
 730         int rx_pause, tx_pause;
 731
 732         fc_conf->autoneg = lc->fc & PAUSE_AUTONEG;
 733         rx_pause = lc->fc & PAUSE_RX;
 734         tx_pause = lc->fc & PAUSE_TX;
 735
 736         if (rx_pause && tx_pause)
 737                 fc_conf->mode = RTE_FC_FULL;
 738         else if (rx_pause)
 739                 fc_conf->mode = RTE_FC_RX_PAUSE;
 740         else if (tx_pause)
 741                 fc_conf->mode = RTE_FC_TX_PAUSE;
 742         else
 743                 fc_conf->mode = RTE_FC_NONE;
 744         return 0;
 745 }
 746
 747 static int cxgbe_flow_ctrl_set(struct rte_eth_dev *eth_dev,
 748                                struct rte_eth_fc_conf *fc_conf)
 749 {
 750         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 751         struct adapter *adapter = pi->adapter;
 752         struct link_config *lc = &pi->link_cfg;
 753
 754         if (lc->supported & FW_PORT_CAP_ANEG) {
 755                 if (fc_conf->autoneg)
 756                         lc->requested_fc |= PAUSE_AUTONEG;
 757                 else
 758                         lc->requested_fc &= ~PAUSE_AUTONEG;
 759         }
 760
 761         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 762             (fc_conf->mode & RTE_FC_RX_PAUSE))
 763                 lc->requested_fc |= PAUSE_RX;
 764         else
 765                 lc->requested_fc &= ~PAUSE_RX;
 766
 767         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 768             (fc_conf->mode & RTE_FC_TX_PAUSE))
 769                 lc->requested_fc |= PAUSE_TX;
 770         else
 771                 lc->requested_fc &= ~PAUSE_TX;
 772
 773         return t4_link_l1cfg(adapter, adapter->mbox, pi->tx_chan,
 774                              &pi->link_cfg);
 775 }
 776
 777 static const uint32_t *
 778 cxgbe_dev_supported_ptypes_get(struct rte_eth_dev *eth_dev)
 779 {
 780         static const uint32_t ptypes[] = {
 781                 RTE_PTYPE_L3_IPV4,
 782                 RTE_PTYPE_L3_IPV6,
 783                 RTE_PTYPE_UNKNOWN
 784         };
 785
 786         if (eth_dev->rx_pkt_burst == cxgbe_recv_pkts)
 787                 return ptypes;
 788         return NULL;
 789 }
 790
 791 static int cxgbe_get_eeprom_length(struct rte_eth_dev *dev)
 792 {
 793         RTE_SET_USED(dev);
 794         return EEPROMSIZE;
 795 }
 796
 797 /**
 798  * eeprom_ptov - translate a physical EEPROM address to virtual
 799  * @phys_addr: the physical EEPROM address
 800  * @fn: the PCI function number
 801  * @sz: size of function-specific area
 802  *
 803  * Translate a physical EEPROM address to virtual.  The first 1K is
 804  * accessed through virtual addresses starting at 31K, the rest is
 805  * accessed through virtual addresses starting at 0.
 806  *
 807  * The mapping is as follows:
 808  * [0..1K) -> [31K..32K)
 809  * [1K..1K+A) -> [31K-A..31K)
 810  * [1K+A..ES) -> [0..ES-A-1K)
 811  *
 812  * where A = @fn * @sz, and ES = EEPROM size.
 813  */
 814 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
 815 {
 816         fn *= sz;
 817         if (phys_addr < 1024)
 818                 return phys_addr + (31 << 10);
 819         if (phys_addr < 1024 + fn)
 820                 return fn + phys_addr - 1024;
 821         if (phys_addr < EEPROMSIZE)
 822                 return phys_addr - 1024 - fn;
 823         if (phys_addr < EEPROMVSIZE)
 824                 return phys_addr - 1024;
 825         return -EINVAL;
 826 }
 827
 828 /* The next two routines implement eeprom read/write from physical addresses.
 829  */
 830 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
 831 {
 832         int vaddr = eeprom_ptov(phys_addr, adap->pf, EEPROMPFSIZE);
 833
 834         if (vaddr >= 0)
 835                 vaddr = t4_seeprom_read(adap, vaddr, v);
 836         return vaddr < 0 ? vaddr : 0;
 837 }
 838
 839 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
 840 {
 841         int vaddr = eeprom_ptov(phys_addr, adap->pf, EEPROMPFSIZE);
 842
 843         if (vaddr >= 0)
 844                 vaddr = t4_seeprom_write(adap, vaddr, v);
 845         return vaddr < 0 ? vaddr : 0;
 846 }
 847
 848 #define EEPROM_MAGIC 0x38E2F10C
 849
 850 static int cxgbe_get_eeprom(struct rte_eth_dev *dev,
 851                             struct rte_dev_eeprom_info *e)
 852 {
 853         struct port_info *pi = (struct port_info *)(dev->data->dev_private);
 854         struct adapter *adapter = pi->adapter;
 855         u32 i, err = 0;
 856         u8 *buf = rte_zmalloc(NULL, EEPROMSIZE, 0);
 857
 858         if (!buf)
 859                 return -ENOMEM;
 860
 861         e->magic = EEPROM_MAGIC;
 862         for (i = e->offset & ~3; !err && i < e->offset + e->length; i += 4)
 863                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
 864
 865         if (!err)
 866                 rte_memcpy(e->data, buf + e->offset, e->length);
 867         rte_free(buf);
 868         return err;
 869 }
 870
 871 static int cxgbe_set_eeprom(struct rte_eth_dev *dev,
 872                             struct rte_dev_eeprom_info *eeprom)
 873 {
 874         struct port_info *pi = (struct port_info *)(dev->data->dev_private);
 875         struct adapter *adapter = pi->adapter;
 876         u8 *buf;
 877         int err = 0;
 878         u32 aligned_offset, aligned_len, *p;
 879
 880         if (eeprom->magic != EEPROM_MAGIC)
 881                 return -EINVAL;
 882
 883         aligned_offset = eeprom->offset & ~3;
 884         aligned_len = (eeprom->length + (eeprom->offset & 3) + 3) & ~3;
 885
 886         if (adapter->pf > 0) {
 887                 u32 start = 1024 + adapter->pf * EEPROMPFSIZE;
 888
 889                 if (aligned_offset < start ||
 890                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
 891                         return -EPERM;
 892         }
 893
 894         if (aligned_offset != eeprom->offset || aligned_len != eeprom->length) {
 895                 /* RMW possibly needed for first or last words.
 896                  */
 897                 buf = rte_zmalloc(NULL, aligned_len, 0);
 898                 if (!buf)
 899                         return -ENOMEM;
 900                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
 901                 if (!err && aligned_len > 4)
 902                         err = eeprom_rd_phys(adapter,
 903                                              aligned_offset + aligned_len - 4,
 904                                              (u32 *)&buf[aligned_len - 4]);
 905                 if (err)
 906                         goto out;
 907                 rte_memcpy(buf + (eeprom->offset & 3), eeprom->data,
 908                            eeprom->length);
 909         } else {
 910                 buf = eeprom->data;
 911         }
 912
 913         err = t4_seeprom_wp(adapter, false);
 914         if (err)
 915                 goto out;
 916
 917         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
 918                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
 919                 aligned_offset += 4;
 920         }
 921
 922         if (!err)
 923                 err = t4_seeprom_wp(adapter, true);
 924 out:
 925         if (buf != eeprom->data)
 926                 rte_free(buf);
 927         return err;
 928 }
 929
 930 static int cxgbe_get_regs_len(struct rte_eth_dev *eth_dev)
 931 {
 932         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 933         struct adapter *adapter = pi->adapter;
 934
 935         return t4_get_regs_len(adapter) / sizeof(uint32_t);
 936 }
 937
 938 static int cxgbe_get_regs(struct rte_eth_dev *eth_dev,
 939                           struct rte_dev_reg_info *regs)
 940 {
 941         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 942         struct adapter *adapter = pi->adapter;
 943
 944         regs->version = CHELSIO_CHIP_VERSION(adapter->params.chip) |
 945                 (CHELSIO_CHIP_RELEASE(adapter->params.chip) << 10) |
 946                 (1 << 16);
 947
 948         if (regs->data == NULL) {
 949                 regs->length = cxgbe_get_regs_len(eth_dev);
 950                 regs->width = sizeof(uint32_t);
 951
 952                 return 0;
 953         }
 954
 955         t4_get_regs(adapter, regs->data, (regs->length * sizeof(uint32_t)));
 956
 957         return 0;
 958 }
 959
 960 static const struct eth_dev_ops cxgbe_eth_dev_ops = {
 961         .dev_start              = cxgbe_dev_start,
 962         .dev_stop               = cxgbe_dev_stop,
 963         .dev_close              = cxgbe_dev_close,
 964         .promiscuous_enable     = cxgbe_dev_promiscuous_enable,
 965         .promiscuous_disable    = cxgbe_dev_promiscuous_disable,
 966         .allmulticast_enable    = cxgbe_dev_allmulticast_enable,
 967         .allmulticast_disable   = cxgbe_dev_allmulticast_disable,
 968         .dev_configure          = cxgbe_dev_configure,
 969         .dev_infos_get          = cxgbe_dev_info_get,
 970         .dev_supported_ptypes_get = cxgbe_dev_supported_ptypes_get,
 971         .link_update            = cxgbe_dev_link_update,
 972         .mtu_set                = cxgbe_dev_mtu_set,
 973         .tx_queue_setup         = cxgbe_dev_tx_queue_setup,
 974         .tx_queue_start         = cxgbe_dev_tx_queue_start,
 975         .tx_queue_stop          = cxgbe_dev_tx_queue_stop,
 976         .tx_queue_release       = cxgbe_dev_tx_queue_release,
 977         .rx_queue_setup         = cxgbe_dev_rx_queue_setup,
 978         .rx_queue_start         = cxgbe_dev_rx_queue_start,
 979         .rx_queue_stop          = cxgbe_dev_rx_queue_stop,
 980         .rx_queue_release       = cxgbe_dev_rx_queue_release,
 981         .stats_get              = cxgbe_dev_stats_get,
 982         .stats_reset            = cxgbe_dev_stats_reset,
 983         .flow_ctrl_get          = cxgbe_flow_ctrl_get,
 984         .flow_ctrl_set          = cxgbe_flow_ctrl_set,
 985         .get_eeprom_length      = cxgbe_get_eeprom_length,
 986         .get_eeprom             = cxgbe_get_eeprom,
 987         .set_eeprom             = cxgbe_set_eeprom,
 988         .get_reg                = cxgbe_get_regs,
 989 };
 990
 991 /*
 992  * Initialize driver
 993  * It returns 0 on success.
 994  */
 995 static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
 996 {
 997         struct rte_pci_device *pci_dev;
 998         struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
 999         struct adapter *adapter = NULL;
1000         char name[RTE_ETH_NAME_MAX_LEN];
1001         int err = 0;
1002
1003         CXGBE_FUNC_TRACE();
1004
1005         eth_dev->dev_ops = &cxgbe_eth_dev_ops;
1006         eth_dev->rx_pkt_burst = &cxgbe_recv_pkts;
1007         eth_dev->tx_pkt_burst = &cxgbe_xmit_pkts;
1008
1009         /* for secondary processes, we don't initialise any further as primary
1010          * has already done this work.
1011          */
1012         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1013                 return 0;
1014
1015         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1016
1017         snprintf(name, sizeof(name), "cxgbeadapter%d", eth_dev->data->port_id);
1018         adapter = rte_zmalloc(name, sizeof(*adapter), 0);
1019         if (!adapter)
1020                 return -1;
1021
1022         adapter->use_unpacked_mode = 1;
1023         adapter->regs = (void *)pci_dev->mem_resource[0].addr;
1024         if (!adapter->regs) {
1025                 dev_err(adapter, "%s: cannot map device registers\n", __func__);
1026                 err = -ENOMEM;
1027                 goto out_free_adapter;
1028         }
1029         adapter->pdev = pci_dev;
1030         adapter->eth_dev = eth_dev;
1031         pi->adapter = adapter;
1032
1033         err = cxgbe_probe(adapter);
1034         if (err) {
1035                 dev_err(adapter, "%s: cxgbe probe failed with err %d\n",
1036                         __func__, err);
1037                 goto out_free_adapter;
1038         }
1039
1040         return 0;
1041
1042 out_free_adapter:
1043         rte_free(adapter);
1044         return err;
1045 }
1046
1047 static int eth_cxgbe_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1048         struct rte_pci_device *pci_dev)
1049 {
1050         return rte_eth_dev_pci_generic_probe(pci_dev,
1051                 sizeof(struct port_info), eth_cxgbe_dev_init);
1052 }
1053
1054 static int eth_cxgbe_pci_remove(struct rte_pci_device *pci_dev)
1055 {
1056         return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
1057 }
1058
1059 static struct rte_pci_driver rte_cxgbe_pmd = {
1060         .id_table = cxgb4_pci_tbl,
1061         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1062         .probe = eth_cxgbe_pci_probe,
1063         .remove = eth_cxgbe_pci_remove,
1064 };
1065
1066 RTE_PMD_REGISTER_PCI(net_cxgbe, rte_cxgbe_pmd);
1067 RTE_PMD_REGISTER_PCI_TABLE(net_cxgbe, cxgb4_pci_tbl);
1068 RTE_PMD_REGISTER_KMOD_DEP(net_cxgbe, "* igb_uio | uio_pci_generic | vfio-pci");