4 * Copyright(c) 2014-2015 Chelsio Communications.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Chelsio Communications nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
42 #include <netinet/in.h>
44 #include <rte_byteorder.h>
45 #include <rte_common.h>
46 #include <rte_cycles.h>
47 #include <rte_interrupts.h>
49 #include <rte_debug.h>
51 #include <rte_atomic.h>
52 #include <rte_branch_prediction.h>
53 #include <rte_memory.h>
54 #include <rte_memzone.h>
55 #include <rte_tailq.h>
57 #include <rte_alarm.h>
58 #include <rte_ether.h>
59 #include <rte_ethdev.h>
60 #include <rte_atomic.h>
61 #include <rte_malloc.h>
62 #include <rte_random.h>
68 * Macros needed to support the PCI Device ID Table ...
70 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
71 static struct rte_pci_id cxgb4_pci_tbl[] = {
72 #define CH_PCI_DEVICE_ID_FUNCTION 0x4
74 #define PCI_VENDOR_ID_CHELSIO 0x1425
76 #define CH_PCI_ID_TABLE_ENTRY(devid) \
77 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CHELSIO, (devid)) }
79 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
84 *... and the PCI ID Table itself ...
86 #include "t4_pci_id_tbl.h"
88 static uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
91 struct sge_eth_txq *txq = (struct sge_eth_txq *)tx_queue;
92 uint16_t pkts_sent, pkts_remain;
93 uint16_t total_sent = 0;
96 CXGBE_DEBUG_TX(adapter, "%s: txq = %p; tx_pkts = %p; nb_pkts = %d\n",
97 __func__, txq, tx_pkts, nb_pkts);
99 t4_os_lock(&txq->txq_lock);
100 /* free up desc from already completed tx */
101 reclaim_completed_tx(&txq->q);
102 while (total_sent < nb_pkts) {
103 pkts_remain = nb_pkts - total_sent;
105 for (pkts_sent = 0; pkts_sent < pkts_remain; pkts_sent++) {
106 ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent]);
112 total_sent += pkts_sent;
113 /* reclaim as much as possible */
114 reclaim_completed_tx(&txq->q);
117 t4_os_unlock(&txq->txq_lock);
121 static uint16_t cxgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
124 struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)rx_queue;
125 unsigned int work_done;
127 CXGBE_DEBUG_RX(adapter, "%s: rxq->rspq.cntxt_id = %u; nb_pkts = %d\n",
128 __func__, rxq->rspq.cntxt_id, nb_pkts);
130 if (cxgbe_poll(&rxq->rspq, rx_pkts, (unsigned int)nb_pkts, &work_done))
131 dev_err(adapter, "error in cxgbe poll\n");
133 CXGBE_DEBUG_RX(adapter, "%s: work_done = %u\n", __func__, work_done);
137 static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev,
138 struct rte_eth_dev_info *device_info)
140 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
141 struct adapter *adapter = pi->adapter;
142 int max_queues = adapter->sge.max_ethqsets / adapter->params.nports;
144 device_info->min_rx_bufsize = 68; /* XXX: Smallest pkt size */
145 device_info->max_rx_pktlen = 1500; /* XXX: For now we support mtu */
146 device_info->max_rx_queues = max_queues;
147 device_info->max_tx_queues = max_queues;
148 device_info->max_mac_addrs = 1;
149 /* XXX: For now we support one MAC/port */
150 device_info->max_vfs = adapter->params.arch.vfcount;
151 device_info->max_vmdq_pools = 0; /* XXX: For now no support for VMDQ */
153 device_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
154 DEV_RX_OFFLOAD_IPV4_CKSUM |
155 DEV_RX_OFFLOAD_UDP_CKSUM |
156 DEV_RX_OFFLOAD_TCP_CKSUM;
158 device_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
159 DEV_TX_OFFLOAD_IPV4_CKSUM |
160 DEV_TX_OFFLOAD_UDP_CKSUM |
161 DEV_TX_OFFLOAD_TCP_CKSUM |
162 DEV_TX_OFFLOAD_TCP_TSO;
164 device_info->reta_size = pi->rss_size;
167 static int cxgbe_dev_tx_queue_start(struct rte_eth_dev *eth_dev,
168 uint16_t tx_queue_id);
169 static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
170 uint16_t tx_queue_id);
171 static void cxgbe_dev_tx_queue_release(void *q);
172 static void cxgbe_dev_rx_queue_release(void *q);
177 static void cxgbe_dev_close(struct rte_eth_dev *eth_dev)
179 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
180 struct adapter *adapter = pi->adapter;
185 if (!(adapter->flags & FULL_INIT_DONE))
191 * We clear queues only if both tx and rx path of the port
194 t4_sge_eth_clear_queues(pi);
196 /* See if all ports are down */
197 for_each_port(adapter, i) {
198 pi = adap2pinfo(adapter, i);
200 * Skip first port of the adapter since it will be closed
205 dev_down += (pi->eth_dev->data->dev_started == 0) ? 1 : 0;
208 /* If rest of the ports are stopped, then free up resources */
209 if (dev_down == (adapter->params.nports - 1))
210 cxgbe_close(adapter);
214 * It returns 0 on success.
216 static int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
218 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
219 struct adapter *adapter = pi->adapter;
225 * If we don't have a connection to the firmware there's nothing we
228 if (!(adapter->flags & FW_OK)) {
233 if (!(adapter->flags & FULL_INIT_DONE)) {
234 err = cxgbe_up(adapter);
243 for (i = 0; i < pi->n_tx_qsets; i++) {
244 err = cxgbe_dev_tx_queue_start(eth_dev, i);
249 for (i = 0; i < pi->n_rx_qsets; i++) {
250 err = cxgbe_dev_rx_queue_start(eth_dev, i);
255 err = link_start(pi);
264 * Stop device: disable rx and tx functions to allow for reconfiguring.
266 static void cxgbe_dev_stop(struct rte_eth_dev *eth_dev)
268 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
269 struct adapter *adapter = pi->adapter;
273 if (!(adapter->flags & FULL_INIT_DONE))
279 * We clear queues only if both tx and rx path of the port
282 t4_sge_eth_clear_queues(pi);
285 static int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
287 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
288 struct adapter *adapter = pi->adapter;
293 if (!(adapter->flags & FW_QUEUE_BOUND)) {
294 err = setup_sge_fwevtq(adapter);
297 adapter->flags |= FW_QUEUE_BOUND;
300 err = cfg_queue_count(eth_dev);
307 static int cxgbe_dev_tx_queue_start(struct rte_eth_dev *eth_dev,
308 uint16_t tx_queue_id)
310 struct sge_eth_txq *txq = (struct sge_eth_txq *)
311 (eth_dev->data->tx_queues[tx_queue_id]);
313 dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id);
315 return t4_sge_eth_txq_start(txq);
318 static int cxgbe_dev_tx_queue_stop(struct rte_eth_dev *eth_dev,
319 uint16_t tx_queue_id)
321 struct sge_eth_txq *txq = (struct sge_eth_txq *)
322 (eth_dev->data->tx_queues[tx_queue_id]);
324 dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id);
326 return t4_sge_eth_txq_stop(txq);
329 static int cxgbe_dev_tx_queue_setup(struct rte_eth_dev *eth_dev,
330 uint16_t queue_idx, uint16_t nb_desc,
331 unsigned int socket_id,
332 const struct rte_eth_txconf *tx_conf)
334 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
335 struct adapter *adapter = pi->adapter;
336 struct sge *s = &adapter->sge;
337 struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset + queue_idx];
339 unsigned int temp_nb_desc;
341 RTE_SET_USED(tx_conf);
343 dev_debug(adapter, "%s: eth_dev->data->nb_tx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; pi->first_qset = %u\n",
344 __func__, eth_dev->data->nb_tx_queues, queue_idx, nb_desc,
345 socket_id, pi->first_qset);
347 /* Free up the existing queue */
348 if (eth_dev->data->tx_queues[queue_idx]) {
349 cxgbe_dev_tx_queue_release(eth_dev->data->tx_queues[queue_idx]);
350 eth_dev->data->tx_queues[queue_idx] = NULL;
353 eth_dev->data->tx_queues[queue_idx] = (void *)txq;
357 * nb_desc should be > 1023 and <= CXGBE_MAX_RING_DESC_SIZE
359 temp_nb_desc = nb_desc;
360 if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) {
361 dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n",
362 __func__, CXGBE_MIN_RING_DESC_SIZE,
363 CXGBE_DEFAULT_TX_DESC_SIZE);
364 temp_nb_desc = CXGBE_DEFAULT_TX_DESC_SIZE;
365 } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) {
366 dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n",
367 __func__, CXGBE_MIN_RING_DESC_SIZE,
368 CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_TX_DESC_SIZE);
372 txq->q.size = temp_nb_desc;
374 err = t4_sge_alloc_eth_txq(adapter, txq, eth_dev, queue_idx,
375 s->fw_evtq.cntxt_id, socket_id);
377 dev_debug(adapter, "%s: txq->q.cntxt_id= %d err = %d\n",
378 __func__, txq->q.cntxt_id, err);
383 static void cxgbe_dev_tx_queue_release(void *q)
385 struct sge_eth_txq *txq = (struct sge_eth_txq *)q;
388 struct port_info *pi = (struct port_info *)
389 (txq->eth_dev->data->dev_private);
390 struct adapter *adap = pi->adapter;
392 dev_debug(adapter, "%s: pi->port_id = %d; tx_queue_id = %d\n",
393 __func__, pi->port_id, txq->q.cntxt_id);
395 t4_sge_eth_txq_release(adap, txq);
399 static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
400 uint16_t rx_queue_id)
402 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
403 struct adapter *adap = pi->adapter;
406 dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
407 __func__, pi->port_id, rx_queue_id);
409 q = eth_dev->data->rx_queues[rx_queue_id];
410 return t4_sge_eth_rxq_start(adap, q);
413 static int cxgbe_dev_rx_queue_stop(struct rte_eth_dev *eth_dev,
414 uint16_t rx_queue_id)
416 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
417 struct adapter *adap = pi->adapter;
420 dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
421 __func__, pi->port_id, rx_queue_id);
423 q = eth_dev->data->rx_queues[rx_queue_id];
424 return t4_sge_eth_rxq_stop(adap, q);
427 static int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
428 uint16_t queue_idx, uint16_t nb_desc,
429 unsigned int socket_id,
430 const struct rte_eth_rxconf *rx_conf,
431 struct rte_mempool *mp)
433 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
434 struct adapter *adapter = pi->adapter;
435 struct sge *s = &adapter->sge;
436 struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset + queue_idx];
439 unsigned int temp_nb_desc;
441 RTE_SET_USED(rx_conf);
443 dev_debug(adapter, "%s: eth_dev->data->nb_rx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; mp = %p\n",
444 __func__, eth_dev->data->nb_rx_queues, queue_idx, nb_desc,
447 /* Free up the existing queue */
448 if (eth_dev->data->rx_queues[queue_idx]) {
449 cxgbe_dev_rx_queue_release(eth_dev->data->rx_queues[queue_idx]);
450 eth_dev->data->rx_queues[queue_idx] = NULL;
453 eth_dev->data->rx_queues[queue_idx] = (void *)rxq;
457 * nb_desc should be > 0 and <= CXGBE_MAX_RING_DESC_SIZE
459 temp_nb_desc = nb_desc;
460 if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) {
461 dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n",
462 __func__, CXGBE_MIN_RING_DESC_SIZE,
463 CXGBE_DEFAULT_RX_DESC_SIZE);
464 temp_nb_desc = CXGBE_DEFAULT_RX_DESC_SIZE;
465 } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) {
466 dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n",
467 __func__, CXGBE_MIN_RING_DESC_SIZE,
468 CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_RX_DESC_SIZE);
472 rxq->rspq.size = temp_nb_desc;
473 if ((&rxq->fl) != NULL)
474 rxq->fl.size = temp_nb_desc;
476 err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx,
477 &rxq->fl, t4_ethrx_handler,
478 t4_get_mps_bg_map(adapter, pi->tx_chan), mp,
479 queue_idx, socket_id);
481 dev_debug(adapter, "%s: err = %d; port_id = %d; cntxt_id = %u\n",
482 __func__, err, pi->port_id, rxq->rspq.cntxt_id);
486 static void cxgbe_dev_rx_queue_release(void *q)
488 struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)q;
489 struct sge_rspq *rq = &rxq->rspq;
492 struct port_info *pi = (struct port_info *)
493 (rq->eth_dev->data->dev_private);
494 struct adapter *adap = pi->adapter;
496 dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
497 __func__, pi->port_id, rxq->rspq.cntxt_id);
499 t4_sge_eth_rxq_release(adap, rxq);
504 * Get port statistics.
506 static void cxgbe_dev_stats_get(struct rte_eth_dev *eth_dev,
507 struct rte_eth_stats *eth_stats)
509 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
510 struct adapter *adapter = pi->adapter;
511 struct sge *s = &adapter->sge;
512 struct port_stats ps;
515 cxgbe_stats_get(pi, &ps);
518 eth_stats->ipackets = ps.rx_frames;
519 eth_stats->ibytes = ps.rx_octets;
520 eth_stats->imcasts = ps.rx_mcast_frames;
521 eth_stats->imissed = ps.rx_ovflow0 + ps.rx_ovflow1 +
522 ps.rx_ovflow2 + ps.rx_ovflow3 +
523 ps.rx_trunc0 + ps.rx_trunc1 +
524 ps.rx_trunc2 + ps.rx_trunc3;
525 eth_stats->ibadcrc = ps.rx_fcs_err;
526 eth_stats->ibadlen = ps.rx_jabber + ps.rx_too_long + ps.rx_runt;
527 eth_stats->ierrors = ps.rx_symbol_err + eth_stats->ibadcrc +
528 eth_stats->ibadlen + ps.rx_len_err +
530 eth_stats->rx_pause_xon = ps.rx_pause;
533 eth_stats->opackets = ps.tx_frames;
534 eth_stats->obytes = ps.tx_octets;
535 eth_stats->oerrors = ps.tx_error_frames;
536 eth_stats->tx_pause_xon = ps.tx_pause;
538 for (i = 0; i < pi->n_rx_qsets; i++) {
539 struct sge_eth_rxq *rxq =
540 &s->ethrxq[pi->first_qset + i];
542 eth_stats->q_ipackets[i] = rxq->stats.pkts;
543 eth_stats->q_ibytes[i] = rxq->stats.rx_bytes;
546 for (i = 0; i < pi->n_tx_qsets; i++) {
547 struct sge_eth_txq *txq =
548 &s->ethtxq[pi->first_qset + i];
550 eth_stats->q_opackets[i] = txq->stats.pkts;
551 eth_stats->q_obytes[i] = txq->stats.tx_bytes;
552 eth_stats->q_errors[i] = txq->stats.mapping_err;
557 * Reset port statistics.
559 static void cxgbe_dev_stats_reset(struct rte_eth_dev *eth_dev)
561 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
562 struct adapter *adapter = pi->adapter;
563 struct sge *s = &adapter->sge;
566 cxgbe_stats_reset(pi);
567 for (i = 0; i < pi->n_rx_qsets; i++) {
568 struct sge_eth_rxq *rxq =
569 &s->ethrxq[pi->first_qset + i];
572 rxq->stats.rx_bytes = 0;
574 for (i = 0; i < pi->n_tx_qsets; i++) {
575 struct sge_eth_txq *txq =
576 &s->ethtxq[pi->first_qset + i];
579 txq->stats.tx_bytes = 0;
580 txq->stats.mapping_err = 0;
584 static struct eth_dev_ops cxgbe_eth_dev_ops = {
585 .dev_start = cxgbe_dev_start,
586 .dev_stop = cxgbe_dev_stop,
587 .dev_close = cxgbe_dev_close,
588 .dev_configure = cxgbe_dev_configure,
589 .dev_infos_get = cxgbe_dev_info_get,
590 .tx_queue_setup = cxgbe_dev_tx_queue_setup,
591 .tx_queue_start = cxgbe_dev_tx_queue_start,
592 .tx_queue_stop = cxgbe_dev_tx_queue_stop,
593 .tx_queue_release = cxgbe_dev_tx_queue_release,
594 .rx_queue_setup = cxgbe_dev_rx_queue_setup,
595 .rx_queue_start = cxgbe_dev_rx_queue_start,
596 .rx_queue_stop = cxgbe_dev_rx_queue_stop,
597 .rx_queue_release = cxgbe_dev_rx_queue_release,
598 .stats_get = cxgbe_dev_stats_get,
599 .stats_reset = cxgbe_dev_stats_reset,
604 * It returns 0 on success.
606 static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
608 struct rte_pci_device *pci_dev;
609 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
610 struct adapter *adapter = NULL;
611 char name[RTE_ETH_NAME_MAX_LEN];
616 eth_dev->dev_ops = &cxgbe_eth_dev_ops;
617 eth_dev->rx_pkt_burst = &cxgbe_recv_pkts;
618 eth_dev->tx_pkt_burst = &cxgbe_xmit_pkts;
620 /* for secondary processes, we don't initialise any further as primary
621 * has already done this work.
623 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
626 pci_dev = eth_dev->pci_dev;
627 snprintf(name, sizeof(name), "cxgbeadapter%d", eth_dev->data->port_id);
628 adapter = rte_zmalloc(name, sizeof(*adapter), 0);
632 adapter->use_unpacked_mode = 1;
633 adapter->regs = (void *)pci_dev->mem_resource[0].addr;
634 if (!adapter->regs) {
635 dev_err(adapter, "%s: cannot map device registers\n", __func__);
637 goto out_free_adapter;
639 adapter->pdev = pci_dev;
640 adapter->eth_dev = eth_dev;
641 pi->adapter = adapter;
643 err = cxgbe_probe(adapter);
645 dev_err(adapter, "%s: cxgbe probe failed with err %d\n",
652 static struct eth_driver rte_cxgbe_pmd = {
654 .name = "rte_cxgbe_pmd",
655 .id_table = cxgb4_pci_tbl,
656 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
658 .eth_dev_init = eth_cxgbe_dev_init,
659 .dev_private_size = sizeof(struct port_info),
663 * Driver initialization routine.
664 * Invoked once at EAL init time.
665 * Register itself as the [Poll Mode] Driver of PCI CXGBE devices.
667 static int rte_cxgbe_pmd_init(const char *name __rte_unused,
668 const char *params __rte_unused)
672 rte_eth_driver_register(&rte_cxgbe_pmd);
676 static struct rte_driver rte_cxgbe_driver = {
677 .name = "cxgbe_driver",
679 .init = rte_cxgbe_pmd_init,
682 PMD_REGISTER_DRIVER(rte_cxgbe_driver);