4 * Copyright(c) 2014-2015 Chelsio Communications.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Chelsio Communications nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
42 #include <netinet/in.h>
44 #include <rte_byteorder.h>
45 #include <rte_common.h>
46 #include <rte_cycles.h>
47 #include <rte_interrupts.h>
49 #include <rte_debug.h>
51 #include <rte_atomic.h>
52 #include <rte_branch_prediction.h>
53 #include <rte_memory.h>
54 #include <rte_memzone.h>
55 #include <rte_tailq.h>
57 #include <rte_alarm.h>
58 #include <rte_ether.h>
59 #include <rte_ethdev.h>
60 #include <rte_atomic.h>
61 #include <rte_malloc.h>
62 #include <rte_random.h>
68 * Macros needed to support the PCI Device ID Table ...
70 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
71 static struct rte_pci_id cxgb4_pci_tbl[] = {
72 #define CH_PCI_DEVICE_ID_FUNCTION 0x4
74 #define PCI_VENDOR_ID_CHELSIO 0x1425
76 #define CH_PCI_ID_TABLE_ENTRY(devid) \
77 { RTE_PCI_DEVICE(PCI_VENDOR_ID_CHELSIO, (devid)) }
79 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
84 *... and the PCI ID Table itself ...
86 #include "t4_pci_id_tbl.h"
88 static uint16_t cxgbe_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
91 struct sge_eth_txq *txq = (struct sge_eth_txq *)tx_queue;
92 uint16_t pkts_sent, pkts_remain;
93 uint16_t total_sent = 0;
96 CXGBE_DEBUG_TX(adapter, "%s: txq = %p; tx_pkts = %p; nb_pkts = %d\n",
97 __func__, txq, tx_pkts, nb_pkts);
99 t4_os_lock(&txq->txq_lock);
100 /* free up desc from already completed tx */
101 reclaim_completed_tx(&txq->q);
102 while (total_sent < nb_pkts) {
103 pkts_remain = nb_pkts - total_sent;
105 for (pkts_sent = 0; pkts_sent < pkts_remain; pkts_sent++) {
106 ret = t4_eth_xmit(txq, tx_pkts[total_sent + pkts_sent]);
112 total_sent += pkts_sent;
113 /* reclaim as much as possible */
114 reclaim_completed_tx(&txq->q);
117 t4_os_unlock(&txq->txq_lock);
121 static uint16_t cxgbe_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
124 struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)rx_queue;
125 unsigned int work_done;
127 CXGBE_DEBUG_RX(adapter, "%s: rxq->rspq.cntxt_id = %u; nb_pkts = %d\n",
128 __func__, rxq->rspq.cntxt_id, nb_pkts);
130 if (cxgbe_poll(&rxq->rspq, rx_pkts, (unsigned int)nb_pkts, &work_done))
131 dev_err(adapter, "error in cxgbe poll\n");
133 CXGBE_DEBUG_RX(adapter, "%s: work_done = %u\n", __func__, work_done);
137 static void cxgbe_dev_info_get(struct rte_eth_dev *eth_dev,
138 struct rte_eth_dev_info *device_info)
140 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
141 struct adapter *adapter = pi->adapter;
142 int max_queues = adapter->sge.max_ethqsets / adapter->params.nports;
144 device_info->min_rx_bufsize = 68; /* XXX: Smallest pkt size */
145 device_info->max_rx_pktlen = 1500; /* XXX: For now we support mtu */
146 device_info->max_rx_queues = max_queues;
147 device_info->max_tx_queues = max_queues;
148 device_info->max_mac_addrs = 1;
149 /* XXX: For now we support one MAC/port */
150 device_info->max_vfs = adapter->params.arch.vfcount;
151 device_info->max_vmdq_pools = 0; /* XXX: For now no support for VMDQ */
153 device_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
154 DEV_RX_OFFLOAD_IPV4_CKSUM |
155 DEV_RX_OFFLOAD_UDP_CKSUM |
156 DEV_RX_OFFLOAD_TCP_CKSUM;
158 device_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
159 DEV_TX_OFFLOAD_IPV4_CKSUM |
160 DEV_TX_OFFLOAD_UDP_CKSUM |
161 DEV_TX_OFFLOAD_TCP_CKSUM |
162 DEV_TX_OFFLOAD_TCP_TSO;
164 device_info->reta_size = pi->rss_size;
167 static int cxgbe_dev_tx_queue_start(struct rte_eth_dev *eth_dev,
168 uint16_t tx_queue_id);
169 static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
170 uint16_t tx_queue_id);
171 static void cxgbe_dev_tx_queue_release(void *q);
172 static void cxgbe_dev_rx_queue_release(void *q);
177 static void cxgbe_dev_close(struct rte_eth_dev *eth_dev)
179 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
180 struct adapter *adapter = pi->adapter;
185 if (!(adapter->flags & FULL_INIT_DONE))
191 * We clear queues only if both tx and rx path of the port
194 t4_sge_eth_clear_queues(pi);
196 /* See if all ports are down */
197 for_each_port(adapter, i) {
198 pi = adap2pinfo(adapter, i);
200 * Skip first port of the adapter since it will be closed
205 dev_down += (pi->eth_dev->data->dev_started == 0) ? 1 : 0;
208 /* If rest of the ports are stopped, then free up resources */
209 if (dev_down == (adapter->params.nports - 1))
210 cxgbe_close(adapter);
214 * It returns 0 on success.
216 static int cxgbe_dev_start(struct rte_eth_dev *eth_dev)
218 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
219 struct adapter *adapter = pi->adapter;
225 * If we don't have a connection to the firmware there's nothing we
228 if (!(adapter->flags & FW_OK)) {
233 if (!(adapter->flags & FULL_INIT_DONE)) {
234 err = cxgbe_up(adapter);
243 for (i = 0; i < pi->n_tx_qsets; i++) {
244 err = cxgbe_dev_tx_queue_start(eth_dev, i);
249 for (i = 0; i < pi->n_rx_qsets; i++) {
250 err = cxgbe_dev_rx_queue_start(eth_dev, i);
255 err = link_start(pi);
264 * Stop device: disable rx and tx functions to allow for reconfiguring.
266 static void cxgbe_dev_stop(struct rte_eth_dev *eth_dev)
268 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
269 struct adapter *adapter = pi->adapter;
273 if (!(adapter->flags & FULL_INIT_DONE))
279 * We clear queues only if both tx and rx path of the port
282 t4_sge_eth_clear_queues(pi);
285 static int cxgbe_dev_configure(struct rte_eth_dev *eth_dev)
287 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
288 struct adapter *adapter = pi->adapter;
293 if (!(adapter->flags & FW_QUEUE_BOUND)) {
294 err = setup_sge_fwevtq(adapter);
297 adapter->flags |= FW_QUEUE_BOUND;
300 err = cfg_queue_count(eth_dev);
307 static int cxgbe_dev_tx_queue_start(struct rte_eth_dev *eth_dev,
308 uint16_t tx_queue_id)
310 struct sge_eth_txq *txq = (struct sge_eth_txq *)
311 (eth_dev->data->tx_queues[tx_queue_id]);
313 dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id);
315 return t4_sge_eth_txq_start(txq);
318 static int cxgbe_dev_tx_queue_stop(struct rte_eth_dev *eth_dev,
319 uint16_t tx_queue_id)
321 struct sge_eth_txq *txq = (struct sge_eth_txq *)
322 (eth_dev->data->tx_queues[tx_queue_id]);
324 dev_debug(NULL, "%s: tx_queue_id = %d\n", __func__, tx_queue_id);
326 return t4_sge_eth_txq_stop(txq);
329 static int cxgbe_dev_tx_queue_setup(struct rte_eth_dev *eth_dev,
330 uint16_t queue_idx, uint16_t nb_desc,
331 unsigned int socket_id,
332 const struct rte_eth_txconf *tx_conf)
334 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
335 struct adapter *adapter = pi->adapter;
336 struct sge *s = &adapter->sge;
337 struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset + queue_idx];
339 unsigned int temp_nb_desc;
341 RTE_SET_USED(tx_conf);
343 dev_debug(adapter, "%s: eth_dev->data->nb_tx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; pi->first_qset = %u\n",
344 __func__, eth_dev->data->nb_tx_queues, queue_idx, nb_desc,
345 socket_id, pi->first_qset);
347 /* Free up the existing queue */
348 if (eth_dev->data->tx_queues[queue_idx]) {
349 cxgbe_dev_tx_queue_release(eth_dev->data->tx_queues[queue_idx]);
350 eth_dev->data->tx_queues[queue_idx] = NULL;
353 eth_dev->data->tx_queues[queue_idx] = (void *)txq;
357 * nb_desc should be > 1023 and <= CXGBE_MAX_RING_DESC_SIZE
359 temp_nb_desc = nb_desc;
360 if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) {
361 dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n",
362 __func__, CXGBE_MIN_RING_DESC_SIZE,
363 CXGBE_DEFAULT_TX_DESC_SIZE);
364 temp_nb_desc = CXGBE_DEFAULT_TX_DESC_SIZE;
365 } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) {
366 dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n",
367 __func__, CXGBE_MIN_RING_DESC_SIZE,
368 CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_TX_DESC_SIZE);
372 txq->q.size = temp_nb_desc;
374 err = t4_sge_alloc_eth_txq(adapter, txq, eth_dev, queue_idx,
375 s->fw_evtq.cntxt_id, socket_id);
377 dev_debug(adapter, "%s: txq->q.cntxt_id= %d err = %d\n",
378 __func__, txq->q.cntxt_id, err);
383 static void cxgbe_dev_tx_queue_release(void *q)
385 struct sge_eth_txq *txq = (struct sge_eth_txq *)q;
388 struct port_info *pi = (struct port_info *)
389 (txq->eth_dev->data->dev_private);
390 struct adapter *adap = pi->adapter;
392 dev_debug(adapter, "%s: pi->port_id = %d; tx_queue_id = %d\n",
393 __func__, pi->port_id, txq->q.cntxt_id);
395 t4_sge_eth_txq_release(adap, txq);
399 static int cxgbe_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
400 uint16_t rx_queue_id)
402 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
403 struct adapter *adap = pi->adapter;
406 dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
407 __func__, pi->port_id, rx_queue_id);
409 q = eth_dev->data->rx_queues[rx_queue_id];
410 return t4_sge_eth_rxq_start(adap, q);
413 static int cxgbe_dev_rx_queue_stop(struct rte_eth_dev *eth_dev,
414 uint16_t rx_queue_id)
416 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
417 struct adapter *adap = pi->adapter;
420 dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
421 __func__, pi->port_id, rx_queue_id);
423 q = eth_dev->data->rx_queues[rx_queue_id];
424 return t4_sge_eth_rxq_stop(adap, q);
427 static int cxgbe_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
428 uint16_t queue_idx, uint16_t nb_desc,
429 unsigned int socket_id,
430 const struct rte_eth_rxconf *rx_conf,
431 struct rte_mempool *mp)
433 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
434 struct adapter *adapter = pi->adapter;
435 struct sge *s = &adapter->sge;
436 struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset + queue_idx];
439 unsigned int temp_nb_desc;
441 RTE_SET_USED(rx_conf);
443 dev_debug(adapter, "%s: eth_dev->data->nb_rx_queues = %d; queue_idx = %d; nb_desc = %d; socket_id = %d; mp = %p\n",
444 __func__, eth_dev->data->nb_rx_queues, queue_idx, nb_desc,
447 /* Free up the existing queue */
448 if (eth_dev->data->rx_queues[queue_idx]) {
449 cxgbe_dev_rx_queue_release(eth_dev->data->rx_queues[queue_idx]);
450 eth_dev->data->rx_queues[queue_idx] = NULL;
453 eth_dev->data->rx_queues[queue_idx] = (void *)rxq;
457 * nb_desc should be > 0 and <= CXGBE_MAX_RING_DESC_SIZE
459 temp_nb_desc = nb_desc;
460 if (nb_desc < CXGBE_MIN_RING_DESC_SIZE) {
461 dev_warn(adapter, "%s: number of descriptors must be >= %d. Using default [%d]\n",
462 __func__, CXGBE_MIN_RING_DESC_SIZE,
463 CXGBE_DEFAULT_RX_DESC_SIZE);
464 temp_nb_desc = CXGBE_DEFAULT_RX_DESC_SIZE;
465 } else if (nb_desc > CXGBE_MAX_RING_DESC_SIZE) {
466 dev_err(adapter, "%s: number of descriptors must be between %d and %d inclusive. Default [%d]\n",
467 __func__, CXGBE_MIN_RING_DESC_SIZE,
468 CXGBE_MAX_RING_DESC_SIZE, CXGBE_DEFAULT_RX_DESC_SIZE);
472 rxq->rspq.size = temp_nb_desc;
473 if ((&rxq->fl) != NULL)
474 rxq->fl.size = temp_nb_desc;
476 err = t4_sge_alloc_rxq(adapter, &rxq->rspq, false, eth_dev, msi_idx,
477 &rxq->fl, t4_ethrx_handler,
478 t4_get_mps_bg_map(adapter, pi->tx_chan), mp,
479 queue_idx, socket_id);
481 dev_debug(adapter, "%s: err = %d; port_id = %d; cntxt_id = %u\n",
482 __func__, err, pi->port_id, rxq->rspq.cntxt_id);
486 static void cxgbe_dev_rx_queue_release(void *q)
488 struct sge_eth_rxq *rxq = (struct sge_eth_rxq *)q;
489 struct sge_rspq *rq = &rxq->rspq;
492 struct port_info *pi = (struct port_info *)
493 (rq->eth_dev->data->dev_private);
494 struct adapter *adap = pi->adapter;
496 dev_debug(adapter, "%s: pi->port_id = %d; rx_queue_id = %d\n",
497 __func__, pi->port_id, rxq->rspq.cntxt_id);
499 t4_sge_eth_rxq_release(adap, rxq);
503 static struct eth_dev_ops cxgbe_eth_dev_ops = {
504 .dev_start = cxgbe_dev_start,
505 .dev_stop = cxgbe_dev_stop,
506 .dev_close = cxgbe_dev_close,
507 .dev_configure = cxgbe_dev_configure,
508 .dev_infos_get = cxgbe_dev_info_get,
509 .tx_queue_setup = cxgbe_dev_tx_queue_setup,
510 .tx_queue_start = cxgbe_dev_tx_queue_start,
511 .tx_queue_stop = cxgbe_dev_tx_queue_stop,
512 .tx_queue_release = cxgbe_dev_tx_queue_release,
513 .rx_queue_setup = cxgbe_dev_rx_queue_setup,
514 .rx_queue_start = cxgbe_dev_rx_queue_start,
515 .rx_queue_stop = cxgbe_dev_rx_queue_stop,
516 .rx_queue_release = cxgbe_dev_rx_queue_release,
521 * It returns 0 on success.
523 static int eth_cxgbe_dev_init(struct rte_eth_dev *eth_dev)
525 struct rte_pci_device *pci_dev;
526 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
527 struct adapter *adapter = NULL;
528 char name[RTE_ETH_NAME_MAX_LEN];
533 eth_dev->dev_ops = &cxgbe_eth_dev_ops;
534 eth_dev->rx_pkt_burst = &cxgbe_recv_pkts;
535 eth_dev->tx_pkt_burst = &cxgbe_xmit_pkts;
537 /* for secondary processes, we don't initialise any further as primary
538 * has already done this work.
540 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
543 pci_dev = eth_dev->pci_dev;
544 snprintf(name, sizeof(name), "cxgbeadapter%d", eth_dev->data->port_id);
545 adapter = rte_zmalloc(name, sizeof(*adapter), 0);
549 adapter->use_unpacked_mode = 1;
550 adapter->regs = (void *)pci_dev->mem_resource[0].addr;
551 if (!adapter->regs) {
552 dev_err(adapter, "%s: cannot map device registers\n", __func__);
554 goto out_free_adapter;
556 adapter->pdev = pci_dev;
557 adapter->eth_dev = eth_dev;
558 pi->adapter = adapter;
560 err = cxgbe_probe(adapter);
562 dev_err(adapter, "%s: cxgbe probe failed with err %d\n",
569 static struct eth_driver rte_cxgbe_pmd = {
571 .name = "rte_cxgbe_pmd",
572 .id_table = cxgb4_pci_tbl,
573 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
575 .eth_dev_init = eth_cxgbe_dev_init,
576 .dev_private_size = sizeof(struct port_info),
580 * Driver initialization routine.
581 * Invoked once at EAL init time.
582 * Register itself as the [Poll Mode] Driver of PCI CXGBE devices.
584 static int rte_cxgbe_pmd_init(const char *name __rte_unused,
585 const char *params __rte_unused)
589 rte_eth_driver_register(&rte_cxgbe_pmd);
593 static struct rte_driver rte_cxgbe_driver = {
594 .name = "cxgbe_driver",
596 .init = rte_cxgbe_pmd_init,
599 PMD_REGISTER_DRIVER(rte_cxgbe_driver);