1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2014-2018 Chelsio Communications.
14 #include <netinet/in.h>
16 #include <rte_byteorder.h>
17 #include <rte_common.h>
18 #include <rte_cycles.h>
19 #include <rte_interrupts.h>
21 #include <rte_debug.h>
23 #include <rte_atomic.h>
24 #include <rte_branch_prediction.h>
25 #include <rte_memory.h>
26 #include <rte_tailq.h>
28 #include <rte_alarm.h>
29 #include <rte_ether.h>
30 #include <rte_ethdev_driver.h>
31 #include <rte_ethdev_pci.h>
32 #include <rte_random.h>
34 #include <rte_kvargs.h>
42 * Allocate a chunk of memory. The allocated memory is cleared.
44 void *t4_alloc_mem(size_t size)
46 return rte_zmalloc(NULL, size, 0);
50 * Free memory allocated through t4_alloc_mem().
52 void t4_free_mem(void *addr)
58 * Response queue handler for the FW event queue.
60 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
61 __rte_unused const struct pkt_gl *gl)
63 u8 opcode = ((const struct rss_header *)rsp)->opcode;
65 rsp++; /* skip RSS header */
68 * FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
70 if (unlikely(opcode == CPL_FW4_MSG &&
71 ((const struct cpl_fw4_msg *)rsp)->type ==
74 opcode = ((const struct rss_header *)rsp)->opcode;
76 if (opcode != CPL_SGE_EGR_UPDATE) {
77 dev_err(q->adapter, "unexpected FW4/CPL %#x on FW event queue\n",
83 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
85 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
86 const struct cpl_fw6_msg *msg = (const void *)rsp;
88 t4_handle_fw_rpl(q->adapter, msg->data);
89 } else if (opcode == CPL_SET_TCB_RPL) {
90 const struct cpl_set_tcb_rpl *p = (const void *)rsp;
92 filter_rpl(q->adapter, p);
94 dev_err(adapter, "unexpected CPL %#x on FW event queue\n",
102 * Setup sge control queues to pass control information.
104 int setup_sge_ctrl_txq(struct adapter *adapter)
106 struct sge *s = &adapter->sge;
109 for_each_port(adapter, i) {
110 char name[RTE_ETH_NAME_MAX_LEN];
111 struct sge_ctrl_txq *q = &s->ctrlq[i];
114 err = t4_sge_alloc_ctrl_txq(adapter, q,
119 dev_err(adapter, "Failed to alloc ctrl txq. Err: %d",
123 snprintf(name, sizeof(name), "cxgbe_ctrl_pool_%d", i);
124 q->mb_pool = rte_pktmbuf_pool_create(name, s->ctrlq[i].q.size,
127 RTE_MBUF_DEFAULT_BUF_SIZE,
130 dev_err(adapter, "Can't create ctrl pool for port: %d",
138 t4_free_sge_resources(adapter);
143 * cxgbe_poll_for_completion: Poll rxq for completion
145 * @us: microseconds to delay
146 * @cnt: number of times to poll
147 * @c: completion to check for 'done' status
149 * Polls the rxq for reples until completion is done or the count
152 int cxgbe_poll_for_completion(struct sge_rspq *q, unsigned int us,
153 unsigned int cnt, struct t4_completion *c)
156 unsigned int work_done, budget = 4;
161 for (i = 0; i < cnt; i++) {
162 cxgbe_poll(q, NULL, budget, &work_done);
163 t4_os_lock(&c->lock);
165 t4_os_unlock(&c->lock);
168 t4_os_unlock(&c->lock);
174 int setup_sge_fwevtq(struct adapter *adapter)
176 struct sge *s = &adapter->sge;
180 err = t4_sge_alloc_rxq(adapter, &s->fw_evtq, true, adapter->eth_dev,
181 msi_idx, NULL, fwevtq_handler, -1, NULL, 0,
186 static int closest_timer(const struct sge *s, int time)
188 unsigned int i, match = 0;
189 int delta, min_delta = INT_MAX;
191 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
192 delta = time - s->timer_val[i];
195 if (delta < min_delta) {
203 static int closest_thres(const struct sge *s, int thres)
205 unsigned int i, match = 0;
206 int delta, min_delta = INT_MAX;
208 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
209 delta = thres - s->counter_val[i];
212 if (delta < min_delta) {
221 * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters
223 * @us: the hold-off time in us, or 0 to disable timer
224 * @cnt: the hold-off packet count, or 0 to disable counter
226 * Sets an Rx queue's interrupt hold-off time and packet count. At least
227 * one of the two needs to be enabled for the queue to generate interrupts.
229 int cxgb4_set_rspq_intr_params(struct sge_rspq *q, unsigned int us,
232 struct adapter *adap = q->adapter;
233 unsigned int timer_val;
239 new_idx = closest_thres(&adap->sge, cnt);
240 if (q->desc && q->pktcnt_idx != new_idx) {
241 /* the queue has already been created, update it */
242 v = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
244 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
245 V_FW_PARAMS_PARAM_YZ(q->cntxt_id);
246 err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
251 q->pktcnt_idx = new_idx;
254 timer_val = (us == 0) ? X_TIMERREG_RESTART_COUNTER :
255 closest_timer(&adap->sge, us);
258 q->intr_params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX);
260 q->intr_params = V_QINTR_TIMER_IDX(timer_val) |
261 V_QINTR_CNT_EN(cnt > 0);
268 static void tid_free(struct tid_info *t)
272 rte_bitmap_free(t->ftid_bmap);
274 if (t->ftid_bmap_array)
275 t4_os_free(t->ftid_bmap_array);
277 t4_os_free(t->tid_tab);
280 memset(t, 0, sizeof(struct tid_info));
284 * Allocate and initialize the TID tables. Returns 0 on success.
286 static int tid_init(struct tid_info *t)
289 unsigned int ftid_bmap_size;
290 unsigned int natids = t->natids;
291 unsigned int max_ftids = t->nftids;
293 ftid_bmap_size = rte_bitmap_get_memory_footprint(t->nftids);
294 size = t->ntids * sizeof(*t->tid_tab) +
295 max_ftids * sizeof(*t->ftid_tab) +
296 natids * sizeof(*t->atid_tab);
298 t->tid_tab = t4_os_alloc(size);
302 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
303 t->ftid_tab = (struct filter_entry *)&t->tid_tab[t->natids];
304 t->ftid_bmap_array = t4_os_alloc(ftid_bmap_size);
305 if (!t->ftid_bmap_array) {
310 t4_os_lock_init(&t->atid_lock);
311 t4_os_lock_init(&t->ftid_lock);
315 rte_atomic32_init(&t->tids_in_use);
316 rte_atomic32_set(&t->tids_in_use, 0);
317 rte_atomic32_init(&t->conns_in_use);
318 rte_atomic32_set(&t->conns_in_use, 0);
320 /* Setup the free list for atid_tab and clear the stid bitmap. */
323 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
324 t->afree = t->atid_tab;
327 t->ftid_bmap = rte_bitmap_init(t->nftids, t->ftid_bmap_array,
337 static inline bool is_x_1g_port(const struct link_config *lc)
339 return (lc->pcaps & FW_PORT_CAP32_SPEED_1G) != 0;
342 static inline bool is_x_10g_port(const struct link_config *lc)
344 unsigned int speeds, high_speeds;
346 speeds = V_FW_PORT_CAP32_SPEED(G_FW_PORT_CAP32_SPEED(lc->pcaps));
347 high_speeds = speeds &
348 ~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G);
350 return high_speeds != 0;
353 inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
354 unsigned int us, unsigned int cnt,
355 unsigned int size, unsigned int iqe_size)
358 cxgb4_set_rspq_intr_params(q, us, cnt);
359 q->iqe_len = iqe_size;
363 int cfg_queue_count(struct rte_eth_dev *eth_dev)
365 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
366 struct adapter *adap = pi->adapter;
367 struct sge *s = &adap->sge;
368 unsigned int max_queues = s->max_ethqsets / adap->params.nports;
370 if ((eth_dev->data->nb_rx_queues < 1) ||
371 (eth_dev->data->nb_tx_queues < 1))
374 if ((eth_dev->data->nb_rx_queues > max_queues) ||
375 (eth_dev->data->nb_tx_queues > max_queues))
378 if (eth_dev->data->nb_rx_queues > pi->rss_size)
381 /* We must configure RSS, since config has changed*/
382 pi->flags &= ~PORT_RSS_DONE;
384 pi->n_rx_qsets = eth_dev->data->nb_rx_queues;
385 pi->n_tx_qsets = eth_dev->data->nb_tx_queues;
390 void cfg_queues(struct rte_eth_dev *eth_dev)
392 struct rte_config *config = rte_eal_get_configuration();
393 struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
394 struct adapter *adap = pi->adapter;
395 struct sge *s = &adap->sge;
396 unsigned int i, nb_ports = 0, qidx = 0;
397 unsigned int q_per_port = 0;
399 if (!(adap->flags & CFG_QUEUES)) {
400 for_each_port(adap, i) {
401 struct port_info *tpi = adap2pinfo(adap, i);
403 nb_ports += (is_x_10g_port(&tpi->link_cfg)) ||
404 is_x_1g_port(&tpi->link_cfg) ? 1 : 0;
408 * We default up to # of cores queues per 1G/10G port.
411 q_per_port = (MAX_ETH_QSETS -
412 (adap->params.nports - nb_ports)) /
415 if (q_per_port > config->lcore_count)
416 q_per_port = config->lcore_count;
418 for_each_port(adap, i) {
419 struct port_info *pi = adap2pinfo(adap, i);
421 pi->first_qset = qidx;
423 /* Initially n_rx_qsets == n_tx_qsets */
424 pi->n_rx_qsets = (is_x_10g_port(&pi->link_cfg) ||
425 is_x_1g_port(&pi->link_cfg)) ?
427 pi->n_tx_qsets = pi->n_rx_qsets;
429 if (pi->n_rx_qsets > pi->rss_size)
430 pi->n_rx_qsets = pi->rss_size;
432 qidx += pi->n_rx_qsets;
435 s->max_ethqsets = qidx;
437 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
438 struct sge_eth_rxq *r = &s->ethrxq[i];
440 init_rspq(adap, &r->rspq, 5, 32, 1024, 64);
442 r->fl.size = (r->usembufs ? 1024 : 72);
445 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
446 s->ethtxq[i].q.size = 1024;
448 init_rspq(adap, &adap->sge.fw_evtq, 0, 0, 1024, 64);
449 adap->flags |= CFG_QUEUES;
453 void cxgbe_stats_get(struct port_info *pi, struct port_stats *stats)
455 t4_get_port_stats_offset(pi->adapter, pi->tx_chan, stats,
459 void cxgbe_stats_reset(struct port_info *pi)
461 t4_clr_port_stats(pi->adapter, pi->tx_chan);
464 static void setup_memwin(struct adapter *adap)
468 /* For T5, only relative offset inside the PCIe BAR is passed */
469 mem_win0_base = MEMWIN0_BASE;
472 * Set up memory window for accessing adapter memory ranges. (Read
473 * back MA register to ensure that changes propagate before we attempt
474 * to use the new values.)
477 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN,
479 mem_win0_base | V_BIR(0) |
480 V_WINDOW(ilog2(MEMWIN0_APERTURE) - X_WINDOW_SHIFT));
482 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN,
486 int init_rss(struct adapter *adap)
493 err = t4_init_rss_mode(adap, adap->mbox);
498 for_each_port(adap, i) {
499 struct port_info *pi = adap2pinfo(adap, i);
501 pi->rss = rte_zmalloc(NULL, pi->rss_size * sizeof(u16), 0);
505 pi->rss_hf = CXGBE_RSS_HF_ALL;
511 * Dump basic information about the adapter.
513 void print_adapter_info(struct adapter *adap)
516 * Hardware/Firmware/etc. Version/Revision IDs.
518 t4_dump_version_info(adap);
521 void print_port_info(struct adapter *adap)
525 struct rte_pci_addr *loc = &adap->pdev->addr;
527 for_each_port(adap, i) {
528 const struct port_info *pi = adap2pinfo(adap, i);
531 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100M)
532 bufp += sprintf(bufp, "100M/");
533 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_1G)
534 bufp += sprintf(bufp, "1G/");
535 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G)
536 bufp += sprintf(bufp, "10G/");
537 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G)
538 bufp += sprintf(bufp, "25G/");
539 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G)
540 bufp += sprintf(bufp, "40G/");
541 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G)
542 bufp += sprintf(bufp, "50G/");
543 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G)
544 bufp += sprintf(bufp, "100G/");
547 sprintf(bufp, "BASE-%s",
548 t4_get_port_type_description(
549 (enum fw_port_type)pi->port_type));
552 " " PCI_PRI_FMT " Chelsio rev %d %s %s\n",
553 loc->domain, loc->bus, loc->devid, loc->function,
554 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
555 (adap->flags & USING_MSIX) ? " MSI-X" :
556 (adap->flags & USING_MSI) ? " MSI" : "");
561 check_devargs_handler(__rte_unused const char *key, const char *value,
562 __rte_unused void *opaque)
564 if (strcmp(value, "1"))
570 int cxgbe_get_devargs(struct rte_devargs *devargs, const char *key)
572 struct rte_kvargs *kvlist;
577 kvlist = rte_kvargs_parse(devargs->args, NULL);
581 if (!rte_kvargs_count(kvlist, key)) {
582 rte_kvargs_free(kvlist);
586 if (rte_kvargs_process(kvlist, key,
587 check_devargs_handler, NULL) < 0) {
588 rte_kvargs_free(kvlist);
591 rte_kvargs_free(kvlist);
596 static void configure_vlan_types(struct adapter *adapter)
598 struct rte_pci_device *pdev = adapter->pdev;
601 for_each_port(adapter, i) {
602 /* OVLAN Type 0x88a8 */
603 t4_set_reg_field(adapter, MPS_PORT_RX_OVLAN_REG(i, A_RX_OVLAN0),
604 V_OVLAN_MASK(M_OVLAN_MASK) |
605 V_OVLAN_ETYPE(M_OVLAN_ETYPE),
606 V_OVLAN_MASK(M_OVLAN_MASK) |
607 V_OVLAN_ETYPE(0x88a8));
608 /* OVLAN Type 0x9100 */
609 t4_set_reg_field(adapter, MPS_PORT_RX_OVLAN_REG(i, A_RX_OVLAN1),
610 V_OVLAN_MASK(M_OVLAN_MASK) |
611 V_OVLAN_ETYPE(M_OVLAN_ETYPE),
612 V_OVLAN_MASK(M_OVLAN_MASK) |
613 V_OVLAN_ETYPE(0x9100));
614 /* OVLAN Type 0x8100 */
615 t4_set_reg_field(adapter, MPS_PORT_RX_OVLAN_REG(i, A_RX_OVLAN2),
616 V_OVLAN_MASK(M_OVLAN_MASK) |
617 V_OVLAN_ETYPE(M_OVLAN_ETYPE),
618 V_OVLAN_MASK(M_OVLAN_MASK) |
619 V_OVLAN_ETYPE(0x8100));
622 t4_set_reg_field(adapter, MPS_PORT_RX_IVLAN(i),
623 V_IVLAN_ETYPE(M_IVLAN_ETYPE),
624 V_IVLAN_ETYPE(0x8100));
626 t4_set_reg_field(adapter, MPS_PORT_RX_CTL(i),
627 F_OVLAN_EN0 | F_OVLAN_EN1 |
628 F_OVLAN_EN2 | F_IVLAN_EN,
629 F_OVLAN_EN0 | F_OVLAN_EN1 |
630 F_OVLAN_EN2 | F_IVLAN_EN);
633 if (cxgbe_get_devargs(pdev->device.devargs, CXGBE_DEVARG_KEEP_OVLAN))
634 t4_tp_wr_bits_indirect(adapter, A_TP_INGRESS_CONFIG,
635 V_RM_OVLAN(1), V_RM_OVLAN(0));
638 static void configure_pcie_ext_tag(struct adapter *adapter)
641 int pos = t4_os_find_pci_capability(adapter, PCI_CAP_ID_EXP);
647 t4_os_pci_read_cfg2(adapter, pos + PCI_EXP_DEVCTL, &v);
648 v |= PCI_EXP_DEVCTL_EXT_TAG;
649 t4_os_pci_write_cfg2(adapter, pos + PCI_EXP_DEVCTL, v);
650 if (is_t6(adapter->params.chip)) {
651 t4_set_reg_field(adapter, A_PCIE_CFG2,
652 V_T6_TOTMAXTAG(M_T6_TOTMAXTAG),
654 t4_set_reg_field(adapter, A_PCIE_CMD_CFG,
655 V_T6_MINTAG(M_T6_MINTAG),
658 t4_set_reg_field(adapter, A_PCIE_CFG2,
659 V_TOTMAXTAG(M_TOTMAXTAG),
661 t4_set_reg_field(adapter, A_PCIE_CMD_CFG,
669 * Tweak configuration based on system architecture, etc. Most of these have
670 * defaults assigned to them by Firmware Configuration Files (if we're using
671 * them) but need to be explicitly set if we're using hard-coded
672 * initialization. So these are essentially common tweaks/settings for
673 * Configuration Files and hard-coded initialization ...
675 static int adap_init0_tweaks(struct adapter *adapter)
680 * Fix up various Host-Dependent Parameters like Page Size, Cache
681 * Line Size, etc. The firmware default is for a 4KB Page Size and
682 * 64B Cache Line Size ...
684 t4_fixup_host_params_compat(adapter, CXGBE_PAGE_SIZE, L1_CACHE_BYTES,
688 * Keep the chip default offset to deliver Ingress packets into our
689 * DMA buffers to zero
692 t4_set_reg_field(adapter, A_SGE_CONTROL, V_PKTSHIFT(M_PKTSHIFT),
693 V_PKTSHIFT(rx_dma_offset));
695 t4_set_reg_field(adapter, A_SGE_FLM_CFG,
696 V_CREDITCNT(M_CREDITCNT) | M_CREDITCNTPACKING,
697 V_CREDITCNT(3) | V_CREDITCNTPACKING(1));
699 t4_set_reg_field(adapter, A_SGE_INGRESS_RX_THRESHOLD,
700 V_THRESHOLD_3(M_THRESHOLD_3), V_THRESHOLD_3(32U));
702 t4_set_reg_field(adapter, A_SGE_CONTROL2, V_IDMAARBROUNDROBIN(1U),
703 V_IDMAARBROUNDROBIN(1U));
706 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
707 * adds the pseudo header itself.
709 t4_tp_wr_bits_indirect(adapter, A_TP_INGRESS_CONFIG,
710 F_CSUM_HAS_PSEUDO_HDR, 0);
716 * Attempt to initialize the adapter via a Firmware Configuration File.
718 static int adap_init0_config(struct adapter *adapter, int reset)
720 struct fw_caps_config_cmd caps_cmd;
721 unsigned long mtype = 0, maddr = 0;
722 u32 finiver, finicsum, cfcsum;
724 int config_issued = 0;
726 char config_name[20];
729 * Reset device if necessary.
732 ret = t4_fw_reset(adapter, adapter->mbox,
733 F_PIORSTMODE | F_PIORST);
735 dev_warn(adapter, "Firmware reset failed, error %d\n",
741 cfg_addr = t4_flash_cfg_addr(adapter);
744 dev_warn(adapter, "Finding address for firmware config file in flash failed, error %d\n",
749 strcpy(config_name, "On Flash");
750 mtype = FW_MEMTYPE_CF_FLASH;
754 * Issue a Capability Configuration command to the firmware to get it
755 * to parse the Configuration File. We don't use t4_fw_config_file()
756 * because we want the ability to modify various features after we've
757 * processed the configuration file ...
759 memset(&caps_cmd, 0, sizeof(caps_cmd));
760 caps_cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
761 F_FW_CMD_REQUEST | F_FW_CMD_READ);
762 caps_cmd.cfvalid_to_len16 =
763 cpu_to_be32(F_FW_CAPS_CONFIG_CMD_CFVALID |
764 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
765 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
767 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
770 * If the CAPS_CONFIG failed with an ENOENT (for a Firmware
771 * Configuration File in FLASH), our last gasp effort is to use the
772 * Firmware Configuration File which is embedded in the firmware. A
773 * very few early versions of the firmware didn't have one embedded
774 * but we can ignore those.
776 if (ret == -ENOENT) {
777 dev_info(adapter, "%s: Going for embedded config in firmware..\n",
780 memset(&caps_cmd, 0, sizeof(caps_cmd));
781 caps_cmd.op_to_write =
782 cpu_to_be32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
783 F_FW_CMD_REQUEST | F_FW_CMD_READ);
784 caps_cmd.cfvalid_to_len16 = cpu_to_be32(FW_LEN16(caps_cmd));
785 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
786 sizeof(caps_cmd), &caps_cmd);
787 strcpy(config_name, "Firmware Default");
794 finiver = be32_to_cpu(caps_cmd.finiver);
795 finicsum = be32_to_cpu(caps_cmd.finicsum);
796 cfcsum = be32_to_cpu(caps_cmd.cfcsum);
797 if (finicsum != cfcsum)
798 dev_warn(adapter, "Configuration File checksum mismatch: [fini] csum=%#x, computed csum=%#x\n",
802 * If we're a pure NIC driver then disable all offloading facilities.
803 * This will allow the firmware to optimize aspects of the hardware
804 * configuration which will result in improved performance.
806 caps_cmd.niccaps &= cpu_to_be16(~FW_CAPS_CONFIG_NIC_ETHOFLD);
807 caps_cmd.toecaps = 0;
808 caps_cmd.iscsicaps = 0;
809 caps_cmd.rdmacaps = 0;
810 caps_cmd.fcoecaps = 0;
813 * And now tell the firmware to use the configuration we just loaded.
815 caps_cmd.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
816 F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
817 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
818 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
821 dev_warn(adapter, "Unable to finalize Firmware Capabilities %d\n",
827 * Tweak configuration based on system architecture, etc.
829 ret = adap_init0_tweaks(adapter);
831 dev_warn(adapter, "Unable to do init0-tweaks %d\n", -ret);
836 * And finally tell the firmware to initialize itself using the
837 * parameters from the Configuration File.
839 ret = t4_fw_initialize(adapter, adapter->mbox);
841 dev_warn(adapter, "Initializing Firmware failed, error %d\n",
847 * Return successfully and note that we're operating with parameters
848 * not supplied by the driver, rather than from hard-wired
849 * initialization constants buried in the driver.
852 "Successfully configured using Firmware Configuration File \"%s\", version %#x, computed checksum %#x\n",
853 config_name, finiver, cfcsum);
858 * Something bad happened. Return the error ... (If the "error"
859 * is that there's no Configuration File on the adapter we don't
860 * want to issue a warning since this is fairly common.)
863 if (config_issued && ret != -ENOENT)
864 dev_warn(adapter, "\"%s\" configuration file error %d\n",
867 dev_debug(adapter, "%s: returning ret = %d ..\n", __func__, ret);
871 static int adap_init0(struct adapter *adap)
873 struct fw_caps_config_cmd caps_cmd;
876 enum dev_state state;
877 u32 params[7], val[7];
879 int mbox = adap->mbox;
882 * Contact FW, advertising Master capability.
884 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
886 dev_err(adap, "%s: could not connect to FW, error %d\n",
891 CXGBE_DEBUG_MBOX(adap, "%s: adap->mbox = %d; ret = %d\n", __func__,
895 adap->flags |= MASTER_PF;
897 if (state == DEV_STATE_INIT) {
899 * Force halt and reset FW because a previous instance may have
900 * exited abnormally without properly shutting down
902 ret = t4_fw_halt(adap, adap->mbox, reset);
904 dev_err(adap, "Failed to halt. Exit.\n");
908 ret = t4_fw_restart(adap, adap->mbox, reset);
910 dev_err(adap, "Failed to restart. Exit.\n");
913 state = (enum dev_state)((unsigned)state & ~DEV_STATE_INIT);
916 t4_get_version_info(adap);
918 ret = t4_get_core_clock(adap, &adap->params.vpd);
920 dev_err(adap, "%s: could not get core clock, error %d\n",
926 * If the firmware is initialized already (and we're not forcing a
927 * master initialization), note that we're living with existing
928 * adapter parameters. Otherwise, it's time to try initializing the
931 if (state == DEV_STATE_INIT) {
932 dev_info(adap, "Coming up as %s: Adapter already initialized\n",
933 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
935 dev_info(adap, "Coming up as MASTER: Initializing adapter\n");
937 ret = adap_init0_config(adap, reset);
938 if (ret == -ENOENT) {
940 "No Configuration File present on adapter. Using hard-wired configuration parameters.\n");
945 dev_err(adap, "could not initialize adapter, error %d\n", -ret);
949 /* Find out what ports are available to us. */
950 v = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
951 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
952 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec);
954 dev_err(adap, "%s: failure in t4_query_params; error = %d\n",
959 adap->params.nports = hweight32(port_vec);
960 adap->params.portvec = port_vec;
962 dev_debug(adap, "%s: adap->params.nports = %u\n", __func__,
963 adap->params.nports);
966 * Give the SGE code a chance to pull in anything that it needs ...
967 * Note that this must be called after we retrieve our VPD parameters
968 * in order to know how to convert core ticks to seconds, etc.
970 ret = t4_sge_init(adap);
972 dev_err(adap, "t4_sge_init failed with error %d\n",
978 * Grab some of our basic fundamental operating parameters.
980 #define FW_PARAM_DEV(param) \
981 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
982 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
984 #define FW_PARAM_PFVF(param) \
985 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
986 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
987 V_FW_PARAMS_PARAM_Y(0) | \
988 V_FW_PARAMS_PARAM_Z(0))
990 params[0] = FW_PARAM_PFVF(FILTER_START);
991 params[1] = FW_PARAM_PFVF(FILTER_END);
992 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val);
995 adap->tids.ftid_base = val[0];
996 adap->tids.nftids = val[1] - val[0] + 1;
999 * Get device capabilities so we can determine what resources we need
1002 memset(&caps_cmd, 0, sizeof(caps_cmd));
1003 caps_cmd.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1004 F_FW_CMD_REQUEST | F_FW_CMD_READ);
1005 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
1006 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
1011 if ((caps_cmd.niccaps & cpu_to_be16(FW_CAPS_CONFIG_NIC_HASHFILTER)) &&
1012 is_t6(adap->params.chip)) {
1013 if (init_hash_filter(adap) < 0)
1017 /* query tid-related parameters */
1018 params[0] = FW_PARAM_DEV(NTID);
1019 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
1023 adap->tids.ntids = val[0];
1024 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
1026 /* If we're running on newer firmware, let it know that we're
1027 * prepared to deal with encapsulated CPL messages. Older
1028 * firmware won't understand this and we'll just get
1029 * unencapsulated messages ...
1031 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
1033 (void)t4_set_params(adap, adap->mbox, adap->pf, 0, 1, params, val);
1036 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
1037 * capability. Earlier versions of the firmware didn't have the
1038 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
1039 * permission to use ULPTX MEMWRITE DSGL.
1041 if (is_t4(adap->params.chip)) {
1042 adap->params.ulptx_memwrite_dsgl = false;
1044 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
1045 ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
1047 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
1051 * The MTU/MSS Table is initialized by now, so load their values. If
1052 * we're initializing the adapter, then we'll make any modifications
1053 * we want to the MTU/MSS Table and also initialize the congestion
1056 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
1057 if (state != DEV_STATE_INIT) {
1061 * The default MTU Table contains values 1492 and 1500.
1062 * However, for TCP, it's better to have two values which are
1063 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
1064 * This allows us to have a TCP Data Payload which is a
1065 * multiple of 8 regardless of what combination of TCP Options
1066 * are in use (always a multiple of 4 bytes) which is
1067 * important for performance reasons. For instance, if no
1068 * options are in use, then we have a 20-byte IP header and a
1069 * 20-byte TCP header. In this case, a 1500-byte MSS would
1070 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
1071 * which is not a multiple of 8. So using an MSS of 1488 in
1072 * this case results in a TCP Data Payload of 1448 bytes which
1073 * is a multiple of 8. On the other hand, if 12-byte TCP Time
1074 * Stamps have been negotiated, then an MTU of 1500 bytes
1075 * results in a TCP Data Payload of 1448 bytes which, as
1076 * above, is a multiple of 8 bytes ...
1078 for (i = 0; i < NMTUS; i++)
1079 if (adap->params.mtus[i] == 1492) {
1080 adap->params.mtus[i] = 1488;
1084 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
1085 adap->params.b_wnd);
1087 t4_init_sge_params(adap);
1088 t4_init_tp_params(adap);
1089 configure_pcie_ext_tag(adap);
1090 configure_vlan_types(adap);
1092 adap->params.drv_memwin = MEMWIN_NIC;
1093 adap->flags |= FW_OK;
1094 dev_debug(adap, "%s: returning zero..\n", __func__);
1098 * Something bad happened. If a command timed out or failed with EIO
1099 * FW does not operate within its spec or something catastrophic
1100 * happened to HW/FW, stop issuing commands.
1103 if (ret != -ETIMEDOUT && ret != -EIO)
1104 t4_fw_bye(adap, adap->mbox);
1109 * t4_os_portmod_changed - handle port module changes
1110 * @adap: the adapter associated with the module change
1111 * @port_id: the port index whose module status has changed
1113 * This is the OS-dependent handler for port module changes. It is
1114 * invoked when a port module is removed or inserted for any OS-specific
1117 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
1119 static const char * const mod_str[] = {
1120 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
1123 const struct port_info *pi = adap2pinfo(adap, port_id);
1125 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
1126 dev_info(adap, "Port%d: port module unplugged\n", pi->port_id);
1127 else if (pi->mod_type < ARRAY_SIZE(mod_str))
1128 dev_info(adap, "Port%d: %s port module inserted\n", pi->port_id,
1129 mod_str[pi->mod_type]);
1130 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
1131 dev_info(adap, "Port%d: unsupported port module inserted\n",
1133 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
1134 dev_info(adap, "Port%d: unknown port module inserted\n",
1136 else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR)
1137 dev_info(adap, "Port%d: transceiver module error\n",
1140 dev_info(adap, "Port%d: unknown module type %d inserted\n",
1141 pi->port_id, pi->mod_type);
1144 inline bool force_linkup(struct adapter *adap)
1146 struct rte_pci_device *pdev = adap->pdev;
1149 return false; /* force_linkup not required for pf driver*/
1150 if (!cxgbe_get_devargs(pdev->device.devargs,
1151 CXGBE_DEVARG_FORCE_LINK_UP))
1157 * link_start - enable a port
1158 * @dev: the port to enable
1160 * Performs the MAC and PHY actions needed to enable a port.
1162 int link_start(struct port_info *pi)
1164 struct adapter *adapter = pi->adapter;
1168 mtu = pi->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len -
1169 (ETHER_HDR_LEN + ETHER_CRC_LEN);
1172 * We do not set address filters and promiscuity here, the stack does
1173 * that step explicitly.
1175 ret = t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu, -1, -1,
1178 ret = t4_change_mac(adapter, adapter->mbox, pi->viid,
1180 (u8 *)&pi->eth_dev->data->mac_addrs[0],
1183 pi->xact_addr_filt = ret;
1187 if (ret == 0 && is_pf4(adapter))
1188 ret = t4_link_l1cfg(adapter, adapter->mbox, pi->tx_chan,
1192 * Enabling a Virtual Interface can result in an interrupt
1193 * during the processing of the VI Enable command and, in some
1194 * paths, result in an attempt to issue another command in the
1195 * interrupt context. Thus, we disable interrupts during the
1196 * course of the VI Enable command ...
1198 ret = t4_enable_vi_params(adapter, adapter->mbox, pi->viid,
1202 if (ret == 0 && force_linkup(adapter))
1203 pi->eth_dev->data->dev_link.link_status = ETH_LINK_UP;
1208 * cxgbe_write_rss_conf - flash the RSS configuration for a given port
1210 * @rss_hf: Hash configuration to apply
1212 int cxgbe_write_rss_conf(const struct port_info *pi, uint64_t rss_hf)
1214 struct adapter *adapter = pi->adapter;
1215 const struct sge_eth_rxq *rxq;
1220 /* Should never be called before setting up sge eth rx queues */
1221 if (!(adapter->flags & FULL_INIT_DONE)) {
1222 dev_err(adap, "%s No RXQs available on port %d\n",
1223 __func__, pi->port_id);
1227 /* Don't allow unsupported hash functions */
1228 if (rss_hf & ~CXGBE_RSS_HF_ALL)
1231 if (rss_hf & ETH_RSS_IPV4)
1232 flags |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
1234 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1235 flags |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
1237 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
1238 flags |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
1239 F_FW_RSS_VI_CONFIG_CMD_UDPEN;
1241 if (rss_hf & ETH_RSS_IPV6)
1242 flags |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
1244 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1245 flags |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
1247 if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
1248 flags |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
1249 F_FW_RSS_VI_CONFIG_CMD_UDPEN;
1251 rxq = &adapter->sge.ethrxq[pi->first_qset];
1252 rss = rxq[0].rspq.abs_id;
1254 /* If Tunnel All Lookup isn't specified in the global RSS
1255 * Configuration, then we need to specify a default Ingress
1256 * Queue for any ingress packets which aren't hashed. We'll
1257 * use our first ingress queue ...
1259 err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid,
1265 * cxgbe_write_rss - write the RSS table for a given port
1267 * @queues: array of queue indices for RSS
1269 * Sets up the portion of the HW RSS table for the port's VI to distribute
1270 * packets to the Rx queues in @queues.
1272 int cxgbe_write_rss(const struct port_info *pi, const u16 *queues)
1276 struct adapter *adapter = pi->adapter;
1277 const struct sge_eth_rxq *rxq;
1279 /* Should never be called before setting up sge eth rx queues */
1280 BUG_ON(!(adapter->flags & FULL_INIT_DONE));
1282 rxq = &adapter->sge.ethrxq[pi->first_qset];
1283 rss = rte_zmalloc(NULL, pi->rss_size * sizeof(u16), 0);
1287 /* map the queue indices to queue ids */
1288 for (i = 0; i < pi->rss_size; i++, queues++)
1289 rss[i] = rxq[*queues].rspq.abs_id;
1291 err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0,
1292 pi->rss_size, rss, pi->rss_size);
1298 * setup_rss - configure RSS
1299 * @adapter: the adapter
1301 * Sets up RSS to distribute packets to multiple receive queues. We
1302 * configure the RSS CPU lookup table to distribute to the number of HW
1303 * receive queues, and the response queue lookup table to narrow that
1304 * down to the response queues actually configured for each port.
1305 * We always configure the RSS mapping for all ports since the mapping
1306 * table has plenty of entries.
1308 int setup_rss(struct port_info *pi)
1311 struct adapter *adapter = pi->adapter;
1313 dev_debug(adapter, "%s: pi->rss_size = %u; pi->n_rx_qsets = %u\n",
1314 __func__, pi->rss_size, pi->n_rx_qsets);
1316 if (!(pi->flags & PORT_RSS_DONE)) {
1317 if (adapter->flags & FULL_INIT_DONE) {
1318 /* Fill default values with equal distribution */
1319 for (j = 0; j < pi->rss_size; j++)
1320 pi->rss[j] = j % pi->n_rx_qsets;
1322 err = cxgbe_write_rss(pi, pi->rss);
1326 err = cxgbe_write_rss_conf(pi, pi->rss_hf);
1329 pi->flags |= PORT_RSS_DONE;
1336 * Enable NAPI scheduling and interrupt generation for all Rx queues.
1338 static void enable_rx(struct adapter *adap, struct sge_rspq *q)
1340 /* 0-increment GTS to start the timer and enable interrupts */
1341 t4_write_reg(adap, is_pf4(adap) ? MYPF_REG(A_SGE_PF_GTS) :
1342 T4VF_SGE_BASE_ADDR + A_SGE_VF_GTS,
1343 V_SEINTARM(q->intr_params) |
1344 V_INGRESSQID(q->cntxt_id));
1347 void cxgbe_enable_rx_queues(struct port_info *pi)
1349 struct adapter *adap = pi->adapter;
1350 struct sge *s = &adap->sge;
1353 for (i = 0; i < pi->n_rx_qsets; i++)
1354 enable_rx(adap, &s->ethrxq[pi->first_qset + i].rspq);
1358 * fw_caps_to_speed_caps - translate Firmware Port Caps to Speed Caps.
1359 * @port_type: Firmware Port Type
1360 * @fw_caps: Firmware Port Capabilities
1361 * @speed_caps: Device Info Speed Capabilities
1363 * Translate a Firmware Port Capabilities specification to Device Info
1364 * Speed Capabilities.
1366 static void fw_caps_to_speed_caps(enum fw_port_type port_type,
1367 unsigned int fw_caps,
1370 #define SET_SPEED(__speed_name) \
1372 *speed_caps |= ETH_LINK_ ## __speed_name; \
1375 #define FW_CAPS_TO_SPEED(__fw_name) \
1377 if (fw_caps & FW_PORT_CAP32_ ## __fw_name) \
1378 SET_SPEED(__fw_name); \
1381 switch (port_type) {
1382 case FW_PORT_TYPE_BT_SGMII:
1383 case FW_PORT_TYPE_BT_XFI:
1384 case FW_PORT_TYPE_BT_XAUI:
1385 FW_CAPS_TO_SPEED(SPEED_100M);
1386 FW_CAPS_TO_SPEED(SPEED_1G);
1387 FW_CAPS_TO_SPEED(SPEED_10G);
1390 case FW_PORT_TYPE_KX4:
1391 case FW_PORT_TYPE_KX:
1392 case FW_PORT_TYPE_FIBER_XFI:
1393 case FW_PORT_TYPE_FIBER_XAUI:
1394 case FW_PORT_TYPE_SFP:
1395 case FW_PORT_TYPE_QSFP_10G:
1396 case FW_PORT_TYPE_QSA:
1397 FW_CAPS_TO_SPEED(SPEED_1G);
1398 FW_CAPS_TO_SPEED(SPEED_10G);
1401 case FW_PORT_TYPE_KR:
1402 SET_SPEED(SPEED_10G);
1405 case FW_PORT_TYPE_BP_AP:
1406 case FW_PORT_TYPE_BP4_AP:
1407 SET_SPEED(SPEED_1G);
1408 SET_SPEED(SPEED_10G);
1411 case FW_PORT_TYPE_BP40_BA:
1412 case FW_PORT_TYPE_QSFP:
1413 SET_SPEED(SPEED_40G);
1416 case FW_PORT_TYPE_CR_QSFP:
1417 case FW_PORT_TYPE_SFP28:
1418 case FW_PORT_TYPE_KR_SFP28:
1419 FW_CAPS_TO_SPEED(SPEED_1G);
1420 FW_CAPS_TO_SPEED(SPEED_10G);
1421 FW_CAPS_TO_SPEED(SPEED_25G);
1424 case FW_PORT_TYPE_CR2_QSFP:
1425 SET_SPEED(SPEED_50G);
1428 case FW_PORT_TYPE_KR4_100G:
1429 case FW_PORT_TYPE_CR4_QSFP:
1430 FW_CAPS_TO_SPEED(SPEED_25G);
1431 FW_CAPS_TO_SPEED(SPEED_40G);
1432 FW_CAPS_TO_SPEED(SPEED_50G);
1433 FW_CAPS_TO_SPEED(SPEED_100G);
1440 #undef FW_CAPS_TO_SPEED
1445 * cxgbe_get_speed_caps - Fetch supported speed capabilities
1446 * @pi: Underlying port's info
1447 * @speed_caps: Device Info speed capabilities
1449 * Fetch supported speed capabilities of the underlying port.
1451 void cxgbe_get_speed_caps(struct port_info *pi, u32 *speed_caps)
1455 fw_caps_to_speed_caps(pi->port_type, pi->link_cfg.pcaps,
1458 if (!(pi->link_cfg.pcaps & FW_PORT_CAP32_ANEG))
1459 *speed_caps |= ETH_LINK_SPEED_FIXED;
1463 * cxgb_up - enable the adapter
1464 * @adap: adapter being enabled
1466 * Called when the first port is enabled, this function performs the
1467 * actions necessary to make an adapter operational, such as completing
1468 * the initialization of HW modules, and enabling interrupts.
1470 int cxgbe_up(struct adapter *adap)
1472 enable_rx(adap, &adap->sge.fw_evtq);
1473 t4_sge_tx_monitor_start(adap);
1475 t4_intr_enable(adap);
1476 adap->flags |= FULL_INIT_DONE;
1478 /* TODO: deadman watchdog ?? */
1485 int cxgbe_down(struct port_info *pi)
1487 struct adapter *adapter = pi->adapter;
1490 err = t4_enable_vi(adapter, adapter->mbox, pi->viid, false, false);
1492 dev_err(adapter, "%s: disable_vi failed: %d\n", __func__, err);
1496 t4_reset_link_config(adapter, pi->pidx);
1501 * Release resources when all the ports have been stopped.
1503 void cxgbe_close(struct adapter *adapter)
1505 struct port_info *pi;
1508 if (adapter->flags & FULL_INIT_DONE) {
1509 if (is_pf4(adapter))
1510 t4_intr_disable(adapter);
1511 tid_free(&adapter->tids);
1512 t4_sge_tx_monitor_stop(adapter);
1513 t4_free_sge_resources(adapter);
1514 for_each_port(adapter, i) {
1515 pi = adap2pinfo(adapter, i);
1517 t4_free_vi(adapter, adapter->mbox,
1518 adapter->pf, 0, pi->viid);
1519 rte_free(pi->eth_dev->data->mac_addrs);
1520 /* Skip first port since it'll be freed by DPDK stack */
1522 rte_free(pi->eth_dev->data->dev_private);
1523 rte_eth_dev_release_port(pi->eth_dev);
1526 adapter->flags &= ~FULL_INIT_DONE;
1529 if (is_pf4(adapter) && (adapter->flags & FW_OK))
1530 t4_fw_bye(adapter, adapter->mbox);
1533 int cxgbe_probe(struct adapter *adapter)
1535 struct port_info *pi;
1541 whoami = t4_read_reg(adapter, A_PL_WHOAMI);
1542 chip = t4_get_chip_type(adapter,
1543 CHELSIO_PCI_ID_VER(adapter->pdev->id.device_id));
1547 func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ?
1548 G_SOURCEPF(whoami) : G_T6_SOURCEPF(whoami);
1550 adapter->mbox = func;
1553 t4_os_lock_init(&adapter->mbox_lock);
1554 TAILQ_INIT(&adapter->mbox_list);
1555 t4_os_lock_init(&adapter->win0_lock);
1557 err = t4_prep_adapter(adapter);
1561 setup_memwin(adapter);
1562 err = adap_init0(adapter);
1564 dev_err(adapter, "%s: Adapter initialization failed, error %d\n",
1569 if (!is_t4(adapter->params.chip)) {
1571 * The userspace doorbell BAR is split evenly into doorbell
1572 * regions, each associated with an egress queue. If this
1573 * per-queue region is large enough (at least UDBS_SEG_SIZE)
1574 * then it can be used to submit a tx work request with an
1575 * implied doorbell. Enable write combining on the BAR if
1576 * there is room for such work requests.
1578 int s_qpp, qpp, num_seg;
1580 s_qpp = (S_QUEUESPERPAGEPF0 +
1581 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) *
1583 qpp = 1 << ((t4_read_reg(adapter,
1584 A_SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp)
1585 & M_QUEUESPERPAGEPF0);
1586 num_seg = CXGBE_PAGE_SIZE / UDBS_SEG_SIZE;
1588 dev_warn(adapter, "Incorrect SGE EGRESS QUEUES_PER_PAGE configuration, continuing in debug mode\n");
1590 adapter->bar2 = (void *)adapter->pdev->mem_resource[2].addr;
1591 if (!adapter->bar2) {
1592 dev_err(adapter, "cannot map device bar2 region\n");
1596 t4_write_reg(adapter, A_SGE_STAT_CFG, V_STATSOURCE_T5(7) |
1600 for_each_port(adapter, i) {
1601 const unsigned int numa_node = rte_socket_id();
1602 char name[RTE_ETH_NAME_MAX_LEN];
1603 struct rte_eth_dev *eth_dev;
1605 snprintf(name, sizeof(name), "%s_%d",
1606 adapter->pdev->device.name, i);
1609 /* First port is already allocated by DPDK */
1610 eth_dev = adapter->eth_dev;
1615 * now do all data allocation - for eth_dev structure,
1616 * and internal (private) data for the remaining ports
1619 /* reserve an ethdev entry */
1620 eth_dev = rte_eth_dev_allocate(name);
1624 eth_dev->data->dev_private =
1625 rte_zmalloc_socket(name, sizeof(struct port_info),
1626 RTE_CACHE_LINE_SIZE, numa_node);
1627 if (!eth_dev->data->dev_private)
1631 pi = (struct port_info *)eth_dev->data->dev_private;
1632 adapter->port[i] = pi;
1633 pi->eth_dev = eth_dev;
1634 pi->adapter = adapter;
1635 pi->xact_addr_filt = -1;
1639 pi->eth_dev->device = &adapter->pdev->device;
1640 pi->eth_dev->dev_ops = adapter->eth_dev->dev_ops;
1641 pi->eth_dev->tx_pkt_burst = adapter->eth_dev->tx_pkt_burst;
1642 pi->eth_dev->rx_pkt_burst = adapter->eth_dev->rx_pkt_burst;
1644 rte_eth_copy_pci_info(pi->eth_dev, adapter->pdev);
1646 pi->eth_dev->data->mac_addrs = rte_zmalloc(name,
1648 if (!pi->eth_dev->data->mac_addrs) {
1649 dev_err(adapter, "%s: Mem allocation failed for storing mac addr, aborting\n",
1656 /* First port will be notified by upper layer */
1657 rte_eth_dev_probing_finish(eth_dev);
1661 if (adapter->flags & FW_OK) {
1662 err = t4_port_init(adapter, adapter->mbox, adapter->pf, 0);
1664 dev_err(adapter, "%s: t4_port_init failed with err %d\n",
1670 cfg_queues(adapter->eth_dev);
1672 print_adapter_info(adapter);
1673 print_port_info(adapter);
1675 if (tid_init(&adapter->tids) < 0) {
1676 /* Disable filtering support */
1677 dev_warn(adapter, "could not allocate TID table, "
1678 "filter support disabled. Continuing\n");
1681 if (is_hashfilter(adapter)) {
1682 if (t4_read_reg(adapter, A_LE_DB_CONFIG) & F_HASHEN) {
1683 u32 hash_base, hash_reg;
1685 hash_reg = A_LE_DB_TID_HASHBASE;
1686 hash_base = t4_read_reg(adapter, hash_reg);
1687 adapter->tids.hash_base = hash_base / 4;
1690 /* Disable hash filtering support */
1692 "Maskless filter support disabled. Continuing\n");
1695 err = init_rss(adapter);
1702 for_each_port(adapter, i) {
1703 pi = adap2pinfo(adapter, i);
1705 t4_free_vi(adapter, adapter->mbox, adapter->pf,
1707 /* Skip first port since it'll be de-allocated by DPDK */
1711 if (pi->eth_dev->data->dev_private)
1712 rte_free(pi->eth_dev->data->dev_private);
1713 rte_eth_dev_release_port(pi->eth_dev);
1717 if (adapter->flags & FW_OK)
1718 t4_fw_bye(adapter, adapter->mbox);