1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2020 Intel Corporation
13 #include <rte_malloc.h>
14 #include <rte_errno.h>
16 #include "base/dlb2_regs_new.h"
17 #include "base/dlb2_hw_types.h"
18 #include "base/dlb2_resource.h"
19 #include "base/dlb2_osdep.h"
20 #include "dlb2_main.h"
21 #include "../dlb2_user.h"
22 #include "../dlb2_priv.h"
23 #include "../dlb2_iface.h"
24 #include "../dlb2_inline_fns.h"
26 #define PF_ID_ZERO 0 /* PF ONLY! */
27 #define NO_OWNER_VF 0 /* PF ONLY! */
28 #define NOT_VF_REQ false /* PF ONLY! */
30 #define DLB2_PCI_CAP_POINTER 0x34
31 #define DLB2_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
32 #define DLB2_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
34 #define DLB2_PCI_LNKCTL 16
35 #define DLB2_PCI_SLTCTL 24
36 #define DLB2_PCI_RTCTL 28
37 #define DLB2_PCI_EXP_DEVCTL2 40
38 #define DLB2_PCI_LNKCTL2 48
39 #define DLB2_PCI_SLTCTL2 56
40 #define DLB2_PCI_CMD 4
41 #define DLB2_PCI_EXP_DEVSTA 10
42 #define DLB2_PCI_EXP_DEVSTA_TRPND 0x20
43 #define DLB2_PCI_EXP_DEVCTL_BCR_FLR 0x8000
45 #define DLB2_PCI_CAP_ID_EXP 0x10
46 #define DLB2_PCI_CAP_ID_MSIX 0x11
47 #define DLB2_PCI_EXT_CAP_ID_PRI 0x13
48 #define DLB2_PCI_EXT_CAP_ID_ACS 0xD
50 #define DLB2_PCI_PRI_CTRL_ENABLE 0x1
51 #define DLB2_PCI_PRI_ALLOC_REQ 0xC
52 #define DLB2_PCI_PRI_CTRL 0x4
53 #define DLB2_PCI_MSIX_FLAGS 0x2
54 #define DLB2_PCI_MSIX_FLAGS_ENABLE 0x8000
55 #define DLB2_PCI_MSIX_FLAGS_MASKALL 0x4000
56 #define DLB2_PCI_ERR_ROOT_STATUS 0x30
57 #define DLB2_PCI_ERR_COR_STATUS 0x10
58 #define DLB2_PCI_ERR_UNCOR_STATUS 0x4
59 #define DLB2_PCI_COMMAND_INTX_DISABLE 0x400
60 #define DLB2_PCI_ACS_CAP 0x4
61 #define DLB2_PCI_ACS_CTRL 0x6
62 #define DLB2_PCI_ACS_SV 0x1
63 #define DLB2_PCI_ACS_RR 0x4
64 #define DLB2_PCI_ACS_CR 0x8
65 #define DLB2_PCI_ACS_UF 0x10
66 #define DLB2_PCI_ACS_EC 0x20
68 static int dlb2_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
74 ret = rte_pci_read_config(pdev, &pos, 1, DLB2_PCI_CAP_POINTER);
81 ret = rte_pci_read_config(pdev, &hdr, 2, pos);
85 if (DLB2_PCI_CAP_ID(hdr) == id)
88 if (DLB2_PCI_CAP_ID(hdr) == 0xFF)
91 pos = DLB2_PCI_CAP_NEXT(hdr);
98 dlb2_pf_init_driver_state(struct dlb2_dev *dlb2_dev)
100 rte_spinlock_init(&dlb2_dev->resource_mutex);
105 static void dlb2_pf_enable_pm(struct dlb2_dev *dlb2_dev)
108 version = DLB2_HW_DEVICE_FROM_PCI_ID(dlb2_dev->pdev);
110 dlb2_clr_pmcsr_disable(&dlb2_dev->hw, version);
113 #define DLB2_READY_RETRY_LIMIT 1000
114 static int dlb2_pf_wait_for_device_ready(struct dlb2_dev *dlb2_dev,
119 /* Allow at least 1s for the device to become active after power-on */
120 for (retries = 0; retries < DLB2_READY_RETRY_LIMIT; retries++) {
122 u32 idle_dlb_func_idle;
127 addr = DLB2_CM_CFG_PM_STATUS(dlb_version);
128 pm_st_val = DLB2_CSR_RD(&dlb2_dev->hw, addr);
129 addr = DLB2_CM_CFG_DIAGNOSTIC_IDLE_STATUS(dlb_version);
130 idle_val = DLB2_CSR_RD(&dlb2_dev->hw, addr);
131 idle_dlb_func_idle = idle_val &
132 DLB2_CM_CFG_DIAGNOSTIC_IDLE_STATUS_DLB_FUNC_IDLE;
133 pm_st_pmsm = pm_st_val & DLB2_CM_CFG_PM_STATUS_PMSM;
134 if (pm_st_pmsm && idle_dlb_func_idle)
140 if (retries == DLB2_READY_RETRY_LIMIT) {
141 DLB2_LOG_ERR("[%s()] wait for device ready timed out\n",
150 dlb2_probe(struct rte_pci_device *pdev)
152 struct dlb2_dev *dlb2_dev;
156 DLB2_INFO(dlb2_dev, "probe\n");
158 dlb2_dev = rte_malloc("DLB2_PF", sizeof(struct dlb2_dev),
159 RTE_CACHE_LINE_SIZE);
161 if (dlb2_dev == NULL) {
163 goto dlb2_dev_malloc_fail;
166 dlb_version = DLB2_HW_DEVICE_FROM_PCI_ID(pdev);
168 /* PCI Bus driver has already mapped bar space into process.
169 * Save off our IO register and FUNC addresses.
173 if (pdev->mem_resource[0].addr == NULL) {
174 DLB2_ERR(dlb2_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
176 goto pci_mmap_bad_addr;
178 dlb2_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
179 dlb2_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
181 DLB2_INFO(dlb2_dev, "DLB2 FUNC VA=%p, PA=%p, len=%p\n",
182 (void *)dlb2_dev->hw.func_kva,
183 (void *)dlb2_dev->hw.func_phys_addr,
184 (void *)(pdev->mem_resource[0].len));
187 if (pdev->mem_resource[2].addr == NULL) {
188 DLB2_ERR(dlb2_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
190 goto pci_mmap_bad_addr;
192 dlb2_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
193 dlb2_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
195 DLB2_INFO(dlb2_dev, "DLB2 CSR VA=%p, PA=%p, len=%p\n",
196 (void *)dlb2_dev->hw.csr_kva,
197 (void *)dlb2_dev->hw.csr_phys_addr,
198 (void *)(pdev->mem_resource[2].len));
200 dlb2_dev->pdev = pdev;
202 /* PM enable must be done before any other MMIO accesses, and this
203 * setting is persistent across device reset.
205 dlb2_pf_enable_pm(dlb2_dev);
207 ret = dlb2_pf_wait_for_device_ready(dlb2_dev, dlb_version);
209 goto wait_for_device_ready_fail;
211 ret = dlb2_pf_reset(dlb2_dev);
213 goto dlb2_reset_fail;
215 ret = dlb2_pf_init_driver_state(dlb2_dev);
217 goto init_driver_state_fail;
219 ret = dlb2_resource_init(&dlb2_dev->hw, dlb_version);
221 goto resource_init_fail;
226 dlb2_resource_free(&dlb2_dev->hw);
227 init_driver_state_fail:
230 wait_for_device_ready_fail:
232 dlb2_dev_malloc_fail:
238 dlb2_pf_reset(struct dlb2_dev *dlb2_dev)
246 uint16_t dev_ctl_word;
247 uint16_t dev_ctl2_word;
252 uint16_t rt_ctl_word;
253 uint32_t pri_reqs_dword;
254 uint16_t pri_ctrl_word;
263 uint16_t devsta_busy_word;
264 uint16_t devctl_word;
266 struct rte_pci_device *pdev = dlb2_dev->pdev;
268 /* Save PCI config state */
270 for (i = 0; i < 16; i++) {
271 if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
275 pcie_cap_offset = dlb2_pci_find_capability(pdev, DLB2_PCI_CAP_ID_EXP);
277 if (pcie_cap_offset < 0) {
278 DLB2_LOG_ERR("[%s()] failed to find the pcie capability\n",
280 return pcie_cap_offset;
283 off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL;
284 if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
287 off = pcie_cap_offset + DLB2_PCI_LNKCTL;
288 if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
291 off = pcie_cap_offset + DLB2_PCI_SLTCTL;
292 if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
295 off = pcie_cap_offset + DLB2_PCI_RTCTL;
296 if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
299 off = pcie_cap_offset + DLB2_PCI_EXP_DEVCTL2;
300 if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
303 off = pcie_cap_offset + DLB2_PCI_LNKCTL2;
304 if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
307 off = pcie_cap_offset + DLB2_PCI_SLTCTL2;
308 if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
311 off = DLB2_PCI_EXT_CAP_ID_PRI;
312 pri_cap_offset = rte_pci_find_ext_capability(pdev, off);
314 if (pri_cap_offset >= 0) {
315 off = pri_cap_offset + DLB2_PCI_PRI_ALLOC_REQ;
316 if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
320 /* clear the PCI command register before issuing the FLR */
324 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
325 DLB2_LOG_ERR("[%s()] failed to write the pci command\n",
331 for (wait_count = 0; wait_count < 4; wait_count++) {
334 off = pcie_cap_offset + DLB2_PCI_EXP_DEVSTA;
335 ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
337 DLB2_LOG_ERR("[%s()] failed to read the pci device status\n",
342 if (!(devsta_busy_word & DLB2_PCI_EXP_DEVSTA_TRPND))
345 sleep_time = (1 << (wait_count)) * 100;
346 rte_delay_ms(sleep_time);
349 if (wait_count == 4) {
350 DLB2_LOG_ERR("[%s()] wait for pci pending transactions timed out\n",
355 off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL;
356 ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
358 DLB2_LOG_ERR("[%s()] failed to read the pcie device control\n",
363 devctl_word |= DLB2_PCI_EXP_DEVCTL_BCR_FLR;
365 ret = rte_pci_write_config(pdev, &devctl_word, 2, off);
367 DLB2_LOG_ERR("[%s()] failed to write the pcie device control\n",
374 /* Restore PCI config state */
376 if (pcie_cap_offset >= 0) {
377 off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL;
378 ret = rte_pci_write_config(pdev, &dev_ctl_word, 2, off);
380 DLB2_LOG_ERR("[%s()] failed to write the pcie device control at offset %d\n",
385 off = pcie_cap_offset + DLB2_PCI_LNKCTL;
386 ret = rte_pci_write_config(pdev, &lnk_word, 2, off);
388 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
393 off = pcie_cap_offset + DLB2_PCI_SLTCTL;
394 ret = rte_pci_write_config(pdev, &slt_word, 2, off);
396 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
401 off = pcie_cap_offset + DLB2_PCI_RTCTL;
402 ret = rte_pci_write_config(pdev, &rt_ctl_word, 2, off);
404 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
409 off = pcie_cap_offset + DLB2_PCI_EXP_DEVCTL2;
410 ret = rte_pci_write_config(pdev, &dev_ctl2_word, 2, off);
412 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
417 off = pcie_cap_offset + DLB2_PCI_LNKCTL2;
418 ret = rte_pci_write_config(pdev, &lnk_word2, 2, off);
420 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
425 off = pcie_cap_offset + DLB2_PCI_SLTCTL2;
426 ret = rte_pci_write_config(pdev, &slt_word2, 2, off);
428 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
434 if (pri_cap_offset >= 0) {
435 pri_ctrl_word = DLB2_PCI_PRI_CTRL_ENABLE;
437 off = pri_cap_offset + DLB2_PCI_PRI_ALLOC_REQ;
438 ret = rte_pci_write_config(pdev, &pri_reqs_dword, 4, off);
440 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
445 off = pri_cap_offset + DLB2_PCI_PRI_CTRL;
446 ret = rte_pci_write_config(pdev, &pri_ctrl_word, 2, off);
448 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
454 off = RTE_PCI_EXT_CAP_ID_ERR;
455 err_cap_offset = rte_pci_find_ext_capability(pdev, off);
457 if (err_cap_offset >= 0) {
460 off = err_cap_offset + DLB2_PCI_ERR_ROOT_STATUS;
461 if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
464 ret = rte_pci_write_config(pdev, &tmp, 4, off);
466 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
471 off = err_cap_offset + DLB2_PCI_ERR_COR_STATUS;
472 if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
475 ret = rte_pci_write_config(pdev, &tmp, 4, off);
477 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
482 off = err_cap_offset + DLB2_PCI_ERR_UNCOR_STATUS;
483 if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
486 ret = rte_pci_write_config(pdev, &tmp, 4, off);
488 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
494 for (i = 16; i > 0; i--) {
496 ret = rte_pci_write_config(pdev, &dword[i - 1], 4, off);
498 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
505 if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
506 cmd &= ~DLB2_PCI_COMMAND_INTX_DISABLE;
507 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
508 DLB2_LOG_ERR("[%s()] failed to write the pci command\n",
514 msix_cap_offset = dlb2_pci_find_capability(pdev,
515 DLB2_PCI_CAP_ID_MSIX);
516 if (msix_cap_offset >= 0) {
517 off = msix_cap_offset + DLB2_PCI_MSIX_FLAGS;
518 if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
519 cmd |= DLB2_PCI_MSIX_FLAGS_ENABLE;
520 cmd |= DLB2_PCI_MSIX_FLAGS_MASKALL;
521 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
522 DLB2_LOG_ERR("[%s()] failed to write msix flags\n",
528 off = msix_cap_offset + DLB2_PCI_MSIX_FLAGS;
529 if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
530 cmd &= ~DLB2_PCI_MSIX_FLAGS_MASKALL;
531 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
532 DLB2_LOG_ERR("[%s()] failed to write msix flags\n",
539 off = DLB2_PCI_EXT_CAP_ID_ACS;
540 acs_cap_offset = rte_pci_find_ext_capability(pdev, off);
542 if (acs_cap_offset >= 0) {
543 uint16_t acs_cap, acs_ctrl, acs_mask;
544 off = acs_cap_offset + DLB2_PCI_ACS_CAP;
545 if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
548 off = acs_cap_offset + DLB2_PCI_ACS_CTRL;
549 if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
552 acs_mask = DLB2_PCI_ACS_SV | DLB2_PCI_ACS_RR;
553 acs_mask |= (DLB2_PCI_ACS_CR | DLB2_PCI_ACS_UF);
554 acs_ctrl |= (acs_cap & acs_mask);
556 ret = rte_pci_write_config(pdev, &acs_ctrl, 2, off);
558 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
563 off = acs_cap_offset + DLB2_PCI_ACS_CTRL;
564 if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
567 acs_mask = DLB2_PCI_ACS_RR | DLB2_PCI_ACS_CR;
568 acs_mask |= DLB2_PCI_ACS_EC;
569 acs_ctrl &= ~acs_mask;
571 off = acs_cap_offset + DLB2_PCI_ACS_CTRL;
572 ret = rte_pci_write_config(pdev, &acs_ctrl, 2, off);
574 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
584 dlb2_pf_create_sched_domain(struct dlb2_hw *hw,
585 struct dlb2_create_sched_domain_args *args,
586 struct dlb2_cmd_response *resp)
588 return dlb2_hw_create_sched_domain(hw, args, resp, NOT_VF_REQ,
593 dlb2_pf_reset_domain(struct dlb2_hw *hw, u32 id)
595 return dlb2_reset_domain(hw, id, NOT_VF_REQ, PF_ID_ZERO);
599 dlb2_pf_create_ldb_queue(struct dlb2_hw *hw,
601 struct dlb2_create_ldb_queue_args *args,
602 struct dlb2_cmd_response *resp)
604 return dlb2_hw_create_ldb_queue(hw, id, args, resp, NOT_VF_REQ,
609 dlb2_pf_create_ldb_port(struct dlb2_hw *hw,
611 struct dlb2_create_ldb_port_args *args,
612 uintptr_t cq_dma_base,
613 struct dlb2_cmd_response *resp)
615 return dlb2_hw_create_ldb_port(hw, id, args,
623 dlb2_pf_create_dir_port(struct dlb2_hw *hw,
625 struct dlb2_create_dir_port_args *args,
626 uintptr_t cq_dma_base,
627 struct dlb2_cmd_response *resp)
629 return dlb2_hw_create_dir_port(hw, id, args,
637 dlb2_pf_create_dir_queue(struct dlb2_hw *hw,
639 struct dlb2_create_dir_queue_args *args,
640 struct dlb2_cmd_response *resp)
642 return dlb2_hw_create_dir_queue(hw, id, args, resp, NOT_VF_REQ,
647 dlb2_pf_start_domain(struct dlb2_hw *hw,
649 struct dlb2_start_domain_args *args,
650 struct dlb2_cmd_response *resp)
652 return dlb2_hw_start_domain(hw, id, args, resp, NOT_VF_REQ,