1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2020 Intel Corporation
13 #include <rte_malloc.h>
14 #include <rte_errno.h>
16 #include "base/dlb2_resource.h"
17 #include "base/dlb2_osdep.h"
18 #include "base/dlb2_regs.h"
19 #include "dlb2_main.h"
20 #include "../dlb2_user.h"
21 #include "../dlb2_priv.h"
22 #include "../dlb2_iface.h"
23 #include "../dlb2_inline_fns.h"
25 #define PF_ID_ZERO 0 /* PF ONLY! */
26 #define NO_OWNER_VF 0 /* PF ONLY! */
27 #define NOT_VF_REQ false /* PF ONLY! */
29 #define DLB2_PCI_CFG_SPACE_SIZE 256
30 #define DLB2_PCI_CAP_POINTER 0x34
31 #define DLB2_PCI_CAP_NEXT(hdr) (((hdr) >> 8) & 0xFC)
32 #define DLB2_PCI_CAP_ID(hdr) ((hdr) & 0xFF)
33 #define DLB2_PCI_EXT_CAP_NEXT(hdr) (((hdr) >> 20) & 0xFFC)
34 #define DLB2_PCI_EXT_CAP_ID(hdr) ((hdr) & 0xFFFF)
35 #define DLB2_PCI_EXT_CAP_ID_ERR 1
36 #define DLB2_PCI_ERR_UNCOR_MASK 8
37 #define DLB2_PCI_ERR_UNC_UNSUP 0x00100000
39 #define DLB2_PCI_EXP_DEVCTL 8
40 #define DLB2_PCI_LNKCTL 16
41 #define DLB2_PCI_SLTCTL 24
42 #define DLB2_PCI_RTCTL 28
43 #define DLB2_PCI_EXP_DEVCTL2 40
44 #define DLB2_PCI_LNKCTL2 48
45 #define DLB2_PCI_SLTCTL2 56
46 #define DLB2_PCI_CMD 4
47 #define DLB2_PCI_X_CMD 2
48 #define DLB2_PCI_EXP_DEVSTA 10
49 #define DLB2_PCI_EXP_DEVSTA_TRPND 0x20
50 #define DLB2_PCI_EXP_DEVCTL_BCR_FLR 0x8000
52 #define DLB2_PCI_CAP_ID_EXP 0x10
53 #define DLB2_PCI_CAP_ID_MSIX 0x11
54 #define DLB2_PCI_EXT_CAP_ID_PAS 0x1B
55 #define DLB2_PCI_EXT_CAP_ID_PRI 0x13
56 #define DLB2_PCI_EXT_CAP_ID_ACS 0xD
58 #define DLB2_PCI_PRI_CTRL_ENABLE 0x1
59 #define DLB2_PCI_PRI_ALLOC_REQ 0xC
60 #define DLB2_PCI_PRI_CTRL 0x4
61 #define DLB2_PCI_MSIX_FLAGS 0x2
62 #define DLB2_PCI_MSIX_FLAGS_ENABLE 0x8000
63 #define DLB2_PCI_MSIX_FLAGS_MASKALL 0x4000
64 #define DLB2_PCI_ERR_ROOT_STATUS 0x30
65 #define DLB2_PCI_ERR_COR_STATUS 0x10
66 #define DLB2_PCI_ERR_UNCOR_STATUS 0x4
67 #define DLB2_PCI_COMMAND_INTX_DISABLE 0x400
68 #define DLB2_PCI_ACS_CAP 0x4
69 #define DLB2_PCI_ACS_CTRL 0x6
70 #define DLB2_PCI_ACS_SV 0x1
71 #define DLB2_PCI_ACS_RR 0x4
72 #define DLB2_PCI_ACS_CR 0x8
73 #define DLB2_PCI_ACS_UF 0x10
74 #define DLB2_PCI_ACS_EC 0x20
77 dlb2_pci_find_ext_capability(struct rte_pci_device *pdev, uint32_t id)
83 pos = DLB2_PCI_CFG_SPACE_SIZE;
87 if (rte_pci_read_config(pdev, &hdr, sz, pos) != (int)sz)
90 if (DLB2_PCI_EXT_CAP_ID(hdr) == id)
93 pos = DLB2_PCI_EXT_CAP_NEXT(hdr);
99 static int dlb2_pci_find_capability(struct rte_pci_device *pdev, uint32_t id)
105 ret = rte_pci_read_config(pdev, &pos, 1, DLB2_PCI_CAP_POINTER);
112 ret = rte_pci_read_config(pdev, &hdr, 2, pos);
116 if (DLB2_PCI_CAP_ID(hdr) == id)
119 if (DLB2_PCI_CAP_ID(hdr) == 0xFF)
122 pos = DLB2_PCI_CAP_NEXT(hdr);
129 dlb2_pf_init_driver_state(struct dlb2_dev *dlb2_dev)
131 rte_spinlock_init(&dlb2_dev->resource_mutex);
136 static void dlb2_pf_enable_pm(struct dlb2_dev *dlb2_dev)
138 dlb2_clr_pmcsr_disable(&dlb2_dev->hw);
141 #define DLB2_READY_RETRY_LIMIT 1000
142 static int dlb2_pf_wait_for_device_ready(struct dlb2_dev *dlb2_dev)
146 /* Allow at least 1s for the device to become active after power-on */
147 for (retries = 0; retries < DLB2_READY_RETRY_LIMIT; retries++) {
148 union dlb2_cfg_mstr_cfg_diagnostic_idle_status idle;
149 union dlb2_cfg_mstr_cfg_pm_status pm_st;
152 addr = DLB2_CFG_MSTR_CFG_PM_STATUS;
153 pm_st.val = DLB2_CSR_RD(&dlb2_dev->hw, addr);
154 addr = DLB2_CFG_MSTR_CFG_DIAGNOSTIC_IDLE_STATUS;
155 idle.val = DLB2_CSR_RD(&dlb2_dev->hw, addr);
156 if (pm_st.field.pmsm == 1 && idle.field.dlb_func_idle == 1)
162 if (retries == DLB2_READY_RETRY_LIMIT) {
163 DLB2_LOG_ERR("[%s()] wait for device ready timed out\n",
172 dlb2_probe(struct rte_pci_device *pdev)
174 struct dlb2_dev *dlb2_dev;
177 DLB2_INFO(dlb2_dev, "probe\n");
179 dlb2_dev = rte_malloc("DLB2_PF", sizeof(struct dlb2_dev),
180 RTE_CACHE_LINE_SIZE);
182 if (dlb2_dev == NULL) {
184 goto dlb2_dev_malloc_fail;
187 /* PCI Bus driver has already mapped bar space into process.
188 * Save off our IO register and FUNC addresses.
192 if (pdev->mem_resource[0].addr == NULL) {
193 DLB2_ERR(dlb2_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
195 goto pci_mmap_bad_addr;
197 dlb2_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
198 dlb2_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
200 DLB2_INFO(dlb2_dev, "DLB2 FUNC VA=%p, PA=%p, len=%p\n",
201 (void *)dlb2_dev->hw.func_kva,
202 (void *)dlb2_dev->hw.func_phys_addr,
203 (void *)(pdev->mem_resource[0].len));
206 if (pdev->mem_resource[2].addr == NULL) {
207 DLB2_ERR(dlb2_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
209 goto pci_mmap_bad_addr;
211 dlb2_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
212 dlb2_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
214 DLB2_INFO(dlb2_dev, "DLB2 CSR VA=%p, PA=%p, len=%p\n",
215 (void *)dlb2_dev->hw.csr_kva,
216 (void *)dlb2_dev->hw.csr_phys_addr,
217 (void *)(pdev->mem_resource[2].len));
219 dlb2_dev->pdev = pdev;
221 /* PM enable must be done before any other MMIO accesses, and this
222 * setting is persistent across device reset.
224 dlb2_pf_enable_pm(dlb2_dev);
226 ret = dlb2_pf_wait_for_device_ready(dlb2_dev);
228 goto wait_for_device_ready_fail;
230 ret = dlb2_pf_reset(dlb2_dev);
232 goto dlb2_reset_fail;
234 ret = dlb2_pf_init_driver_state(dlb2_dev);
236 goto init_driver_state_fail;
238 ret = dlb2_resource_init(&dlb2_dev->hw);
240 goto resource_init_fail;
245 dlb2_resource_free(&dlb2_dev->hw);
246 init_driver_state_fail:
249 wait_for_device_ready_fail:
251 dlb2_dev_malloc_fail:
257 dlb2_pf_reset(struct dlb2_dev *dlb2_dev)
265 uint16_t dev_ctl_word;
266 uint16_t dev_ctl2_word;
271 uint16_t rt_ctl_word;
272 uint32_t pri_reqs_dword;
273 uint16_t pri_ctrl_word;
282 uint16_t devsta_busy_word;
283 uint16_t devctl_word;
285 struct rte_pci_device *pdev = dlb2_dev->pdev;
287 /* Save PCI config state */
289 for (i = 0; i < 16; i++) {
290 if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
294 pcie_cap_offset = dlb2_pci_find_capability(pdev, DLB2_PCI_CAP_ID_EXP);
296 if (pcie_cap_offset < 0) {
297 DLB2_LOG_ERR("[%s()] failed to find the pcie capability\n",
299 return pcie_cap_offset;
302 off = pcie_cap_offset + DLB2_PCI_EXP_DEVCTL;
303 if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
306 off = pcie_cap_offset + DLB2_PCI_LNKCTL;
307 if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
310 off = pcie_cap_offset + DLB2_PCI_SLTCTL;
311 if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
314 off = pcie_cap_offset + DLB2_PCI_RTCTL;
315 if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
318 off = pcie_cap_offset + DLB2_PCI_EXP_DEVCTL2;
319 if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
322 off = pcie_cap_offset + DLB2_PCI_LNKCTL2;
323 if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
326 off = pcie_cap_offset + DLB2_PCI_SLTCTL2;
327 if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
330 off = DLB2_PCI_EXT_CAP_ID_PRI;
331 pri_cap_offset = dlb2_pci_find_ext_capability(pdev, off);
333 if (pri_cap_offset >= 0) {
334 off = pri_cap_offset + DLB2_PCI_PRI_ALLOC_REQ;
335 if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
339 /* clear the PCI command register before issuing the FLR */
343 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
344 DLB2_LOG_ERR("[%s()] failed to write the pci command\n",
350 for (wait_count = 0; wait_count < 4; wait_count++) {
353 off = pcie_cap_offset + DLB2_PCI_EXP_DEVSTA;
354 ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
356 DLB2_LOG_ERR("[%s()] failed to read the pci device status\n",
361 if (!(devsta_busy_word & DLB2_PCI_EXP_DEVSTA_TRPND))
364 sleep_time = (1 << (wait_count)) * 100;
365 rte_delay_ms(sleep_time);
368 if (wait_count == 4) {
369 DLB2_LOG_ERR("[%s()] wait for pci pending transactions timed out\n",
374 off = pcie_cap_offset + DLB2_PCI_EXP_DEVCTL;
375 ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
377 DLB2_LOG_ERR("[%s()] failed to read the pcie device control\n",
382 devctl_word |= DLB2_PCI_EXP_DEVCTL_BCR_FLR;
384 ret = rte_pci_write_config(pdev, &devctl_word, 2, off);
386 DLB2_LOG_ERR("[%s()] failed to write the pcie device control\n",
393 /* Restore PCI config state */
395 if (pcie_cap_offset >= 0) {
396 off = pcie_cap_offset + DLB2_PCI_EXP_DEVCTL;
397 ret = rte_pci_write_config(pdev, &dev_ctl_word, 2, off);
399 DLB2_LOG_ERR("[%s()] failed to write the pcie device control at offset %d\n",
404 off = pcie_cap_offset + DLB2_PCI_LNKCTL;
405 ret = rte_pci_write_config(pdev, &lnk_word, 2, off);
407 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
412 off = pcie_cap_offset + DLB2_PCI_SLTCTL;
413 ret = rte_pci_write_config(pdev, &slt_word, 2, off);
415 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
420 off = pcie_cap_offset + DLB2_PCI_RTCTL;
421 ret = rte_pci_write_config(pdev, &rt_ctl_word, 2, off);
423 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
428 off = pcie_cap_offset + DLB2_PCI_EXP_DEVCTL2;
429 ret = rte_pci_write_config(pdev, &dev_ctl2_word, 2, off);
431 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
436 off = pcie_cap_offset + DLB2_PCI_LNKCTL2;
437 ret = rte_pci_write_config(pdev, &lnk_word2, 2, off);
439 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
444 off = pcie_cap_offset + DLB2_PCI_SLTCTL2;
445 ret = rte_pci_write_config(pdev, &slt_word2, 2, off);
447 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
453 if (pri_cap_offset >= 0) {
454 pri_ctrl_word = DLB2_PCI_PRI_CTRL_ENABLE;
456 off = pri_cap_offset + DLB2_PCI_PRI_ALLOC_REQ;
457 ret = rte_pci_write_config(pdev, &pri_reqs_dword, 4, off);
459 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
464 off = pri_cap_offset + DLB2_PCI_PRI_CTRL;
465 ret = rte_pci_write_config(pdev, &pri_ctrl_word, 2, off);
467 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
473 off = DLB2_PCI_EXT_CAP_ID_ERR;
474 err_cap_offset = dlb2_pci_find_ext_capability(pdev, off);
476 if (err_cap_offset >= 0) {
479 off = err_cap_offset + DLB2_PCI_ERR_ROOT_STATUS;
480 if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
483 ret = rte_pci_write_config(pdev, &tmp, 4, off);
485 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
490 off = err_cap_offset + DLB2_PCI_ERR_COR_STATUS;
491 if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
494 ret = rte_pci_write_config(pdev, &tmp, 4, off);
496 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
501 off = err_cap_offset + DLB2_PCI_ERR_UNCOR_STATUS;
502 if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
505 ret = rte_pci_write_config(pdev, &tmp, 4, off);
507 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
513 for (i = 16; i > 0; i--) {
515 ret = rte_pci_write_config(pdev, &dword[i - 1], 4, off);
517 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
524 if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
525 cmd &= ~DLB2_PCI_COMMAND_INTX_DISABLE;
526 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
527 DLB2_LOG_ERR("[%s()] failed to write the pci command\n",
533 msix_cap_offset = dlb2_pci_find_capability(pdev,
534 DLB2_PCI_CAP_ID_MSIX);
535 if (msix_cap_offset >= 0) {
536 off = msix_cap_offset + DLB2_PCI_MSIX_FLAGS;
537 if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
538 cmd |= DLB2_PCI_MSIX_FLAGS_ENABLE;
539 cmd |= DLB2_PCI_MSIX_FLAGS_MASKALL;
540 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
541 DLB2_LOG_ERR("[%s()] failed to write msix flags\n",
547 off = msix_cap_offset + DLB2_PCI_MSIX_FLAGS;
548 if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
549 cmd &= ~DLB2_PCI_MSIX_FLAGS_MASKALL;
550 if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
551 DLB2_LOG_ERR("[%s()] failed to write msix flags\n",
558 off = DLB2_PCI_EXT_CAP_ID_ACS;
559 acs_cap_offset = dlb2_pci_find_ext_capability(pdev, off);
561 if (acs_cap_offset >= 0) {
562 uint16_t acs_cap, acs_ctrl, acs_mask;
563 off = acs_cap_offset + DLB2_PCI_ACS_CAP;
564 if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
567 off = acs_cap_offset + DLB2_PCI_ACS_CTRL;
568 if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
571 acs_mask = DLB2_PCI_ACS_SV | DLB2_PCI_ACS_RR;
572 acs_mask |= (DLB2_PCI_ACS_CR | DLB2_PCI_ACS_UF);
573 acs_ctrl |= (acs_cap & acs_mask);
575 ret = rte_pci_write_config(pdev, &acs_ctrl, 2, off);
577 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
582 off = acs_cap_offset + DLB2_PCI_ACS_CTRL;
583 if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
586 acs_mask = DLB2_PCI_ACS_RR | DLB2_PCI_ACS_CR;
587 acs_mask |= DLB2_PCI_ACS_EC;
588 acs_ctrl &= ~acs_mask;
590 off = acs_cap_offset + DLB2_PCI_ACS_CTRL;
591 ret = rte_pci_write_config(pdev, &acs_ctrl, 2, off);
593 DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d\n",
603 dlb2_pf_create_sched_domain(struct dlb2_hw *hw,
604 struct dlb2_create_sched_domain_args *args,
605 struct dlb2_cmd_response *resp)
607 return dlb2_hw_create_sched_domain(hw, args, resp, NOT_VF_REQ,
612 dlb2_pf_reset_domain(struct dlb2_hw *hw, u32 id)
614 return dlb2_reset_domain(hw, id, NOT_VF_REQ, PF_ID_ZERO);
618 dlb2_pf_create_ldb_queue(struct dlb2_hw *hw,
620 struct dlb2_create_ldb_queue_args *args,
621 struct dlb2_cmd_response *resp)
623 return dlb2_hw_create_ldb_queue(hw, id, args, resp, NOT_VF_REQ,
628 dlb2_pf_create_ldb_port(struct dlb2_hw *hw,
630 struct dlb2_create_ldb_port_args *args,
631 uintptr_t cq_dma_base,
632 struct dlb2_cmd_response *resp)
634 return dlb2_hw_create_ldb_port(hw, id, args,
642 dlb2_pf_create_dir_port(struct dlb2_hw *hw,
644 struct dlb2_create_dir_port_args *args,
645 uintptr_t cq_dma_base,
646 struct dlb2_cmd_response *resp)
648 return dlb2_hw_create_dir_port(hw, id, args,