1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Huawei Technologies Co., Ltd
5 #include <rte_bus_pci.h>
7 #include "hinic_compat.h"
9 #include "hinic_pmd_hwdev.h"
10 #include "hinic_pmd_hwif.h"
12 #define HINIC_CFG_REGS_BAR 0
13 #define HINIC_INTR_MSI_BAR 2
14 #define HINIC_DB_MEM_BAR 4
16 #define PAGE_SIZE_4K 0x1000
17 #define PAGE_SIZE_64K 0x10000
19 #define HINIC_MSIX_CNT_RESEND_TIMER_SHIFT 29
20 #define HINIC_MSIX_CNT_RESEND_TIMER_MASK 0x7U
22 #define HINIC_MSIX_CNT_SET(val, member) \
23 (((val) & HINIC_MSIX_CNT_##member##_MASK) << \
24 HINIC_MSIX_CNT_##member##_SHIFT)
27 * hwif_ready - test if the HW initialization passed
28 * @hwdev: the pointer to the private hardware device object
29 * Return: 0 - success, negative - failure
31 static int hwif_ready(struct hinic_hwdev *hwdev)
33 u32 addr, attr0, attr1;
35 addr = HINIC_CSR_FUNC_ATTR1_ADDR;
36 attr1 = hinic_hwif_read_reg(hwdev->hwif, addr);
37 if (!HINIC_AF1_GET(attr1, MGMT_INIT_STATUS))
40 addr = HINIC_CSR_FUNC_ATTR0_ADDR;
41 attr0 = hinic_hwif_read_reg(hwdev->hwif, addr);
42 if ((HINIC_AF0_GET(attr0, FUNC_TYPE) == TYPE_VF) &&
43 !HINIC_AF1_GET(attr1, PF_INIT_STATUS))
50 * set_hwif_attr - set the attributes as members in hwif
51 * @hwif: the hardware interface of a pci function device
52 * @attr0: the first attribute that was read from the hw
53 * @attr1: the second attribute that was read from the hw
54 * @attr2: the third attribute that was read from the hw
56 static void set_hwif_attr(struct hinic_hwif *hwif, u32 attr0, u32 attr1,
59 hwif->attr.func_global_idx = HINIC_AF0_GET(attr0, FUNC_GLOBAL_IDX);
60 hwif->attr.port_to_port_idx = HINIC_AF0_GET(attr0, P2P_IDX);
61 hwif->attr.pci_intf_idx = HINIC_AF0_GET(attr0, PCI_INTF_IDX);
62 hwif->attr.vf_in_pf = HINIC_AF0_GET(attr0, VF_IN_PF);
63 hwif->attr.func_type = HINIC_AF0_GET(attr0, FUNC_TYPE);
65 hwif->attr.ppf_idx = HINIC_AF1_GET(attr1, PPF_IDX);
67 hwif->attr.num_aeqs = BIT(HINIC_AF1_GET(attr1, AEQS_PER_FUNC));
68 hwif->attr.num_ceqs = BIT(HINIC_AF1_GET(attr1, CEQS_PER_FUNC));
69 hwif->attr.num_irqs = BIT(HINIC_AF1_GET(attr1, IRQS_PER_FUNC));
70 hwif->attr.num_dma_attr = BIT(HINIC_AF1_GET(attr1, DMA_ATTR_PER_FUNC));
72 hwif->attr.global_vf_id_of_pf = HINIC_AF2_GET(attr2,
77 * get_hwif_attr - read and set the attributes as members in hwif
78 * @hwif: the hardware interface of a pci function device
80 static void get_hwif_attr(struct hinic_hwif *hwif)
82 u32 addr, attr0, attr1, attr2;
84 addr = HINIC_CSR_FUNC_ATTR0_ADDR;
85 attr0 = hinic_hwif_read_reg(hwif, addr);
87 addr = HINIC_CSR_FUNC_ATTR1_ADDR;
88 attr1 = hinic_hwif_read_reg(hwif, addr);
90 addr = HINIC_CSR_FUNC_ATTR2_ADDR;
91 attr2 = hinic_hwif_read_reg(hwif, addr);
93 set_hwif_attr(hwif, attr0, attr1, attr2);
96 void hinic_set_pf_status(struct hinic_hwif *hwif, enum hinic_pf_status status)
98 u32 attr5 = HINIC_AF5_SET(status, PF_STATUS);
99 u32 addr = HINIC_CSR_FUNC_ATTR5_ADDR;
101 if (hwif->attr.func_type == TYPE_VF) {
102 PMD_DRV_LOG(INFO, "VF doesn't support to set attr5");
106 hinic_hwif_write_reg(hwif, addr, attr5);
109 enum hinic_pf_status hinic_get_pf_status(struct hinic_hwif *hwif)
111 u32 attr5 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR5_ADDR);
113 return HINIC_AF5_GET(attr5, PF_STATUS);
116 static enum hinic_doorbell_ctrl
117 hinic_get_doorbell_ctrl_status(struct hinic_hwif *hwif)
119 u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR);
121 return HINIC_AF4_GET(attr4, DOORBELL_CTRL);
124 static enum hinic_outbound_ctrl
125 hinic_get_outbound_ctrl_status(struct hinic_hwif *hwif)
127 u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR);
129 return HINIC_AF4_GET(attr4, OUTBOUND_CTRL);
132 void hinic_enable_doorbell(struct hinic_hwif *hwif)
136 addr = HINIC_CSR_FUNC_ATTR4_ADDR;
137 attr4 = hinic_hwif_read_reg(hwif, addr);
139 attr4 = HINIC_AF4_CLEAR(attr4, DOORBELL_CTRL);
140 attr4 |= HINIC_AF4_SET(ENABLE_DOORBELL, DOORBELL_CTRL);
142 hinic_hwif_write_reg(hwif, addr, attr4);
145 void hinic_disable_doorbell(struct hinic_hwif *hwif)
149 addr = HINIC_CSR_FUNC_ATTR4_ADDR;
150 attr4 = hinic_hwif_read_reg(hwif, addr);
152 attr4 = HINIC_AF4_CLEAR(attr4, DOORBELL_CTRL);
153 attr4 |= HINIC_AF4_SET(DISABLE_DOORBELL, DOORBELL_CTRL);
155 hinic_hwif_write_reg(hwif, addr, attr4);
159 * set_ppf - try to set hwif as ppf and set the type of hwif in this case
160 * @hwif: the hardware interface of a pci function device
162 static void set_ppf(struct hinic_hwif *hwif)
164 struct hinic_func_attr *attr = &hwif->attr;
165 u32 addr, val, ppf_election;
167 /* Read Modify Write */
168 addr = HINIC_CSR_PPF_ELECTION_ADDR;
170 val = hinic_hwif_read_reg(hwif, addr);
171 val = HINIC_PPF_ELECTION_CLEAR(val, IDX);
173 ppf_election = HINIC_PPF_ELECTION_SET(attr->func_global_idx, IDX);
176 hinic_hwif_write_reg(hwif, addr, val);
179 val = hinic_hwif_read_reg(hwif, addr);
181 attr->ppf_idx = HINIC_PPF_ELECTION_GET(val, IDX);
182 if (attr->ppf_idx == attr->func_global_idx)
183 attr->func_type = TYPE_PPF;
186 static void init_db_area_idx(struct hinic_hwif *hwif)
188 struct hinic_free_db_area *free_db_area = &hwif->free_db_area;
189 u32 db_max_areas = hwif->db_max_areas;
192 for (i = 0; i < db_max_areas; i++)
193 free_db_area->db_idx[i] = i;
195 free_db_area->alloc_pos = 0;
196 free_db_area->return_pos = 0;
198 free_db_area->num_free = db_max_areas;
200 spin_lock_init(&free_db_area->idx_lock);
203 static int get_db_idx(struct hinic_hwif *hwif, u32 *idx)
205 struct hinic_free_db_area *free_db_area = &hwif->free_db_area;
209 spin_lock(&free_db_area->idx_lock);
211 if (free_db_area->num_free == 0) {
212 spin_unlock(&free_db_area->idx_lock);
216 free_db_area->num_free--;
218 pos = free_db_area->alloc_pos++;
219 pos &= (hwif->db_max_areas - 1);
221 pg_idx = free_db_area->db_idx[pos];
223 free_db_area->db_idx[pos] = 0xFFFFFFFF;
225 spin_unlock(&free_db_area->idx_lock);
232 static void free_db_idx(struct hinic_hwif *hwif, u32 idx)
234 struct hinic_free_db_area *free_db_area = &hwif->free_db_area;
237 spin_lock(&free_db_area->idx_lock);
239 pos = free_db_area->return_pos++;
240 pos &= (hwif->db_max_areas - 1);
242 free_db_area->db_idx[pos] = idx;
244 free_db_area->num_free++;
246 spin_unlock(&free_db_area->idx_lock);
249 void hinic_free_db_addr(void *hwdev, void __iomem *db_base)
251 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
252 u32 idx = DB_IDX(db_base, hwif->db_base);
254 free_db_idx(hwif, idx);
257 int hinic_alloc_db_addr(void *hwdev, void __iomem **db_base)
259 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
263 err = get_db_idx(hwif, &idx);
267 *db_base = hwif->db_base + idx * HINIC_DB_PAGE_SIZE;
272 void hinic_set_msix_state(void *hwdev, u16 msix_idx, enum hinic_msix_state flag)
274 struct hinic_hwdev *hw = hwdev;
275 struct hinic_hwif *hwif = hw->hwif;
276 u32 offset = msix_idx * HINIC_PCI_MSIX_ENTRY_SIZE
277 + HINIC_PCI_MSIX_ENTRY_VECTOR_CTRL;
280 /* vfio-pci does not mmap msi-x vector table to user space,
281 * we can not access the space when kernel driver is vfio-pci
283 if (hw->pcidev_hdl->kdrv == RTE_KDRV_VFIO)
286 mask_bits = readl(hwif->intr_regs_base + offset);
287 mask_bits &= ~HINIC_PCI_MSIX_ENTRY_CTRL_MASKBIT;
289 mask_bits |= HINIC_PCI_MSIX_ENTRY_CTRL_MASKBIT;
291 writel(mask_bits, hwif->intr_regs_base + offset);
294 static void disable_all_msix(struct hinic_hwdev *hwdev)
296 u16 num_irqs = hwdev->hwif->attr.num_irqs;
299 for (i = 0; i < num_irqs; i++)
300 hinic_set_msix_state(hwdev, i, HINIC_MSIX_DISABLE);
304 * Wait for up enable or disable doorbell flush finished.
305 * @hwif: the hardware interface of a pci function device.
306 * @states: Disable or Enable.
308 int wait_until_doorbell_flush_states(struct hinic_hwif *hwif,
309 enum hinic_doorbell_ctrl states)
312 enum hinic_doorbell_ctrl db_ctrl;
315 msecs_to_jiffies(HINIC_WAIT_DOORBELL_AND_OUTBOUND_TIMEOUT);
317 db_ctrl = hinic_get_doorbell_ctrl_status(hwif);
318 if (db_ctrl == states)
322 } while (time_before(jiffies, end));
327 static int wait_until_doorbell_and_outbound_enabled(struct hinic_hwif *hwif)
330 enum hinic_doorbell_ctrl db_ctrl;
331 enum hinic_outbound_ctrl outbound_ctrl;
334 msecs_to_jiffies(HINIC_WAIT_DOORBELL_AND_OUTBOUND_TIMEOUT);
336 db_ctrl = hinic_get_doorbell_ctrl_status(hwif);
337 outbound_ctrl = hinic_get_outbound_ctrl_status(hwif);
339 if (outbound_ctrl == ENABLE_OUTBOUND &&
340 db_ctrl == ENABLE_DOORBELL)
344 } while (time_before(jiffies, end));
349 u16 hinic_global_func_id(void *hwdev)
351 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
353 return hwif->attr.func_global_idx;
356 enum func_type hinic_func_type(void *hwdev)
358 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
360 return hwif->attr.func_type;
363 u8 hinic_ppf_idx(void *hwdev)
365 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
367 return hwif->attr.ppf_idx;
371 * hinic_dma_attr_entry_num - get number id of DMA attribute table.
372 * @hwdev: the pointer to the private hardware device object.
373 * Return: The number id of DMA attribute table.
375 u8 hinic_dma_attr_entry_num(void *hwdev)
377 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
378 return hwif->attr.num_dma_attr;
382 * hinic_init_hwif - initialize the hw interface
383 * @hwdev: the pointer to the private hardware device object
384 * @cfg_reg_base: base physical address of configuration registers
385 * @intr_reg_base: base physical address of msi-x vector table
386 * @db_base_phy: base physical address of doorbell registers
387 * @db_base: base virtual address of doorbell registers
388 * @dwqe_mapping: direct wqe io mapping address
389 * Return: 0 - success, negative - failure
391 static int hinic_init_hwif(struct hinic_hwdev *hwdev, void *cfg_reg_base,
392 void *intr_reg_base, u64 db_base_phy,
393 void *db_base, __rte_unused void *dwqe_mapping)
395 struct hinic_hwif *hwif;
396 struct rte_pci_device *pci_dev;
400 pci_dev = (struct rte_pci_device *)(hwdev->pcidev_hdl);
401 db_bar_len = pci_dev->mem_resource[HINIC_DB_MEM_BAR].len;
405 hwif->cfg_regs_base = (u8 __iomem *)cfg_reg_base;
406 hwif->intr_regs_base = (u8 __iomem *)intr_reg_base;
408 hwif->db_base_phy = db_base_phy;
409 hwif->db_base = (u8 __iomem *)db_base;
410 hwif->db_max_areas = db_bar_len / HINIC_DB_PAGE_SIZE;
411 if (hwif->db_max_areas > HINIC_DB_MAX_AREAS)
412 hwif->db_max_areas = HINIC_DB_MAX_AREAS;
414 init_db_area_idx(hwif);
418 err = hwif_ready(hwdev);
420 PMD_DRV_LOG(ERR, "Hwif is not ready");
424 err = wait_until_doorbell_and_outbound_enabled(hwif);
426 PMD_DRV_LOG(ERR, "Hw doorbell/outbound is disabled");
430 if (!HINIC_IS_VF(hwdev))
433 /* disable mgmt cpu report any event */
434 hinic_set_pf_status(hwdev->hwif, HINIC_PF_STATUS_INIT);
439 spin_lock_deinit(&hwif->free_db_area.idx_lock);
444 #define HINIC_HWIF_ATTR_REG_PRINT_NUM (6)
445 #define HINIC_HWIF_APICMD_REG_PRINT_NUM (2)
446 #define HINIC_HWIF_EQ_REG_PRINT_NUM (2)
448 static void hinic_parse_hwif_attr(struct hinic_hwdev *hwdev)
450 struct hinic_hwif *hwif = hwdev->hwif;
452 PMD_DRV_LOG(INFO, "Device %s hwif attribute:", hwdev->pcidev_hdl->name);
453 PMD_DRV_LOG(INFO, "func_idx: %u, p2p_idx: %u, pciintf_idx: %u, "
454 "vf_in_pf: %u, ppf_idx: %u, global_vf_id: %u, func_type: %u",
455 hwif->attr.func_global_idx,
456 hwif->attr.port_to_port_idx, hwif->attr.pci_intf_idx,
457 hwif->attr.vf_in_pf, hwif->attr.ppf_idx,
458 hwif->attr.global_vf_id_of_pf, hwif->attr.func_type);
459 PMD_DRV_LOG(INFO, "num_aeqs:%u, num_ceqs:%u, num_irqs:%u, dma_attr:%u",
460 hwif->attr.num_aeqs, hwif->attr.num_ceqs,
461 hwif->attr.num_irqs, hwif->attr.num_dma_attr);
464 static void hinic_get_mmio(struct hinic_hwdev *hwdev, void **cfg_regs_base,
465 void **intr_base, void **db_base)
467 struct rte_pci_device *pci_dev = hwdev->pcidev_hdl;
470 uint64_t bar0_phy_addr;
471 uint64_t pagesize = sysconf(_SC_PAGESIZE);
473 *cfg_regs_base = pci_dev->mem_resource[HINIC_CFG_REGS_BAR].addr;
474 *intr_base = pci_dev->mem_resource[HINIC_INTR_MSI_BAR].addr;
475 *db_base = pci_dev->mem_resource[HINIC_DB_MEM_BAR].addr;
477 bar0_size = pci_dev->mem_resource[HINIC_CFG_REGS_BAR].len;
478 bar2_size = pci_dev->mem_resource[HINIC_INTR_MSI_BAR].len;
480 if (pagesize == PAGE_SIZE_64K && (bar0_size % pagesize != 0)) {
482 pci_dev->mem_resource[HINIC_CFG_REGS_BAR].phys_addr;
483 if (bar0_phy_addr % pagesize != 0 &&
484 (bar0_size + bar2_size <= pagesize) &&
485 bar2_size >= bar0_size) {
486 *cfg_regs_base = (void *)((uint8_t *)(*intr_base)
492 void hinic_hwif_res_free(struct hinic_hwdev *hwdev)
494 rte_free(hwdev->hwif);
498 int hinic_hwif_res_init(struct hinic_hwdev *hwdev)
500 int err = HINIC_ERROR;
501 void *cfg_regs_base, *db_base, *intr_base = NULL;
503 /* hinic related init */
504 hwdev->hwif = rte_zmalloc("hinic_hwif", sizeof(*hwdev->hwif),
505 RTE_CACHE_LINE_SIZE);
507 PMD_DRV_LOG(ERR, "Allocate hwif failed, dev_name: %s",
508 hwdev->pcidev_hdl->name);
512 hinic_get_mmio(hwdev, &cfg_regs_base, &intr_base, &db_base);
514 err = hinic_init_hwif(hwdev, cfg_regs_base,
515 intr_base, 0, db_base, NULL);
517 PMD_DRV_LOG(ERR, "Initialize hwif failed, dev_name: %s",
518 hwdev->pcidev_hdl->name);
522 /* disable msix interrupt in hw device */
523 disable_all_msix(hwdev);
525 /* print hwif attributes */
526 hinic_parse_hwif_attr(hwdev);
531 rte_free(hwdev->hwif);
538 * hinic_misx_intr_clear_resend_bit - clear interrupt resend configuration
539 * @hwdev: the hardware interface of a nic device
540 * @msix_idx: Index of msix interrupt
541 * @clear_resend_en: enable flag of clear resend configuration
543 void hinic_misx_intr_clear_resend_bit(void *hwdev, u16 msix_idx,
546 struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
547 u32 msix_ctrl = 0, addr;
549 msix_ctrl = HINIC_MSIX_CNT_SET(clear_resend_en, RESEND_TIMER);
551 addr = HINIC_CSR_MSIX_CNT_ADDR(msix_idx);
553 hinic_hwif_write_reg(hwif, addr, msix_ctrl);