1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 HiSilicon Limited
8 #include <rte_bus_pci.h>
9 #include <rte_cycles.h>
13 #include <rte_malloc.h>
14 #include <rte_memzone.h>
16 #include <rte_dmadev_pmd.h>
18 #include "hisi_dmadev.h"
20 RTE_LOG_REGISTER_DEFAULT(hisi_dma_logtype, INFO);
21 #define HISI_DMA_LOG(level, fmt, args...) \
22 rte_log(RTE_LOG_ ## level, hisi_dma_logtype, \
23 "%s(): " fmt "\n", __func__, ##args)
24 #define HISI_DMA_LOG_RAW(hw, level, fmt, args...) \
25 rte_log(RTE_LOG_ ## level, hisi_dma_logtype, \
26 "%s %s(): " fmt "\n", (hw)->data->dev_name, \
28 #define HISI_DMA_DEBUG(hw, fmt, args...) \
29 HISI_DMA_LOG_RAW(hw, DEBUG, fmt, ## args)
30 #define HISI_DMA_INFO(hw, fmt, args...) \
31 HISI_DMA_LOG_RAW(hw, INFO, fmt, ## args)
32 #define HISI_DMA_WARN(hw, fmt, args...) \
33 HISI_DMA_LOG_RAW(hw, WARNING, fmt, ## args)
34 #define HISI_DMA_ERR(hw, fmt, args...) \
35 HISI_DMA_LOG_RAW(hw, ERR, fmt, ## args)
38 hisi_dma_queue_base(struct hisi_dma_dev *hw)
40 if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
41 return HISI_DMA_HIP08_QUEUE_BASE;
46 static volatile void *
47 hisi_dma_queue_regaddr(struct hisi_dma_dev *hw, uint32_t qoff)
49 uint32_t off = hisi_dma_queue_base(hw) +
50 hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
51 return (volatile void *)((char *)hw->io_base + off);
55 hisi_dma_write_reg(void *base, uint32_t off, uint32_t val)
57 rte_write32(rte_cpu_to_le_32(val),
58 (volatile void *)((char *)base + off));
62 hisi_dma_write_dev(struct hisi_dma_dev *hw, uint32_t off, uint32_t val)
64 hisi_dma_write_reg(hw->io_base, off, val);
68 hisi_dma_write_queue(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t val)
70 uint32_t off = hisi_dma_queue_base(hw) +
71 hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
72 hisi_dma_write_dev(hw, off, val);
76 hisi_dma_read_reg(void *base, uint32_t off)
78 uint32_t val = rte_read32((volatile void *)((char *)base + off));
79 return rte_le_to_cpu_32(val);
83 hisi_dma_read_dev(struct hisi_dma_dev *hw, uint32_t off)
85 return hisi_dma_read_reg(hw->io_base, off);
89 hisi_dma_read_queue(struct hisi_dma_dev *hw, uint32_t qoff)
91 uint32_t off = hisi_dma_queue_base(hw) +
92 hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
93 return hisi_dma_read_dev(hw, off);
97 hisi_dma_update_bit(struct hisi_dma_dev *hw, uint32_t off, uint32_t pos,
100 uint32_t tmp = hisi_dma_read_dev(hw, off);
101 uint32_t mask = 1u << pos;
102 tmp = set ? tmp | mask : tmp & ~mask;
103 hisi_dma_write_dev(hw, off, tmp);
107 hisi_dma_update_queue_bit(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t pos,
110 uint32_t tmp = hisi_dma_read_queue(hw, qoff);
111 uint32_t mask = 1u << pos;
112 tmp = set ? tmp | mask : tmp & ~mask;
113 hisi_dma_write_queue(hw, qoff, tmp);
117 hisi_dma_update_queue_mbit(struct hisi_dma_dev *hw, uint32_t qoff,
118 uint32_t mask, bool set)
120 uint32_t tmp = hisi_dma_read_queue(hw, qoff);
121 tmp = set ? tmp | mask : tmp & ~mask;
122 hisi_dma_write_queue(hw, qoff, tmp);
125 #define hisi_dma_poll_hw_state(hw, val, cond, sleep_us, timeout_us) ({ \
126 uint32_t timeout = 0; \
127 while (timeout++ <= (timeout_us)) { \
128 (val) = hisi_dma_read_queue(hw, HISI_DMA_QUEUE_FSM_REG); \
131 rte_delay_us(sleep_us); \
133 (cond) ? 0 : -ETIME; \
137 hisi_dma_reset_hw(struct hisi_dma_dev *hw)
139 #define POLL_SLEEP_US 100
140 #define POLL_TIMEOUT_US 10000
145 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
146 HISI_DMA_QUEUE_CTRL0_PAUSE_B, true);
147 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
148 HISI_DMA_QUEUE_CTRL0_EN_B, false);
150 ret = hisi_dma_poll_hw_state(hw, tmp,
151 FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) != HISI_DMA_STATE_RUN,
152 POLL_SLEEP_US, POLL_TIMEOUT_US);
154 HISI_DMA_ERR(hw, "disable dma timeout!");
158 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL1_REG,
159 HISI_DMA_QUEUE_CTRL1_RESET_B, true);
160 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_TAIL_REG, 0);
161 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_HEAD_REG, 0);
162 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
163 HISI_DMA_QUEUE_CTRL0_PAUSE_B, false);
165 ret = hisi_dma_poll_hw_state(hw, tmp,
166 FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) == HISI_DMA_STATE_IDLE,
167 POLL_SLEEP_US, POLL_TIMEOUT_US);
169 HISI_DMA_ERR(hw, "reset dma timeout!");
177 hisi_dma_init_hw(struct hisi_dma_dev *hw)
179 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_BASE_L_REG,
180 lower_32_bits(hw->sqe_iova));
181 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_BASE_H_REG,
182 upper_32_bits(hw->sqe_iova));
183 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_BASE_L_REG,
184 lower_32_bits(hw->cqe_iova));
185 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_BASE_H_REG,
186 upper_32_bits(hw->cqe_iova));
187 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_DEPTH_REG,
189 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_DEPTH_REG, hw->cq_depth - 1);
190 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_TAIL_REG, 0);
191 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_HEAD_REG, 0);
192 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM0_REG, 0);
193 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM1_REG, 0);
194 hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM2_REG, 0);
196 if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
197 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM3_REG,
199 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM4_REG,
201 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM5_REG,
203 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM6_REG,
205 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
206 HISI_DMA_HIP08_QUEUE_CTRL0_ERR_ABORT_B, false);
207 hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_STATUS_REG,
208 HISI_DMA_HIP08_QUEUE_INT_MASK_M, true);
209 hisi_dma_update_queue_mbit(hw,
210 HISI_DMA_HIP08_QUEUE_INT_MASK_REG,
211 HISI_DMA_HIP08_QUEUE_INT_MASK_M, true);
216 hisi_dma_init_gbl(void *pci_bar, uint8_t revision)
218 struct hisi_dma_dev hw;
220 memset(&hw, 0, sizeof(hw));
221 hw.io_base = pci_bar;
223 if (revision == HISI_DMA_REVISION_HIP08B)
224 hisi_dma_update_bit(&hw, HISI_DMA_HIP08_MODE_REG,
225 HISI_DMA_HIP08_MODE_SEL_B, true);
229 hisi_dma_reg_layout(uint8_t revision)
231 if (revision == HISI_DMA_REVISION_HIP08B)
232 return HISI_DMA_REG_LAYOUT_HIP08;
234 return HISI_DMA_REG_LAYOUT_INVALID;
238 hisi_dma_zero_iomem(struct hisi_dma_dev *hw)
240 memset(hw->iomz->addr, 0, hw->iomz_sz);
244 hisi_dma_alloc_iomem(struct hisi_dma_dev *hw, uint16_t ring_size,
245 const char *dev_name)
247 uint32_t sq_size = sizeof(struct hisi_dma_sqe) * ring_size;
248 uint32_t cq_size = sizeof(struct hisi_dma_cqe) *
249 (ring_size + HISI_DMA_CQ_RESERVED);
250 uint32_t status_size = sizeof(uint16_t) * ring_size;
251 char mz_name[RTE_MEMZONE_NAMESIZE];
252 const struct rte_memzone *iomz;
255 sq_size = RTE_CACHE_LINE_ROUNDUP(sq_size);
256 cq_size = RTE_CACHE_LINE_ROUNDUP(cq_size);
257 status_size = RTE_CACHE_LINE_ROUNDUP(status_size);
258 total_size = sq_size + cq_size + status_size;
260 (void)snprintf(mz_name, sizeof(mz_name), "hisi_dma:%s", dev_name);
261 iomz = rte_memzone_reserve(mz_name, total_size, hw->data->numa_node,
262 RTE_MEMZONE_IOVA_CONTIG);
264 HISI_DMA_ERR(hw, "malloc %s iomem fail!", mz_name);
269 hw->iomz_sz = total_size;
270 hw->sqe = iomz->addr;
271 hw->cqe = (void *)((char *)iomz->addr + sq_size);
272 hw->status = (void *)((char *)iomz->addr + sq_size + cq_size);
273 hw->sqe_iova = iomz->iova;
274 hw->cqe_iova = iomz->iova + sq_size;
275 hw->sq_depth_mask = ring_size - 1;
276 hw->cq_depth = ring_size + HISI_DMA_CQ_RESERVED;
277 hisi_dma_zero_iomem(hw);
283 hisi_dma_free_iomem(struct hisi_dma_dev *hw)
285 if (hw->iomz != NULL)
286 rte_memzone_free(hw->iomz);
294 hw->sq_depth_mask = 0;
299 hisi_dma_info_get(const struct rte_dma_dev *dev,
300 struct rte_dma_info *dev_info,
304 RTE_SET_USED(info_sz);
306 dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
307 RTE_DMA_CAPA_OPS_COPY;
308 dev_info->max_vchans = 1;
309 dev_info->max_desc = HISI_DMA_MAX_DESC_NUM;
310 dev_info->min_desc = HISI_DMA_MIN_DESC_NUM;
316 hisi_dma_configure(struct rte_dma_dev *dev,
317 const struct rte_dma_conf *conf,
322 RTE_SET_USED(conf_sz);
327 hisi_dma_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
328 const struct rte_dma_vchan_conf *conf,
331 struct hisi_dma_dev *hw = dev->data->dev_private;
335 RTE_SET_USED(conf_sz);
337 if (!rte_is_power_of_2(conf->nb_desc)) {
338 HISI_DMA_ERR(hw, "Number of desc must be power of 2!");
342 hisi_dma_free_iomem(hw);
343 ret = hisi_dma_alloc_iomem(hw, conf->nb_desc, dev->data->dev_name);
351 hisi_dma_start(struct rte_dma_dev *dev)
353 struct hisi_dma_dev *hw = dev->data->dev_private;
355 if (hw->iomz == NULL) {
356 HISI_DMA_ERR(hw, "Vchan was not setup, start fail!\n");
360 /* Reset the dmadev to a known state, include:
361 * 1) zero iomem, also include status fields.
362 * 2) init hardware register.
363 * 3) init index values to zero.
364 * 4) init running statistics.
366 hisi_dma_zero_iomem(hw);
367 hisi_dma_init_hw(hw);
374 hw->cqs_completed = 0;
380 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
381 HISI_DMA_QUEUE_CTRL0_EN_B, true);
387 hisi_dma_stop(struct rte_dma_dev *dev)
389 return hisi_dma_reset_hw(dev->data->dev_private);
393 hisi_dma_close(struct rte_dma_dev *dev)
395 /* The dmadev already stopped */
396 hisi_dma_free_iomem(dev->data->dev_private);
401 hisi_dma_stats_get(const struct rte_dma_dev *dev, uint16_t vchan,
402 struct rte_dma_stats *stats,
405 struct hisi_dma_dev *hw = dev->data->dev_private;
408 RTE_SET_USED(stats_sz);
409 stats->submitted = hw->submitted;
410 stats->completed = hw->completed;
411 stats->errors = hw->errors;
417 hisi_dma_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
419 struct hisi_dma_dev *hw = dev->data->dev_private;
430 hisi_dma_get_dump_range(struct hisi_dma_dev *hw, uint32_t *start, uint32_t *end)
432 if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
433 *start = HISI_DMA_HIP08_DUMP_START_REG;
434 *end = HISI_DMA_HIP08_DUMP_END_REG;
442 hisi_dma_dump_common(struct hisi_dma_dev *hw, FILE *f)
444 #define DUMP_REGNUM_PER_LINE 4
449 hisi_dma_get_dump_range(hw, &start, &end);
451 (void)fprintf(f, " common-register:\n");
454 for (i = start; i <= end; i += sizeof(uint32_t)) {
455 if (cnt % DUMP_REGNUM_PER_LINE == 0)
456 (void)fprintf(f, " [%4x]:", i);
457 (void)fprintf(f, " 0x%08x", hisi_dma_read_dev(hw, i));
459 if (cnt % DUMP_REGNUM_PER_LINE == 0)
460 (void)fprintf(f, "\n");
462 if (cnt % DUMP_REGNUM_PER_LINE)
463 (void)fprintf(f, "\n");
467 hisi_dma_dump_read_queue(struct hisi_dma_dev *hw, uint32_t qoff,
468 char *buffer, int max_sz)
470 memset(buffer, 0, max_sz);
472 /* Address-related registers are not printed for security reasons. */
473 if (qoff == HISI_DMA_QUEUE_SQ_BASE_L_REG ||
474 qoff == HISI_DMA_QUEUE_SQ_BASE_H_REG ||
475 qoff == HISI_DMA_QUEUE_CQ_BASE_L_REG ||
476 qoff == HISI_DMA_QUEUE_CQ_BASE_H_REG) {
477 (void)snprintf(buffer, max_sz, "**********");
481 (void)snprintf(buffer, max_sz, "0x%08x", hisi_dma_read_queue(hw, qoff));
485 hisi_dma_dump_queue(struct hisi_dma_dev *hw, FILE *f)
487 #define REG_FMT_LEN 32
488 char buf[REG_FMT_LEN] = { 0 };
491 (void)fprintf(f, " queue-register:\n");
492 for (i = 0; i < HISI_DMA_QUEUE_REGION_SIZE; ) {
493 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
494 (void)fprintf(f, " [%2x]: %s", i, buf);
495 i += sizeof(uint32_t);
496 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
497 (void)fprintf(f, " %s", buf);
498 i += sizeof(uint32_t);
499 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
500 (void)fprintf(f, " %s", buf);
501 i += sizeof(uint32_t);
502 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
503 (void)fprintf(f, " %s\n", buf);
504 i += sizeof(uint32_t);
509 hisi_dma_dump(const struct rte_dma_dev *dev, FILE *f)
511 struct hisi_dma_dev *hw = dev->data->dev_private;
514 " revision: 0x%x queue_id: %u ring_size: %u\n"
515 " ridx: %u cridx: %u\n"
516 " sq_head: %u sq_tail: %u cq_sq_head: %u\n"
517 " cq_head: %u cqs_completed: %u cqe_vld: %u\n"
518 " submitted: %" PRIu64 " completed: %" PRIu64 " errors %"
520 hw->revision, hw->queue_id,
521 hw->sq_depth_mask > 0 ? hw->sq_depth_mask + 1 : 0,
523 hw->sq_head, hw->sq_tail, hw->cq_sq_head,
524 hw->cq_head, hw->cqs_completed, hw->cqe_vld,
525 hw->submitted, hw->completed, hw->errors);
526 hisi_dma_dump_queue(hw, f);
527 hisi_dma_dump_common(hw, f);
533 hisi_dma_gen_pci_device_name(const struct rte_pci_device *pci_dev,
534 char *name, size_t size)
536 memset(name, 0, size);
537 (void)snprintf(name, size, "%x:%x.%x",
538 pci_dev->addr.bus, pci_dev->addr.devid,
539 pci_dev->addr.function);
543 hisi_dma_gen_dev_name(const struct rte_pci_device *pci_dev,
544 uint8_t queue_id, char *name, size_t size)
546 memset(name, 0, size);
547 (void)snprintf(name, size, "%x:%x.%x-ch%u",
548 pci_dev->addr.bus, pci_dev->addr.devid,
549 pci_dev->addr.function, queue_id);
553 * Hardware queue state machine:
555 * ----------- dmadev_create ------------------
556 * | Unknown | ---------------> | IDLE |
557 * ----------- ------------------
567 static const struct rte_dma_dev_ops hisi_dmadev_ops = {
568 .dev_info_get = hisi_dma_info_get,
569 .dev_configure = hisi_dma_configure,
570 .dev_start = hisi_dma_start,
571 .dev_stop = hisi_dma_stop,
572 .dev_close = hisi_dma_close,
573 .vchan_setup = hisi_dma_vchan_setup,
574 .stats_get = hisi_dma_stats_get,
575 .stats_reset = hisi_dma_stats_reset,
576 .dev_dump = hisi_dma_dump,
580 hisi_dma_create(struct rte_pci_device *pci_dev, uint8_t queue_id,
583 #define REG_PCI_BAR_INDEX 2
585 char name[RTE_DEV_NAME_MAX_LEN];
586 struct rte_dma_dev *dev;
587 struct hisi_dma_dev *hw;
590 hisi_dma_gen_dev_name(pci_dev, queue_id, name, sizeof(name));
591 dev = rte_dma_pmd_allocate(name, pci_dev->device.numa_node,
594 HISI_DMA_LOG(ERR, "%s allocate dmadev fail!", name);
598 dev->device = &pci_dev->device;
599 dev->dev_ops = &hisi_dmadev_ops;
601 hw = dev->data->dev_private;
602 hw->data = dev->data;
603 hw->revision = revision;
604 hw->reg_layout = hisi_dma_reg_layout(revision);
605 hw->io_base = pci_dev->mem_resource[REG_PCI_BAR_INDEX].addr;
606 hw->queue_id = queue_id;
607 hw->sq_tail_reg = hisi_dma_queue_regaddr(hw,
608 HISI_DMA_QUEUE_SQ_TAIL_REG);
609 hw->cq_head_reg = hisi_dma_queue_regaddr(hw,
610 HISI_DMA_QUEUE_CQ_HEAD_REG);
612 ret = hisi_dma_reset_hw(hw);
614 HISI_DMA_LOG(ERR, "%s init device fail!", name);
615 (void)rte_dma_pmd_release(name);
619 dev->state = RTE_DMA_DEV_READY;
620 HISI_DMA_LOG(DEBUG, "%s create dmadev success!", name);
626 hisi_dma_check_revision(struct rte_pci_device *pci_dev, const char *name,
627 uint8_t *out_revision)
632 ret = rte_pci_read_config(pci_dev, &revision, 1,
633 HISI_DMA_PCI_REVISION_ID_REG);
635 HISI_DMA_LOG(ERR, "%s read PCI revision failed!", name);
638 if (hisi_dma_reg_layout(revision) == HISI_DMA_REG_LAYOUT_INVALID) {
639 HISI_DMA_LOG(ERR, "%s revision: 0x%x not supported!",
644 *out_revision = revision;
649 hisi_dma_probe(struct rte_pci_driver *pci_drv __rte_unused,
650 struct rte_pci_device *pci_dev)
652 char name[RTE_DEV_NAME_MAX_LEN] = { 0 };
657 hisi_dma_gen_pci_device_name(pci_dev, name, sizeof(name));
659 if (pci_dev->mem_resource[2].addr == NULL) {
660 HISI_DMA_LOG(ERR, "%s BAR2 is NULL!\n", name);
664 ret = hisi_dma_check_revision(pci_dev, name, &revision);
667 HISI_DMA_LOG(DEBUG, "%s read PCI revision: 0x%x", name, revision);
669 hisi_dma_init_gbl(pci_dev->mem_resource[2].addr, revision);
671 for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
672 ret = hisi_dma_create(pci_dev, i, revision);
674 HISI_DMA_LOG(ERR, "%s create dmadev %u failed!",
684 hisi_dma_remove(struct rte_pci_device *pci_dev)
686 char name[RTE_DEV_NAME_MAX_LEN];
690 for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
691 hisi_dma_gen_dev_name(pci_dev, i, name, sizeof(name));
692 ret = rte_dma_pmd_release(name);
700 static const struct rte_pci_id pci_id_hisi_dma_map[] = {
701 { RTE_PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HISI_DMA_DEVICE_ID) },
702 { .vendor_id = 0, }, /* sentinel */
705 static struct rte_pci_driver hisi_dma_pmd_drv = {
706 .id_table = pci_id_hisi_dma_map,
707 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
708 .probe = hisi_dma_probe,
709 .remove = hisi_dma_remove,
712 RTE_PMD_REGISTER_PCI(dma_hisilicon, hisi_dma_pmd_drv);
713 RTE_PMD_REGISTER_PCI_TABLE(dma_hisilicon, pci_id_hisi_dma_map);
714 RTE_PMD_REGISTER_KMOD_DEP(dma_hisilicon, "vfio-pci");