1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 HiSilicon Limited
8 #include <rte_byteorder.h>
9 #include <rte_common.h>
11 #define BIT(x) (1ul << (x))
12 #define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
13 #define GENMASK(h, l) \
14 (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
15 #define BF_SHF(x) (__builtin_ffsll(x) - 1)
16 #define FIELD_GET(mask, reg) \
17 ((typeof(mask))(((reg) & (mask)) >> BF_SHF(mask)))
19 #define lower_32_bits(x) ((uint32_t)(x))
20 #define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16))
22 #define PCI_VENDOR_ID_HUAWEI 0x19e5
23 #define HISI_DMA_DEVICE_ID 0xA122
24 #define HISI_DMA_PCI_REVISION_ID_REG 0x08
25 #define HISI_DMA_REVISION_HIP08B 0x21
26 #define HISI_DMA_REVISION_HIP09A 0x30
28 #define HISI_DMA_MAX_HW_QUEUES 4
29 #define HISI_DMA_MAX_DESC_NUM 8192
30 #define HISI_DMA_MIN_DESC_NUM 32
33 * The HIP08B(HiSilicon IP08) and HIP09B(HiSilicon IP09) are DMA iEPs, they
34 * have the same pci device id but different pci revision.
35 * Unfortunately, they have different register layouts, so two layout
36 * enumerations are defined.
39 HISI_DMA_REG_LAYOUT_INVALID = 0,
40 HISI_DMA_REG_LAYOUT_HIP08,
41 HISI_DMA_REG_LAYOUT_HIP09
45 * Hardware PCI bar register MAP:
50 * -------------- -> Queue base
56 * | Queue-1 | Queue region
66 * As described above, a single queue register is continuous and occupies the
67 * length of queue-region. The global offset for a single queue register is
69 * offset = queue-base + (queue-id * queue-region) + reg-offset-in-region.
71 * The first part of queue region is basically the same for HIP08 and HIP09
72 * register layouts, therefore, HISI_QUEUE_* registers are defined for it.
74 #define HISI_DMA_QUEUE_SQ_BASE_L_REG 0x0
75 #define HISI_DMA_QUEUE_SQ_BASE_H_REG 0x4
76 #define HISI_DMA_QUEUE_SQ_DEPTH_REG 0x8
77 #define HISI_DMA_QUEUE_SQ_TAIL_REG 0xC
78 #define HISI_DMA_QUEUE_CQ_BASE_L_REG 0x10
79 #define HISI_DMA_QUEUE_CQ_BASE_H_REG 0x14
80 #define HISI_DMA_QUEUE_CQ_DEPTH_REG 0x18
81 #define HISI_DMA_QUEUE_CQ_HEAD_REG 0x1C
82 #define HISI_DMA_QUEUE_CTRL0_REG 0x20
83 #define HISI_DMA_QUEUE_CTRL0_EN_B 0
84 #define HISI_DMA_QUEUE_CTRL0_PAUSE_B 4
85 #define HISI_DMA_QUEUE_CTRL1_REG 0x24
86 #define HISI_DMA_QUEUE_CTRL1_RESET_B 0
87 #define HISI_DMA_QUEUE_FSM_REG 0x30
88 #define HISI_DMA_QUEUE_FSM_STS_M GENMASK(3, 0)
89 #define HISI_DMA_QUEUE_INT_STATUS_REG 0x40
90 #define HISI_DMA_QUEUE_INT_MASK_REG 0x44
91 #define HISI_DMA_QUEUE_ERR_INT_NUM0_REG 0x84
92 #define HISI_DMA_QUEUE_ERR_INT_NUM1_REG 0x88
93 #define HISI_DMA_QUEUE_ERR_INT_NUM2_REG 0x8C
94 #define HISI_DMA_QUEUE_REGION_SIZE 0x100
97 * HiSilicon IP08 DMA register and field define:
99 #define HISI_DMA_HIP08_QUEUE_BASE 0x0
100 #define HISI_DMA_HIP08_QUEUE_CTRL0_ERR_ABORT_B 2
101 #define HISI_DMA_HIP08_QUEUE_INT_MASK_M GENMASK(14, 0)
102 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM3_REG 0x90
103 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM4_REG 0x94
104 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM5_REG 0x98
105 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM6_REG 0x48
106 #define HISI_DMA_HIP08_MODE_REG 0x217C
107 #define HISI_DMA_HIP08_MODE_SEL_B 0
108 #define HISI_DMA_HIP08_DUMP_START_REG 0x2000
109 #define HISI_DMA_HIP08_DUMP_END_REG 0x2280
112 * HiSilicon IP09 DMA register and field define:
114 #define HISI_DMA_HIP09_QUEUE_BASE 0x2000
115 #define HISI_DMA_HIP09_QUEUE_CTRL0_ERR_ABORT_M GENMASK(31, 28)
116 #define HISI_DMA_HIP09_QUEUE_CTRL1_VA_ENABLE_B 2
117 #define HISI_DMA_HIP09_QUEUE_INT_MASK_M 0x1
118 #define HISI_DMA_HIP09_QUEUE_ERR_INT_STATUS_REG 0x48
119 #define HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_REG 0x4C
120 #define HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_M GENMASK(18, 1)
121 #define HISI_DMA_HIP09_QUEUE_CFG_REG(queue_id) (0x800 + \
123 #define HISI_DMA_HIP09_QUEUE_CFG_LINK_DOWN_MASK_B 16
124 #define HISI_DMA_HIP09_DUMP_REGION_A_START_REG 0x0
125 #define HISI_DMA_HIP09_DUMP_REGION_A_END_REG 0x368
126 #define HISI_DMA_HIP09_DUMP_REGION_B_START_REG 0x800
127 #define HISI_DMA_HIP09_DUMP_REGION_B_END_REG 0xA08
128 #define HISI_DMA_HIP09_DUMP_REGION_C_START_REG 0x1800
129 #define HISI_DMA_HIP09_DUMP_REGION_C_END_REG 0x1A4C
130 #define HISI_DMA_HIP09_DUMP_REGION_D_START_REG 0x1C00
131 #define HISI_DMA_HIP09_DUMP_REGION_D_END_REG 0x1CC4
134 * In fact, there are multiple states, but it need to pay attention to
135 * the following two states for the driver:
138 HISI_DMA_STATE_IDLE = 0,
143 * Hardware complete status define:
145 #define HISI_DMA_STATUS_SUCCESS 0x0
146 #define HISI_DMA_STATUS_INVALID_OPCODE 0x1
147 #define HISI_DMA_STATUS_INVALID_LENGTH 0x2
148 #define HISI_DMA_STATUS_USER_ABORT 0x4
149 #define HISI_DMA_STATUS_REMOTE_READ_ERROR 0x10
150 #define HISI_DMA_STATUS_AXI_READ_ERROR 0x20
151 #define HISI_DMA_STATUS_AXI_WRITE_ERROR 0x40
152 #define HISI_DMA_STATUS_DATA_POISON 0x80
153 #define HISI_DMA_STATUS_SQE_READ_ERROR 0x100
154 #define HISI_DMA_STATUS_SQE_READ_POISION 0x200
155 #define HISI_DMA_STATUS_REMOTE_DATA_POISION 0x400
156 #define HISI_DMA_STATUS_LINK_DOWN_ERROR 0x800
159 * After scanning the CQ array, the CQ head register needs to be updated.
160 * Updating the register involves write memory barrier operations.
161 * Here use the following method to reduce WMB operations:
162 * a) malloc more CQEs, which correspond to the macro HISI_DMA_CQ_RESERVED.
163 * b) update the CQ head register after accumulated number of completed CQs
164 * is greater than or equal to HISI_DMA_CQ_RESERVED.
166 #define HISI_DMA_CQ_RESERVED 64
168 struct hisi_dma_sqe {
170 #define SQE_FENCE_FLAG BIT(10)
171 #define SQE_OPCODE_M2M 0x4
179 struct hisi_dma_cqe {
182 #define CQE_SQ_HEAD_MASK GENMASK(15, 0)
183 #define CQE_VALID_B BIT(48)
184 #define CQE_STATUS_MASK GENMASK(63, 49)
187 struct hisi_dma_dev {
188 struct hisi_dma_sqe *sqe;
189 volatile struct hisi_dma_cqe *cqe;
190 uint16_t *status; /* the completion status array of SQEs. */
192 volatile void *sq_tail_reg; /**< register address for doorbell. */
193 volatile void *cq_head_reg; /**< register address for answer CQ. */
195 uint16_t sq_depth_mask; /**< SQ depth - 1, the SQ depth is power of 2 */
196 uint16_t cq_depth; /* CQ depth */
198 uint16_t ridx; /**< ring index which will assign to the next request. */
199 /** ring index which returned by hisi_dmadev_completed APIs. */
203 * SQE array management fields:
205 * -----------------------------------------------------
206 * | SQE0 | SQE1 | SQE2 | ... | SQEx | ... | SQEn-1 |
207 * -----------------------------------------------------
210 * sq_head cq_sq_head sq_tail
212 * sq_head: index to the oldest completed request, this filed was
213 * updated by hisi_dmadev_completed* APIs.
214 * sq_tail: index of the next new request, this field was updated by
215 * hisi_dmadev_copy API.
216 * cq_sq_head: next index of index that has been completed by hardware,
217 * this filed was updated by hisi_dmadev_completed* APIs.
219 * [sq_head, cq_sq_head): the SQEs that hardware already completed.
220 * [cq_sq_head, sq_tail): the SQEs that hardware processing.
226 * The driver scans the CQE array, if the valid bit changes, the CQE is
228 * Note: One CQE is corresponding to one or several SQEs, e.g. app
229 * submits two copy requests, the hardware processes the two SQEs,
230 * but it may write back only one CQE and the CQE's sq_head field
231 * indicates the index of the second copy request in the SQE
234 uint16_t cq_head; /**< CQ index for next scans. */
235 /** accumulated number of completed CQs
236 * @see HISI_DMA_CQ_RESERVED
238 uint16_t cqs_completed;
239 uint8_t cqe_vld; /**< valid bit for CQE, will change for every round. */
246 * The following fields are not accessed in the I/O path, so they are
249 struct rte_dma_dev_data *data;
250 uint8_t revision; /**< PCI revision. */
251 uint8_t reg_layout; /**< hardware register layout. */
253 uint8_t queue_id; /**< hardware DMA queue index. */
254 const struct rte_memzone *iomz;
260 #endif /* HISI_DMADEV_H */