dma/hisilicon: support multi-process
[dpdk.git] / drivers / dma / hisilicon / hisi_dmadev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(C) 2021 HiSilicon Limited
3  */
4
5 #include <inttypes.h>
6 #include <string.h>
7
8 #include <rte_bus_pci.h>
9 #include <rte_cycles.h>
10 #include <rte_eal.h>
11 #include <rte_io.h>
12 #include <rte_log.h>
13 #include <rte_malloc.h>
14 #include <rte_memzone.h>
15 #include <rte_pci.h>
16 #include <rte_dmadev_pmd.h>
17
18 #include "hisi_dmadev.h"
19
20 RTE_LOG_REGISTER_DEFAULT(hisi_dma_logtype, INFO);
21 #define HISI_DMA_LOG(level, fmt, args...) \
22                 rte_log(RTE_LOG_ ## level, hisi_dma_logtype, \
23                 "%s(): " fmt "\n", __func__, ##args)
24 #define HISI_DMA_LOG_RAW(hw, level, fmt, args...) \
25                 rte_log(RTE_LOG_ ## level, hisi_dma_logtype, \
26                 "%s %s(): " fmt "\n", (hw)->data->dev_name, \
27                 __func__, ##args)
28 #define HISI_DMA_DEBUG(hw, fmt, args...) \
29                 HISI_DMA_LOG_RAW(hw, DEBUG, fmt, ## args)
30 #define HISI_DMA_INFO(hw, fmt, args...) \
31                 HISI_DMA_LOG_RAW(hw, INFO, fmt, ## args)
32 #define HISI_DMA_WARN(hw, fmt, args...) \
33                 HISI_DMA_LOG_RAW(hw, WARNING, fmt, ## args)
34 #define HISI_DMA_ERR(hw, fmt, args...) \
35                 HISI_DMA_LOG_RAW(hw, ERR, fmt, ## args)
36
37 static uint32_t
38 hisi_dma_queue_base(struct hisi_dma_dev *hw)
39 {
40         if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
41                 return HISI_DMA_HIP08_QUEUE_BASE;
42         else
43                 return 0;
44 }
45
46 static volatile void *
47 hisi_dma_queue_regaddr(struct hisi_dma_dev *hw, uint32_t qoff)
48 {
49         uint32_t off = hisi_dma_queue_base(hw) +
50                         hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
51         return (volatile void *)((char *)hw->io_base + off);
52 }
53
54 static void
55 hisi_dma_write_reg(void *base, uint32_t off, uint32_t val)
56 {
57         rte_write32(rte_cpu_to_le_32(val),
58                     (volatile void *)((char *)base + off));
59 }
60
61 static void
62 hisi_dma_write_dev(struct hisi_dma_dev *hw, uint32_t off, uint32_t val)
63 {
64         hisi_dma_write_reg(hw->io_base, off, val);
65 }
66
67 static void
68 hisi_dma_write_queue(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t val)
69 {
70         uint32_t off = hisi_dma_queue_base(hw) +
71                         hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
72         hisi_dma_write_dev(hw, off, val);
73 }
74
75 static uint32_t
76 hisi_dma_read_reg(void *base, uint32_t off)
77 {
78         uint32_t val = rte_read32((volatile void *)((char *)base + off));
79         return rte_le_to_cpu_32(val);
80 }
81
82 static uint32_t
83 hisi_dma_read_dev(struct hisi_dma_dev *hw, uint32_t off)
84 {
85         return hisi_dma_read_reg(hw->io_base, off);
86 }
87
88 static uint32_t
89 hisi_dma_read_queue(struct hisi_dma_dev *hw, uint32_t qoff)
90 {
91         uint32_t off = hisi_dma_queue_base(hw) +
92                         hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
93         return hisi_dma_read_dev(hw, off);
94 }
95
96 static void
97 hisi_dma_update_bit(struct hisi_dma_dev *hw, uint32_t off, uint32_t pos,
98                     bool set)
99 {
100         uint32_t tmp = hisi_dma_read_dev(hw, off);
101         uint32_t mask = 1u << pos;
102         tmp = set ? tmp | mask : tmp & ~mask;
103         hisi_dma_write_dev(hw, off, tmp);
104 }
105
106 static void
107 hisi_dma_update_queue_bit(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t pos,
108                           bool set)
109 {
110         uint32_t tmp = hisi_dma_read_queue(hw, qoff);
111         uint32_t mask = 1u << pos;
112         tmp = set ? tmp | mask : tmp & ~mask;
113         hisi_dma_write_queue(hw, qoff, tmp);
114 }
115
116 static void
117 hisi_dma_update_queue_mbit(struct hisi_dma_dev *hw, uint32_t qoff,
118                            uint32_t mask, bool set)
119 {
120         uint32_t tmp = hisi_dma_read_queue(hw, qoff);
121         tmp = set ? tmp | mask : tmp & ~mask;
122         hisi_dma_write_queue(hw, qoff, tmp);
123 }
124
125 #define hisi_dma_poll_hw_state(hw, val, cond, sleep_us, timeout_us) ({ \
126         uint32_t timeout = 0; \
127         while (timeout++ <= (timeout_us)) { \
128                 (val) = hisi_dma_read_queue(hw, HISI_DMA_QUEUE_FSM_REG); \
129                 if (cond) \
130                         break; \
131                 rte_delay_us(sleep_us); \
132         } \
133         (cond) ? 0 : -ETIME; \
134 })
135
136 static int
137 hisi_dma_reset_hw(struct hisi_dma_dev *hw)
138 {
139 #define POLL_SLEEP_US   100
140 #define POLL_TIMEOUT_US 10000
141
142         uint32_t tmp;
143         int ret;
144
145         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
146                                   HISI_DMA_QUEUE_CTRL0_PAUSE_B, true);
147         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
148                                   HISI_DMA_QUEUE_CTRL0_EN_B, false);
149
150         ret = hisi_dma_poll_hw_state(hw, tmp,
151                 FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) != HISI_DMA_STATE_RUN,
152                 POLL_SLEEP_US, POLL_TIMEOUT_US);
153         if (ret) {
154                 HISI_DMA_ERR(hw, "disable dma timeout!");
155                 return ret;
156         }
157
158         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL1_REG,
159                                   HISI_DMA_QUEUE_CTRL1_RESET_B, true);
160         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_TAIL_REG, 0);
161         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_HEAD_REG, 0);
162         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
163                                   HISI_DMA_QUEUE_CTRL0_PAUSE_B, false);
164
165         ret = hisi_dma_poll_hw_state(hw, tmp,
166                 FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) == HISI_DMA_STATE_IDLE,
167                 POLL_SLEEP_US, POLL_TIMEOUT_US);
168         if (ret) {
169                 HISI_DMA_ERR(hw, "reset dma timeout!");
170                 return ret;
171         }
172
173         return 0;
174 }
175
176 static void
177 hisi_dma_init_hw(struct hisi_dma_dev *hw)
178 {
179         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_BASE_L_REG,
180                              lower_32_bits(hw->sqe_iova));
181         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_BASE_H_REG,
182                              upper_32_bits(hw->sqe_iova));
183         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_BASE_L_REG,
184                              lower_32_bits(hw->cqe_iova));
185         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_BASE_H_REG,
186                              upper_32_bits(hw->cqe_iova));
187         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_DEPTH_REG,
188                              hw->sq_depth_mask);
189         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_DEPTH_REG, hw->cq_depth - 1);
190         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_TAIL_REG, 0);
191         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_HEAD_REG, 0);
192         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM0_REG, 0);
193         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM1_REG, 0);
194         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM2_REG, 0);
195
196         if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
197                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM3_REG,
198                                      0);
199                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM4_REG,
200                                      0);
201                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM5_REG,
202                                      0);
203                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM6_REG,
204                                      0);
205                 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
206                                 HISI_DMA_HIP08_QUEUE_CTRL0_ERR_ABORT_B, false);
207                 hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_STATUS_REG,
208                                 HISI_DMA_HIP08_QUEUE_INT_MASK_M, true);
209                 hisi_dma_update_queue_mbit(hw,
210                                 HISI_DMA_HIP08_QUEUE_INT_MASK_REG,
211                                 HISI_DMA_HIP08_QUEUE_INT_MASK_M, true);
212         }
213 }
214
215 static void
216 hisi_dma_init_gbl(void *pci_bar, uint8_t revision)
217 {
218         struct hisi_dma_dev hw;
219
220         memset(&hw, 0, sizeof(hw));
221         hw.io_base = pci_bar;
222
223         if (revision == HISI_DMA_REVISION_HIP08B)
224                 hisi_dma_update_bit(&hw, HISI_DMA_HIP08_MODE_REG,
225                                     HISI_DMA_HIP08_MODE_SEL_B, true);
226 }
227
228 static uint8_t
229 hisi_dma_reg_layout(uint8_t revision)
230 {
231         if (revision == HISI_DMA_REVISION_HIP08B)
232                 return HISI_DMA_REG_LAYOUT_HIP08;
233         else
234                 return HISI_DMA_REG_LAYOUT_INVALID;
235 }
236
237 static void
238 hisi_dma_zero_iomem(struct hisi_dma_dev *hw)
239 {
240         memset(hw->iomz->addr, 0, hw->iomz_sz);
241 }
242
243 static int
244 hisi_dma_alloc_iomem(struct hisi_dma_dev *hw, uint16_t ring_size,
245                      const char *dev_name)
246 {
247         uint32_t sq_size = sizeof(struct hisi_dma_sqe) * ring_size;
248         uint32_t cq_size = sizeof(struct hisi_dma_cqe) *
249                            (ring_size + HISI_DMA_CQ_RESERVED);
250         uint32_t status_size = sizeof(uint16_t) * ring_size;
251         char mz_name[RTE_MEMZONE_NAMESIZE];
252         const struct rte_memzone *iomz;
253         uint32_t total_size;
254
255         sq_size = RTE_CACHE_LINE_ROUNDUP(sq_size);
256         cq_size = RTE_CACHE_LINE_ROUNDUP(cq_size);
257         status_size = RTE_CACHE_LINE_ROUNDUP(status_size);
258         total_size = sq_size + cq_size + status_size;
259
260         (void)snprintf(mz_name, sizeof(mz_name), "hisi_dma:%s", dev_name);
261         iomz = rte_memzone_reserve(mz_name, total_size, hw->data->numa_node,
262                                    RTE_MEMZONE_IOVA_CONTIG);
263         if (iomz == NULL) {
264                 HISI_DMA_ERR(hw, "malloc %s iomem fail!", mz_name);
265                 return -ENOMEM;
266         }
267
268         hw->iomz = iomz;
269         hw->iomz_sz = total_size;
270         hw->sqe = iomz->addr;
271         hw->cqe = (void *)((char *)iomz->addr + sq_size);
272         hw->status = (void *)((char *)iomz->addr + sq_size + cq_size);
273         hw->sqe_iova = iomz->iova;
274         hw->cqe_iova = iomz->iova + sq_size;
275         hw->sq_depth_mask = ring_size - 1;
276         hw->cq_depth = ring_size + HISI_DMA_CQ_RESERVED;
277         hisi_dma_zero_iomem(hw);
278
279         return 0;
280 }
281
282 static void
283 hisi_dma_free_iomem(struct hisi_dma_dev *hw)
284 {
285         if (hw->iomz != NULL)
286                 rte_memzone_free(hw->iomz);
287
288         hw->iomz = NULL;
289         hw->sqe = NULL;
290         hw->cqe = NULL;
291         hw->status = NULL;
292         hw->sqe_iova = 0;
293         hw->cqe_iova = 0;
294         hw->sq_depth_mask = 0;
295         hw->cq_depth = 0;
296 }
297
298 static int
299 hisi_dma_info_get(const struct rte_dma_dev *dev,
300                   struct rte_dma_info *dev_info,
301                   uint32_t info_sz)
302 {
303         RTE_SET_USED(dev);
304         RTE_SET_USED(info_sz);
305
306         dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
307                              RTE_DMA_CAPA_OPS_COPY;
308         dev_info->max_vchans = 1;
309         dev_info->max_desc = HISI_DMA_MAX_DESC_NUM;
310         dev_info->min_desc = HISI_DMA_MIN_DESC_NUM;
311
312         return 0;
313 }
314
315 static int
316 hisi_dma_configure(struct rte_dma_dev *dev,
317                    const struct rte_dma_conf *conf,
318                    uint32_t conf_sz)
319 {
320         RTE_SET_USED(dev);
321         RTE_SET_USED(conf);
322         RTE_SET_USED(conf_sz);
323         return 0;
324 }
325
326 static int
327 hisi_dma_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
328                      const struct rte_dma_vchan_conf *conf,
329                      uint32_t conf_sz)
330 {
331         struct hisi_dma_dev *hw = dev->data->dev_private;
332         int ret;
333
334         RTE_SET_USED(vchan);
335         RTE_SET_USED(conf_sz);
336
337         if (!rte_is_power_of_2(conf->nb_desc)) {
338                 HISI_DMA_ERR(hw, "Number of desc must be power of 2!");
339                 return -EINVAL;
340         }
341
342         hisi_dma_free_iomem(hw);
343         ret = hisi_dma_alloc_iomem(hw, conf->nb_desc, dev->data->dev_name);
344         if (ret)
345                 return ret;
346
347         return 0;
348 }
349
350 static int
351 hisi_dma_start(struct rte_dma_dev *dev)
352 {
353         struct hisi_dma_dev *hw = dev->data->dev_private;
354
355         if (hw->iomz == NULL) {
356                 HISI_DMA_ERR(hw, "Vchan was not setup, start fail!\n");
357                 return -EINVAL;
358         }
359
360         /* Reset the dmadev to a known state, include:
361          *   1) zero iomem, also include status fields.
362          *   2) init hardware register.
363          *   3) init index values to zero.
364          *   4) init running statistics.
365          */
366         hisi_dma_zero_iomem(hw);
367         hisi_dma_init_hw(hw);
368         hw->ridx = 0;
369         hw->cridx = 0;
370         hw->sq_head = 0;
371         hw->sq_tail = 0;
372         hw->cq_sq_head = 0;
373         hw->cq_head = 0;
374         hw->cqs_completed = 0;
375         hw->cqe_vld = 1;
376         hw->submitted = 0;
377         hw->completed = 0;
378         hw->errors = 0;
379
380         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
381                                   HISI_DMA_QUEUE_CTRL0_EN_B, true);
382
383         return 0;
384 }
385
386 static int
387 hisi_dma_stop(struct rte_dma_dev *dev)
388 {
389         return hisi_dma_reset_hw(dev->data->dev_private);
390 }
391
392 static int
393 hisi_dma_close(struct rte_dma_dev *dev)
394 {
395         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
396                 /* The dmadev already stopped */
397                 hisi_dma_free_iomem(dev->data->dev_private);
398         }
399         return 0;
400 }
401
402 static int
403 hisi_dma_stats_get(const struct rte_dma_dev *dev, uint16_t vchan,
404                    struct rte_dma_stats *stats,
405                    uint32_t stats_sz)
406 {
407         struct hisi_dma_dev *hw = dev->data->dev_private;
408
409         RTE_SET_USED(vchan);
410         RTE_SET_USED(stats_sz);
411         stats->submitted = hw->submitted;
412         stats->completed = hw->completed;
413         stats->errors = hw->errors;
414
415         return 0;
416 }
417
418 static int
419 hisi_dma_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
420 {
421         struct hisi_dma_dev *hw = dev->data->dev_private;
422
423         RTE_SET_USED(vchan);
424         hw->submitted = 0;
425         hw->completed = 0;
426         hw->errors = 0;
427
428         return 0;
429 }
430
431 static void
432 hisi_dma_get_dump_range(struct hisi_dma_dev *hw, uint32_t *start, uint32_t *end)
433 {
434         if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
435                 *start = HISI_DMA_HIP08_DUMP_START_REG;
436                 *end = HISI_DMA_HIP08_DUMP_END_REG;
437         } else {
438                 *start = 0;
439                 *end = 0;
440         }
441 }
442
443 static void
444 hisi_dma_dump_common(struct hisi_dma_dev *hw, FILE *f)
445 {
446 #define DUMP_REGNUM_PER_LINE    4
447
448         uint32_t start, end;
449         uint32_t cnt, i;
450
451         hisi_dma_get_dump_range(hw, &start, &end);
452
453         (void)fprintf(f, "    common-register:\n");
454
455         cnt = 0;
456         for (i = start; i <= end; i += sizeof(uint32_t)) {
457                 if (cnt % DUMP_REGNUM_PER_LINE == 0)
458                         (void)fprintf(f, "      [%4x]:", i);
459                 (void)fprintf(f, " 0x%08x", hisi_dma_read_dev(hw, i));
460                 cnt++;
461                 if (cnt % DUMP_REGNUM_PER_LINE == 0)
462                         (void)fprintf(f, "\n");
463         }
464         if (cnt % DUMP_REGNUM_PER_LINE)
465                 (void)fprintf(f, "\n");
466 }
467
468 static void
469 hisi_dma_dump_read_queue(struct hisi_dma_dev *hw, uint32_t qoff,
470                          char *buffer, int max_sz)
471 {
472         memset(buffer, 0, max_sz);
473
474         /* Address-related registers are not printed for security reasons. */
475         if (qoff == HISI_DMA_QUEUE_SQ_BASE_L_REG ||
476             qoff == HISI_DMA_QUEUE_SQ_BASE_H_REG ||
477             qoff == HISI_DMA_QUEUE_CQ_BASE_L_REG ||
478             qoff == HISI_DMA_QUEUE_CQ_BASE_H_REG) {
479                 (void)snprintf(buffer, max_sz, "**********");
480                 return;
481         }
482
483         (void)snprintf(buffer, max_sz, "0x%08x", hisi_dma_read_queue(hw, qoff));
484 }
485
486 static void
487 hisi_dma_dump_queue(struct hisi_dma_dev *hw, FILE *f)
488 {
489 #define REG_FMT_LEN     32
490         char buf[REG_FMT_LEN] = { 0 };
491         uint32_t i;
492
493         (void)fprintf(f, "    queue-register:\n");
494         for (i = 0; i < HISI_DMA_QUEUE_REGION_SIZE; ) {
495                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
496                 (void)fprintf(f, "      [%2x]: %s", i, buf);
497                 i += sizeof(uint32_t);
498                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
499                 (void)fprintf(f, " %s", buf);
500                 i += sizeof(uint32_t);
501                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
502                 (void)fprintf(f, " %s", buf);
503                 i += sizeof(uint32_t);
504                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
505                 (void)fprintf(f, " %s\n", buf);
506                 i += sizeof(uint32_t);
507         }
508 }
509
510 static int
511 hisi_dma_dump(const struct rte_dma_dev *dev, FILE *f)
512 {
513         struct hisi_dma_dev *hw = dev->data->dev_private;
514
515         (void)fprintf(f,
516                 "    revision: 0x%x queue_id: %u ring_size: %u\n"
517                 "    ridx: %u cridx: %u\n"
518                 "    sq_head: %u sq_tail: %u cq_sq_head: %u\n"
519                 "    cq_head: %u cqs_completed: %u cqe_vld: %u\n"
520                 "    submitted: %" PRIu64 " completed: %" PRIu64 " errors %"
521                 PRIu64"\n",
522                 hw->revision, hw->queue_id,
523                 hw->sq_depth_mask > 0 ? hw->sq_depth_mask + 1 : 0,
524                 hw->ridx, hw->cridx,
525                 hw->sq_head, hw->sq_tail, hw->cq_sq_head,
526                 hw->cq_head, hw->cqs_completed, hw->cqe_vld,
527                 hw->submitted, hw->completed, hw->errors);
528         hisi_dma_dump_queue(hw, f);
529         hisi_dma_dump_common(hw, f);
530
531         return 0;
532 }
533
534 static int
535 hisi_dma_copy(void *dev_private, uint16_t vchan,
536                  rte_iova_t src, rte_iova_t dst,
537                  uint32_t length, uint64_t flags)
538 {
539         struct hisi_dma_dev *hw = dev_private;
540         struct hisi_dma_sqe *sqe = &hw->sqe[hw->sq_tail];
541
542         RTE_SET_USED(vchan);
543
544         if (((hw->sq_tail + 1) & hw->sq_depth_mask) == hw->sq_head)
545                 return -ENOSPC;
546
547         sqe->dw0 = rte_cpu_to_le_32(SQE_OPCODE_M2M);
548         sqe->dw1 = 0;
549         sqe->dw2 = 0;
550         sqe->length = rte_cpu_to_le_32(length);
551         sqe->src_addr = rte_cpu_to_le_64(src);
552         sqe->dst_addr = rte_cpu_to_le_64(dst);
553         hw->sq_tail = (hw->sq_tail + 1) & hw->sq_depth_mask;
554         hw->submitted++;
555
556         if (flags & RTE_DMA_OP_FLAG_FENCE)
557                 sqe->dw0 |= rte_cpu_to_le_32(SQE_FENCE_FLAG);
558         if (flags & RTE_DMA_OP_FLAG_SUBMIT)
559                 rte_write32(rte_cpu_to_le_32(hw->sq_tail), hw->sq_tail_reg);
560
561         return hw->ridx++;
562 }
563
564 static int
565 hisi_dma_submit(void *dev_private, uint16_t vchan)
566 {
567         struct hisi_dma_dev *hw = dev_private;
568
569         RTE_SET_USED(vchan);
570         rte_write32(rte_cpu_to_le_32(hw->sq_tail), hw->sq_tail_reg);
571
572         return 0;
573 }
574
575 static inline void
576 hisi_dma_scan_cq(struct hisi_dma_dev *hw)
577 {
578         volatile struct hisi_dma_cqe *cqe;
579         uint16_t csq_head = hw->cq_sq_head;
580         uint16_t cq_head = hw->cq_head;
581         uint16_t count = 0;
582         uint64_t misc;
583
584         while (true) {
585                 cqe = &hw->cqe[cq_head];
586                 misc = cqe->misc;
587                 misc = rte_le_to_cpu_64(misc);
588                 if (FIELD_GET(CQE_VALID_B, misc) != hw->cqe_vld)
589                         break;
590
591                 csq_head = FIELD_GET(CQE_SQ_HEAD_MASK, misc);
592                 if (unlikely(misc & CQE_STATUS_MASK))
593                         hw->status[csq_head] = FIELD_GET(CQE_STATUS_MASK,
594                                                          misc);
595
596                 count++;
597                 cq_head++;
598                 if (cq_head == hw->cq_depth) {
599                         hw->cqe_vld = !hw->cqe_vld;
600                         cq_head = 0;
601                 }
602         }
603
604         if (count == 0)
605                 return;
606
607         hw->cq_head = cq_head;
608         hw->cq_sq_head = (csq_head + 1) & hw->sq_depth_mask;
609         hw->cqs_completed += count;
610         if (hw->cqs_completed >= HISI_DMA_CQ_RESERVED) {
611                 rte_write32(rte_cpu_to_le_32(cq_head), hw->cq_head_reg);
612                 hw->cqs_completed = 0;
613         }
614 }
615
616 static inline uint16_t
617 hisi_dma_calc_cpls(struct hisi_dma_dev *hw, const uint16_t nb_cpls)
618 {
619         uint16_t cpl_num;
620
621         if (hw->cq_sq_head >= hw->sq_head)
622                 cpl_num = hw->cq_sq_head - hw->sq_head;
623         else
624                 cpl_num = hw->sq_depth_mask + 1 - hw->sq_head + hw->cq_sq_head;
625
626         if (cpl_num > nb_cpls)
627                 cpl_num = nb_cpls;
628
629         return cpl_num;
630 }
631
632 static uint16_t
633 hisi_dma_completed(void *dev_private,
634                    uint16_t vchan, const uint16_t nb_cpls,
635                    uint16_t *last_idx, bool *has_error)
636 {
637         struct hisi_dma_dev *hw = dev_private;
638         uint16_t sq_head = hw->sq_head;
639         uint16_t cpl_num, i;
640
641         RTE_SET_USED(vchan);
642         hisi_dma_scan_cq(hw);
643
644         cpl_num = hisi_dma_calc_cpls(hw, nb_cpls);
645         for (i = 0; i < cpl_num; i++) {
646                 if (hw->status[sq_head]) {
647                         *has_error = true;
648                         break;
649                 }
650                 sq_head = (sq_head + 1) & hw->sq_depth_mask;
651         }
652         if (i > 0) {
653                 hw->cridx += i;
654                 *last_idx = hw->cridx - 1;
655                 hw->sq_head = sq_head;
656         }
657         hw->completed += i;
658
659         return i;
660 }
661
662 static enum rte_dma_status_code
663 hisi_dma_convert_status(uint16_t status)
664 {
665         switch (status) {
666         case HISI_DMA_STATUS_SUCCESS:
667                 return RTE_DMA_STATUS_SUCCESSFUL;
668         case HISI_DMA_STATUS_INVALID_OPCODE:
669                 return RTE_DMA_STATUS_INVALID_OPCODE;
670         case HISI_DMA_STATUS_INVALID_LENGTH:
671                 return RTE_DMA_STATUS_INVALID_LENGTH;
672         case HISI_DMA_STATUS_USER_ABORT:
673                 return RTE_DMA_STATUS_USER_ABORT;
674         case HISI_DMA_STATUS_REMOTE_READ_ERROR:
675         case HISI_DMA_STATUS_AXI_READ_ERROR:
676                 return RTE_DMA_STATUS_BUS_READ_ERROR;
677         case HISI_DMA_STATUS_AXI_WRITE_ERROR:
678                 return RTE_DMA_STATUS_BUS_WRITE_ERROR;
679         case HISI_DMA_STATUS_DATA_POISON:
680         case HISI_DMA_STATUS_REMOTE_DATA_POISION:
681                 return RTE_DMA_STATUS_DATA_POISION;
682         case HISI_DMA_STATUS_SQE_READ_ERROR:
683         case HISI_DMA_STATUS_SQE_READ_POISION:
684                 return RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR;
685         case HISI_DMA_STATUS_LINK_DOWN_ERROR:
686                 return RTE_DMA_STATUS_DEV_LINK_ERROR;
687         default:
688                 return RTE_DMA_STATUS_ERROR_UNKNOWN;
689         }
690 }
691
692 static uint16_t
693 hisi_dma_completed_status(void *dev_private,
694                           uint16_t vchan, const uint16_t nb_cpls,
695                           uint16_t *last_idx, enum rte_dma_status_code *status)
696 {
697         struct hisi_dma_dev *hw = dev_private;
698         uint16_t sq_head = hw->sq_head;
699         uint16_t cpl_num, i;
700
701         RTE_SET_USED(vchan);
702         hisi_dma_scan_cq(hw);
703
704         cpl_num = hisi_dma_calc_cpls(hw, nb_cpls);
705         for (i = 0; i < cpl_num; i++) {
706                 status[i] = hisi_dma_convert_status(hw->status[sq_head]);
707                 hw->errors += !!status[i];
708                 hw->status[sq_head] = HISI_DMA_STATUS_SUCCESS;
709                 sq_head = (sq_head + 1) & hw->sq_depth_mask;
710         }
711         if (likely(cpl_num > 0)) {
712                 hw->cridx += cpl_num;
713                 *last_idx = hw->cridx - 1;
714                 hw->sq_head = sq_head;
715         }
716         hw->completed += cpl_num;
717
718         return cpl_num;
719 }
720
721 static uint16_t
722 hisi_dma_burst_capacity(const void *dev_private, uint16_t vchan)
723 {
724         const struct hisi_dma_dev *hw = dev_private;
725         uint16_t sq_head = hw->sq_head;
726         uint16_t sq_tail = hw->sq_tail;
727
728         RTE_SET_USED(vchan);
729
730         return (sq_tail >= sq_head) ? hw->sq_depth_mask - sq_tail + sq_head :
731                                       sq_head - 1 - sq_tail;
732 }
733
734 static void
735 hisi_dma_gen_pci_device_name(const struct rte_pci_device *pci_dev,
736                              char *name, size_t size)
737 {
738         memset(name, 0, size);
739         (void)snprintf(name, size, "%x:%x.%x",
740                  pci_dev->addr.bus, pci_dev->addr.devid,
741                  pci_dev->addr.function);
742 }
743
744 static void
745 hisi_dma_gen_dev_name(const struct rte_pci_device *pci_dev,
746                       uint8_t queue_id, char *name, size_t size)
747 {
748         memset(name, 0, size);
749         (void)snprintf(name, size, "%x:%x.%x-ch%u",
750                  pci_dev->addr.bus, pci_dev->addr.devid,
751                  pci_dev->addr.function, queue_id);
752 }
753
754 /**
755  * Hardware queue state machine:
756  *
757  *   -----------  dmadev_create   ------------------
758  *   | Unknown | ---------------> |      IDLE      |
759  *   -----------                  ------------------
760  *                                   ^          |
761  *                                   |          |dev_start
762  *                           dev_stop|          |
763  *                                   |          v
764  *                                ------------------
765  *                                |      RUN       |
766  *                                ------------------
767  *
768  */
769 static const struct rte_dma_dev_ops hisi_dmadev_ops = {
770         .dev_info_get     = hisi_dma_info_get,
771         .dev_configure    = hisi_dma_configure,
772         .dev_start        = hisi_dma_start,
773         .dev_stop         = hisi_dma_stop,
774         .dev_close        = hisi_dma_close,
775         .vchan_setup      = hisi_dma_vchan_setup,
776         .stats_get        = hisi_dma_stats_get,
777         .stats_reset      = hisi_dma_stats_reset,
778         .dev_dump         = hisi_dma_dump,
779 };
780
781 static int
782 hisi_dma_create(struct rte_pci_device *pci_dev, uint8_t queue_id,
783                 uint8_t revision)
784 {
785 #define REG_PCI_BAR_INDEX       2
786
787         char name[RTE_DEV_NAME_MAX_LEN];
788         struct rte_dma_dev *dev;
789         struct hisi_dma_dev *hw;
790         int ret;
791
792         hisi_dma_gen_dev_name(pci_dev, queue_id, name, sizeof(name));
793         dev = rte_dma_pmd_allocate(name, pci_dev->device.numa_node,
794                                    sizeof(*hw));
795         if (dev == NULL) {
796                 HISI_DMA_LOG(ERR, "%s allocate dmadev fail!", name);
797                 return -EINVAL;
798         }
799
800         dev->device = &pci_dev->device;
801         dev->dev_ops = &hisi_dmadev_ops;
802         dev->fp_obj->dev_private = dev->data->dev_private;
803         dev->fp_obj->copy = hisi_dma_copy;
804         dev->fp_obj->submit = hisi_dma_submit;
805         dev->fp_obj->completed = hisi_dma_completed;
806         dev->fp_obj->completed_status = hisi_dma_completed_status;
807         dev->fp_obj->burst_capacity = hisi_dma_burst_capacity;
808
809         hw = dev->data->dev_private;
810         hw->data = dev->data;
811         hw->revision = revision;
812         hw->reg_layout = hisi_dma_reg_layout(revision);
813         hw->io_base = pci_dev->mem_resource[REG_PCI_BAR_INDEX].addr;
814         hw->queue_id = queue_id;
815         hw->sq_tail_reg = hisi_dma_queue_regaddr(hw,
816                                                  HISI_DMA_QUEUE_SQ_TAIL_REG);
817         hw->cq_head_reg = hisi_dma_queue_regaddr(hw,
818                                                  HISI_DMA_QUEUE_CQ_HEAD_REG);
819
820         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
821                 ret = hisi_dma_reset_hw(hw);
822                 if (ret) {
823                         HISI_DMA_LOG(ERR, "%s init device fail!", name);
824                         (void)rte_dma_pmd_release(name);
825                         return -EIO;
826                 }
827         }
828
829         dev->state = RTE_DMA_DEV_READY;
830         HISI_DMA_LOG(DEBUG, "%s create dmadev success!", name);
831
832         return 0;
833 }
834
835 static int
836 hisi_dma_check_revision(struct rte_pci_device *pci_dev, const char *name,
837                         uint8_t *out_revision)
838 {
839         uint8_t revision;
840         int ret;
841
842         ret = rte_pci_read_config(pci_dev, &revision, 1,
843                                   HISI_DMA_PCI_REVISION_ID_REG);
844         if (ret != 1) {
845                 HISI_DMA_LOG(ERR, "%s read PCI revision failed!", name);
846                 return -EINVAL;
847         }
848         if (hisi_dma_reg_layout(revision) == HISI_DMA_REG_LAYOUT_INVALID) {
849                 HISI_DMA_LOG(ERR, "%s revision: 0x%x not supported!",
850                              name, revision);
851                 return -EINVAL;
852         }
853
854         *out_revision = revision;
855         return 0;
856 }
857
858 static int
859 hisi_dma_probe(struct rte_pci_driver *pci_drv __rte_unused,
860                struct rte_pci_device *pci_dev)
861 {
862         char name[RTE_DEV_NAME_MAX_LEN] = { 0 };
863         uint8_t revision;
864         uint8_t i;
865         int ret;
866
867         hisi_dma_gen_pci_device_name(pci_dev, name, sizeof(name));
868
869         if (pci_dev->mem_resource[2].addr == NULL) {
870                 HISI_DMA_LOG(ERR, "%s BAR2 is NULL!\n", name);
871                 return -ENODEV;
872         }
873
874         ret = hisi_dma_check_revision(pci_dev, name, &revision);
875         if (ret)
876                 return ret;
877         HISI_DMA_LOG(DEBUG, "%s read PCI revision: 0x%x", name, revision);
878
879         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
880                 hisi_dma_init_gbl(pci_dev->mem_resource[2].addr, revision);
881
882         for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
883                 ret = hisi_dma_create(pci_dev, i, revision);
884                 if (ret) {
885                         HISI_DMA_LOG(ERR, "%s create dmadev %u failed!",
886                                      name, i);
887                         break;
888                 }
889         }
890
891         return ret;
892 }
893
894 static int
895 hisi_dma_remove(struct rte_pci_device *pci_dev)
896 {
897         char name[RTE_DEV_NAME_MAX_LEN];
898         uint8_t i;
899         int ret;
900
901         for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
902                 hisi_dma_gen_dev_name(pci_dev, i, name, sizeof(name));
903                 ret = rte_dma_pmd_release(name);
904                 if (ret)
905                         return ret;
906         }
907
908         return 0;
909 }
910
911 static const struct rte_pci_id pci_id_hisi_dma_map[] = {
912         { RTE_PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HISI_DMA_DEVICE_ID) },
913         { .vendor_id = 0, }, /* sentinel */
914 };
915
916 static struct rte_pci_driver hisi_dma_pmd_drv = {
917         .id_table  = pci_id_hisi_dma_map,
918         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
919         .probe     = hisi_dma_probe,
920         .remove    = hisi_dma_remove,
921 };
922
923 RTE_PMD_REGISTER_PCI(dma_hisilicon, hisi_dma_pmd_drv);
924 RTE_PMD_REGISTER_PCI_TABLE(dma_hisilicon, pci_id_hisi_dma_map);
925 RTE_PMD_REGISTER_KMOD_DEP(dma_hisilicon, "vfio-pci");