drivers/dma/hisilicon/hisi_dmadev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 HiSilicon Limited
   3  */
   4
   5 #include <inttypes.h>
   6 #include <string.h>
   7
   8 #include <rte_bus_pci.h>
   9 #include <rte_cycles.h>
  10 #include <rte_eal.h>
  11 #include <rte_io.h>
  12 #include <rte_log.h>
  13 #include <rte_malloc.h>
  14 #include <rte_memzone.h>
  15 #include <rte_pci.h>
  16 #include <rte_dmadev_pmd.h>
  17
  18 #include "hisi_dmadev.h"
  19
  20 RTE_LOG_REGISTER_DEFAULT(hisi_dma_logtype, INFO);
  21 #define HISI_DMA_LOG(level, fmt, args...) \
  22                 rte_log(RTE_LOG_ ## level, hisi_dma_logtype, \
  23                 "%s(): " fmt "\n", __func__, ##args)
  24 #define HISI_DMA_LOG_RAW(hw, level, fmt, args...) \
  25                 rte_log(RTE_LOG_ ## level, hisi_dma_logtype, \
  26                 "%s %s(): " fmt "\n", (hw)->data->dev_name, \
  27                 __func__, ##args)
  28 #define HISI_DMA_DEBUG(hw, fmt, args...) \
  29                 HISI_DMA_LOG_RAW(hw, DEBUG, fmt, ## args)
  30 #define HISI_DMA_INFO(hw, fmt, args...) \
  31                 HISI_DMA_LOG_RAW(hw, INFO, fmt, ## args)
  32 #define HISI_DMA_WARN(hw, fmt, args...) \
  33                 HISI_DMA_LOG_RAW(hw, WARNING, fmt, ## args)
  34 #define HISI_DMA_ERR(hw, fmt, args...) \
  35                 HISI_DMA_LOG_RAW(hw, ERR, fmt, ## args)
  36
  37 static uint32_t
  38 hisi_dma_queue_base(struct hisi_dma_dev *hw)
  39 {
  40         if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
  41                 return HISI_DMA_HIP08_QUEUE_BASE;
  42         else
  43                 return 0;
  44 }
  45
  46 static volatile void *
  47 hisi_dma_queue_regaddr(struct hisi_dma_dev *hw, uint32_t qoff)
  48 {
  49         uint32_t off = hisi_dma_queue_base(hw) +
  50                         hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
  51         return (volatile void *)((char *)hw->io_base + off);
  52 }
  53
  54 static void
  55 hisi_dma_write_reg(void *base, uint32_t off, uint32_t val)
  56 {
  57         rte_write32(rte_cpu_to_le_32(val),
  58                     (volatile void *)((char *)base + off));
  59 }
  60
  61 static void
  62 hisi_dma_write_dev(struct hisi_dma_dev *hw, uint32_t off, uint32_t val)
  63 {
  64         hisi_dma_write_reg(hw->io_base, off, val);
  65 }
  66
  67 static void
  68 hisi_dma_write_queue(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t val)
  69 {
  70         uint32_t off = hisi_dma_queue_base(hw) +
  71                         hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
  72         hisi_dma_write_dev(hw, off, val);
  73 }
  74
  75 static uint32_t
  76 hisi_dma_read_reg(void *base, uint32_t off)
  77 {
  78         uint32_t val = rte_read32((volatile void *)((char *)base + off));
  79         return rte_le_to_cpu_32(val);
  80 }
  81
  82 static uint32_t
  83 hisi_dma_read_dev(struct hisi_dma_dev *hw, uint32_t off)
  84 {
  85         return hisi_dma_read_reg(hw->io_base, off);
  86 }
  87
  88 static uint32_t
  89 hisi_dma_read_queue(struct hisi_dma_dev *hw, uint32_t qoff)
  90 {
  91         uint32_t off = hisi_dma_queue_base(hw) +
  92                         hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
  93         return hisi_dma_read_dev(hw, off);
  94 }
  95
  96 static void
  97 hisi_dma_update_bit(struct hisi_dma_dev *hw, uint32_t off, uint32_t pos,
  98                     bool set)
  99 {
 100         uint32_t tmp = hisi_dma_read_dev(hw, off);
 101         uint32_t mask = 1u << pos;
 102         tmp = set ? tmp | mask : tmp & ~mask;
 103         hisi_dma_write_dev(hw, off, tmp);
 104 }
 105
 106 static void
 107 hisi_dma_update_queue_bit(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t pos,
 108                           bool set)
 109 {
 110         uint32_t tmp = hisi_dma_read_queue(hw, qoff);
 111         uint32_t mask = 1u << pos;
 112         tmp = set ? tmp | mask : tmp & ~mask;
 113         hisi_dma_write_queue(hw, qoff, tmp);
 114 }
 115
 116 static void
 117 hisi_dma_update_queue_mbit(struct hisi_dma_dev *hw, uint32_t qoff,
 118                            uint32_t mask, bool set)
 119 {
 120         uint32_t tmp = hisi_dma_read_queue(hw, qoff);
 121         tmp = set ? tmp | mask : tmp & ~mask;
 122         hisi_dma_write_queue(hw, qoff, tmp);
 123 }
 124
 125 #define hisi_dma_poll_hw_state(hw, val, cond, sleep_us, timeout_us) ({ \
 126         uint32_t timeout = 0; \
 127         while (timeout++ <= (timeout_us)) { \
 128                 (val) = hisi_dma_read_queue(hw, HISI_DMA_QUEUE_FSM_REG); \
 129                 if (cond) \
 130                         break; \
 131                 rte_delay_us(sleep_us); \
 132         } \
 133         (cond) ? 0 : -ETIME; \
 134 })
 135
 136 static int
 137 hisi_dma_reset_hw(struct hisi_dma_dev *hw)
 138 {
 139 #define POLL_SLEEP_US   100
 140 #define POLL_TIMEOUT_US 10000
 141
 142         uint32_t tmp;
 143         int ret;
 144
 145         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 146                                   HISI_DMA_QUEUE_CTRL0_PAUSE_B, true);
 147         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 148                                   HISI_DMA_QUEUE_CTRL0_EN_B, false);
 149
 150         ret = hisi_dma_poll_hw_state(hw, tmp,
 151                 FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) != HISI_DMA_STATE_RUN,
 152                 POLL_SLEEP_US, POLL_TIMEOUT_US);
 153         if (ret) {
 154                 HISI_DMA_ERR(hw, "disable dma timeout!");
 155                 return ret;
 156         }
 157
 158         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL1_REG,
 159                                   HISI_DMA_QUEUE_CTRL1_RESET_B, true);
 160         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_TAIL_REG, 0);
 161         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_HEAD_REG, 0);
 162         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 163                                   HISI_DMA_QUEUE_CTRL0_PAUSE_B, false);
 164
 165         ret = hisi_dma_poll_hw_state(hw, tmp,
 166                 FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) == HISI_DMA_STATE_IDLE,
 167                 POLL_SLEEP_US, POLL_TIMEOUT_US);
 168         if (ret) {
 169                 HISI_DMA_ERR(hw, "reset dma timeout!");
 170                 return ret;
 171         }
 172
 173         return 0;
 174 }
 175
 176 static void
 177 hisi_dma_init_hw(struct hisi_dma_dev *hw)
 178 {
 179         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_BASE_L_REG,
 180                              lower_32_bits(hw->sqe_iova));
 181         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_BASE_H_REG,
 182                              upper_32_bits(hw->sqe_iova));
 183         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_BASE_L_REG,
 184                              lower_32_bits(hw->cqe_iova));
 185         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_BASE_H_REG,
 186                              upper_32_bits(hw->cqe_iova));
 187         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_DEPTH_REG,
 188                              hw->sq_depth_mask);
 189         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_DEPTH_REG, hw->cq_depth - 1);
 190         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_TAIL_REG, 0);
 191         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_HEAD_REG, 0);
 192         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM0_REG, 0);
 193         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM1_REG, 0);
 194         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM2_REG, 0);
 195
 196         if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
 197                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM3_REG,
 198                                      0);
 199                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM4_REG,
 200                                      0);
 201                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM5_REG,
 202                                      0);
 203                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM6_REG,
 204                                      0);
 205                 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 206                                 HISI_DMA_HIP08_QUEUE_CTRL0_ERR_ABORT_B, false);
 207                 hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_STATUS_REG,
 208                                 HISI_DMA_HIP08_QUEUE_INT_MASK_M, true);
 209                 hisi_dma_update_queue_mbit(hw,
 210                                 HISI_DMA_HIP08_QUEUE_INT_MASK_REG,
 211                                 HISI_DMA_HIP08_QUEUE_INT_MASK_M, true);
 212         }
 213 }
 214
 215 static void
 216 hisi_dma_init_gbl(void *pci_bar, uint8_t revision)
 217 {
 218         struct hisi_dma_dev hw;
 219
 220         memset(&hw, 0, sizeof(hw));
 221         hw.io_base = pci_bar;
 222
 223         if (revision == HISI_DMA_REVISION_HIP08B)
 224                 hisi_dma_update_bit(&hw, HISI_DMA_HIP08_MODE_REG,
 225                                     HISI_DMA_HIP08_MODE_SEL_B, true);
 226 }
 227
 228 static uint8_t
 229 hisi_dma_reg_layout(uint8_t revision)
 230 {
 231         if (revision == HISI_DMA_REVISION_HIP08B)
 232                 return HISI_DMA_REG_LAYOUT_HIP08;
 233         else
 234                 return HISI_DMA_REG_LAYOUT_INVALID;
 235 }
 236
 237 static void
 238 hisi_dma_zero_iomem(struct hisi_dma_dev *hw)
 239 {
 240         memset(hw->iomz->addr, 0, hw->iomz_sz);
 241 }
 242
 243 static int
 244 hisi_dma_alloc_iomem(struct hisi_dma_dev *hw, uint16_t ring_size,
 245                      const char *dev_name)
 246 {
 247         uint32_t sq_size = sizeof(struct hisi_dma_sqe) * ring_size;
 248         uint32_t cq_size = sizeof(struct hisi_dma_cqe) *
 249                            (ring_size + HISI_DMA_CQ_RESERVED);
 250         uint32_t status_size = sizeof(uint16_t) * ring_size;
 251         char mz_name[RTE_MEMZONE_NAMESIZE];
 252         const struct rte_memzone *iomz;
 253         uint32_t total_size;
 254
 255         sq_size = RTE_CACHE_LINE_ROUNDUP(sq_size);
 256         cq_size = RTE_CACHE_LINE_ROUNDUP(cq_size);
 257         status_size = RTE_CACHE_LINE_ROUNDUP(status_size);
 258         total_size = sq_size + cq_size + status_size;
 259
 260         (void)snprintf(mz_name, sizeof(mz_name), "hisi_dma:%s", dev_name);
 261         iomz = rte_memzone_reserve(mz_name, total_size, hw->data->numa_node,
 262                                    RTE_MEMZONE_IOVA_CONTIG);
 263         if (iomz == NULL) {
 264                 HISI_DMA_ERR(hw, "malloc %s iomem fail!", mz_name);
 265                 return -ENOMEM;
 266         }
 267
 268         hw->iomz = iomz;
 269         hw->iomz_sz = total_size;
 270         hw->sqe = iomz->addr;
 271         hw->cqe = (void *)((char *)iomz->addr + sq_size);
 272         hw->status = (void *)((char *)iomz->addr + sq_size + cq_size);
 273         hw->sqe_iova = iomz->iova;
 274         hw->cqe_iova = iomz->iova + sq_size;
 275         hw->sq_depth_mask = ring_size - 1;
 276         hw->cq_depth = ring_size + HISI_DMA_CQ_RESERVED;
 277         hisi_dma_zero_iomem(hw);
 278
 279         return 0;
 280 }
 281
 282 static void
 283 hisi_dma_free_iomem(struct hisi_dma_dev *hw)
 284 {
 285         if (hw->iomz != NULL)
 286                 rte_memzone_free(hw->iomz);
 287
 288         hw->iomz = NULL;
 289         hw->sqe = NULL;
 290         hw->cqe = NULL;
 291         hw->status = NULL;
 292         hw->sqe_iova = 0;
 293         hw->cqe_iova = 0;
 294         hw->sq_depth_mask = 0;
 295         hw->cq_depth = 0;
 296 }
 297
 298 static int
 299 hisi_dma_info_get(const struct rte_dma_dev *dev,
 300                   struct rte_dma_info *dev_info,
 301                   uint32_t info_sz)
 302 {
 303         RTE_SET_USED(dev);
 304         RTE_SET_USED(info_sz);
 305
 306         dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
 307                              RTE_DMA_CAPA_OPS_COPY;
 308         dev_info->max_vchans = 1;
 309         dev_info->max_desc = HISI_DMA_MAX_DESC_NUM;
 310         dev_info->min_desc = HISI_DMA_MIN_DESC_NUM;
 311
 312         return 0;
 313 }
 314
 315 static int
 316 hisi_dma_configure(struct rte_dma_dev *dev,
 317                    const struct rte_dma_conf *conf,
 318                    uint32_t conf_sz)
 319 {
 320         RTE_SET_USED(dev);
 321         RTE_SET_USED(conf);
 322         RTE_SET_USED(conf_sz);
 323         return 0;
 324 }
 325
 326 static int
 327 hisi_dma_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
 328                      const struct rte_dma_vchan_conf *conf,
 329                      uint32_t conf_sz)
 330 {
 331         struct hisi_dma_dev *hw = dev->data->dev_private;
 332         int ret;
 333
 334         RTE_SET_USED(vchan);
 335         RTE_SET_USED(conf_sz);
 336
 337         if (!rte_is_power_of_2(conf->nb_desc)) {
 338                 HISI_DMA_ERR(hw, "Number of desc must be power of 2!");
 339                 return -EINVAL;
 340         }
 341
 342         hisi_dma_free_iomem(hw);
 343         ret = hisi_dma_alloc_iomem(hw, conf->nb_desc, dev->data->dev_name);
 344         if (ret)
 345                 return ret;
 346
 347         return 0;
 348 }
 349
 350 static int
 351 hisi_dma_start(struct rte_dma_dev *dev)
 352 {
 353         struct hisi_dma_dev *hw = dev->data->dev_private;
 354
 355         if (hw->iomz == NULL) {
 356                 HISI_DMA_ERR(hw, "Vchan was not setup, start fail!\n");
 357                 return -EINVAL;
 358         }
 359
 360         /* Reset the dmadev to a known state, include:
 361          *   1) zero iomem, also include status fields.
 362          *   2) init hardware register.
 363          *   3) init index values to zero.
 364          *   4) init running statistics.
 365          */
 366         hisi_dma_zero_iomem(hw);
 367         hisi_dma_init_hw(hw);
 368         hw->ridx = 0;
 369         hw->cridx = 0;
 370         hw->sq_head = 0;
 371         hw->sq_tail = 0;
 372         hw->cq_sq_head = 0;
 373         hw->cq_head = 0;
 374         hw->cqs_completed = 0;
 375         hw->cqe_vld = 1;
 376         hw->submitted = 0;
 377         hw->completed = 0;
 378         hw->errors = 0;
 379
 380         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 381                                   HISI_DMA_QUEUE_CTRL0_EN_B, true);
 382
 383         return 0;
 384 }
 385
 386 static int
 387 hisi_dma_stop(struct rte_dma_dev *dev)
 388 {
 389         return hisi_dma_reset_hw(dev->data->dev_private);
 390 }
 391
 392 static int
 393 hisi_dma_close(struct rte_dma_dev *dev)
 394 {
 395         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 396                 /* The dmadev already stopped */
 397                 hisi_dma_free_iomem(dev->data->dev_private);
 398         }
 399         return 0;
 400 }
 401
 402 static int
 403 hisi_dma_stats_get(const struct rte_dma_dev *dev, uint16_t vchan,
 404                    struct rte_dma_stats *stats,
 405                    uint32_t stats_sz)
 406 {
 407         struct hisi_dma_dev *hw = dev->data->dev_private;
 408
 409         RTE_SET_USED(vchan);
 410         RTE_SET_USED(stats_sz);
 411         stats->submitted = hw->submitted;
 412         stats->completed = hw->completed;
 413         stats->errors = hw->errors;
 414
 415         return 0;
 416 }
 417
 418 static int
 419 hisi_dma_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
 420 {
 421         struct hisi_dma_dev *hw = dev->data->dev_private;
 422
 423         RTE_SET_USED(vchan);
 424         hw->submitted = 0;
 425         hw->completed = 0;
 426         hw->errors = 0;
 427
 428         return 0;
 429 }
 430
 431 static void
 432 hisi_dma_get_dump_range(struct hisi_dma_dev *hw, uint32_t *start, uint32_t *end)
 433 {
 434         if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
 435                 *start = HISI_DMA_HIP08_DUMP_START_REG;
 436                 *end = HISI_DMA_HIP08_DUMP_END_REG;
 437         } else {
 438                 *start = 0;
 439                 *end = 0;
 440         }
 441 }
 442
 443 static void
 444 hisi_dma_dump_common(struct hisi_dma_dev *hw, FILE *f)
 445 {
 446 #define DUMP_REGNUM_PER_LINE    4
 447
 448         uint32_t start, end;
 449         uint32_t cnt, i;
 450
 451         hisi_dma_get_dump_range(hw, &start, &end);
 452
 453         (void)fprintf(f, "    common-register:\n");
 454
 455         cnt = 0;
 456         for (i = start; i <= end; i += sizeof(uint32_t)) {
 457                 if (cnt % DUMP_REGNUM_PER_LINE == 0)
 458                         (void)fprintf(f, "      [%4x]:", i);
 459                 (void)fprintf(f, " 0x%08x", hisi_dma_read_dev(hw, i));
 460                 cnt++;
 461                 if (cnt % DUMP_REGNUM_PER_LINE == 0)
 462                         (void)fprintf(f, "\n");
 463         }
 464         if (cnt % DUMP_REGNUM_PER_LINE)
 465                 (void)fprintf(f, "\n");
 466 }
 467
 468 static void
 469 hisi_dma_dump_read_queue(struct hisi_dma_dev *hw, uint32_t qoff,
 470                          char *buffer, int max_sz)
 471 {
 472         memset(buffer, 0, max_sz);
 473
 474         /* Address-related registers are not printed for security reasons. */
 475         if (qoff == HISI_DMA_QUEUE_SQ_BASE_L_REG ||
 476             qoff == HISI_DMA_QUEUE_SQ_BASE_H_REG ||
 477             qoff == HISI_DMA_QUEUE_CQ_BASE_L_REG ||
 478             qoff == HISI_DMA_QUEUE_CQ_BASE_H_REG) {
 479                 (void)snprintf(buffer, max_sz, "**********");
 480                 return;
 481         }
 482
 483         (void)snprintf(buffer, max_sz, "0x%08x", hisi_dma_read_queue(hw, qoff));
 484 }
 485
 486 static void
 487 hisi_dma_dump_queue(struct hisi_dma_dev *hw, FILE *f)
 488 {
 489 #define REG_FMT_LEN     32
 490         char buf[REG_FMT_LEN] = { 0 };
 491         uint32_t i;
 492
 493         (void)fprintf(f, "    queue-register:\n");
 494         for (i = 0; i < HISI_DMA_QUEUE_REGION_SIZE; ) {
 495                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
 496                 (void)fprintf(f, "      [%2x]: %s", i, buf);
 497                 i += sizeof(uint32_t);
 498                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
 499                 (void)fprintf(f, " %s", buf);
 500                 i += sizeof(uint32_t);
 501                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
 502                 (void)fprintf(f, " %s", buf);
 503                 i += sizeof(uint32_t);
 504                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
 505                 (void)fprintf(f, " %s\n", buf);
 506                 i += sizeof(uint32_t);
 507         }
 508 }
 509
 510 static int
 511 hisi_dma_dump(const struct rte_dma_dev *dev, FILE *f)
 512 {
 513         struct hisi_dma_dev *hw = dev->data->dev_private;
 514
 515         (void)fprintf(f,
 516                 "    revision: 0x%x queue_id: %u ring_size: %u\n"
 517                 "    ridx: %u cridx: %u\n"
 518                 "    sq_head: %u sq_tail: %u cq_sq_head: %u\n"
 519                 "    cq_head: %u cqs_completed: %u cqe_vld: %u\n"
 520                 "    submitted: %" PRIu64 " completed: %" PRIu64 " errors %"
 521                 PRIu64"\n",
 522                 hw->revision, hw->queue_id,
 523                 hw->sq_depth_mask > 0 ? hw->sq_depth_mask + 1 : 0,
 524                 hw->ridx, hw->cridx,
 525                 hw->sq_head, hw->sq_tail, hw->cq_sq_head,
 526                 hw->cq_head, hw->cqs_completed, hw->cqe_vld,
 527                 hw->submitted, hw->completed, hw->errors);
 528         hisi_dma_dump_queue(hw, f);
 529         hisi_dma_dump_common(hw, f);
 530
 531         return 0;
 532 }
 533
 534 static int
 535 hisi_dma_copy(void *dev_private, uint16_t vchan,
 536                  rte_iova_t src, rte_iova_t dst,
 537                  uint32_t length, uint64_t flags)
 538 {
 539         struct hisi_dma_dev *hw = dev_private;
 540         struct hisi_dma_sqe *sqe = &hw->sqe[hw->sq_tail];
 541
 542         RTE_SET_USED(vchan);
 543
 544         if (((hw->sq_tail + 1) & hw->sq_depth_mask) == hw->sq_head)
 545                 return -ENOSPC;
 546
 547         sqe->dw0 = rte_cpu_to_le_32(SQE_OPCODE_M2M);
 548         sqe->dw1 = 0;
 549         sqe->dw2 = 0;
 550         sqe->length = rte_cpu_to_le_32(length);
 551         sqe->src_addr = rte_cpu_to_le_64(src);
 552         sqe->dst_addr = rte_cpu_to_le_64(dst);
 553         hw->sq_tail = (hw->sq_tail + 1) & hw->sq_depth_mask;
 554         hw->submitted++;
 555
 556         if (flags & RTE_DMA_OP_FLAG_FENCE)
 557                 sqe->dw0 |= rte_cpu_to_le_32(SQE_FENCE_FLAG);
 558         if (flags & RTE_DMA_OP_FLAG_SUBMIT)
 559                 rte_write32(rte_cpu_to_le_32(hw->sq_tail), hw->sq_tail_reg);
 560
 561         return hw->ridx++;
 562 }
 563
 564 static int
 565 hisi_dma_submit(void *dev_private, uint16_t vchan)
 566 {
 567         struct hisi_dma_dev *hw = dev_private;
 568
 569         RTE_SET_USED(vchan);
 570         rte_write32(rte_cpu_to_le_32(hw->sq_tail), hw->sq_tail_reg);
 571
 572         return 0;
 573 }
 574
 575 static inline void
 576 hisi_dma_scan_cq(struct hisi_dma_dev *hw)
 577 {
 578         volatile struct hisi_dma_cqe *cqe;
 579         uint16_t csq_head = hw->cq_sq_head;
 580         uint16_t cq_head = hw->cq_head;
 581         uint16_t count = 0;
 582         uint64_t misc;
 583
 584         while (true) {
 585                 cqe = &hw->cqe[cq_head];
 586                 misc = cqe->misc;
 587                 misc = rte_le_to_cpu_64(misc);
 588                 if (FIELD_GET(CQE_VALID_B, misc) != hw->cqe_vld)
 589                         break;
 590
 591                 csq_head = FIELD_GET(CQE_SQ_HEAD_MASK, misc);
 592                 if (unlikely(misc & CQE_STATUS_MASK))
 593                         hw->status[csq_head] = FIELD_GET(CQE_STATUS_MASK,
 594                                                          misc);
 595
 596                 count++;
 597                 cq_head++;
 598                 if (cq_head == hw->cq_depth) {
 599                         hw->cqe_vld = !hw->cqe_vld;
 600                         cq_head = 0;
 601                 }
 602         }
 603
 604         if (count == 0)
 605                 return;
 606
 607         hw->cq_head = cq_head;
 608         hw->cq_sq_head = (csq_head + 1) & hw->sq_depth_mask;
 609         hw->cqs_completed += count;
 610         if (hw->cqs_completed >= HISI_DMA_CQ_RESERVED) {
 611                 rte_write32(rte_cpu_to_le_32(cq_head), hw->cq_head_reg);
 612                 hw->cqs_completed = 0;
 613         }
 614 }
 615
 616 static inline uint16_t
 617 hisi_dma_calc_cpls(struct hisi_dma_dev *hw, const uint16_t nb_cpls)
 618 {
 619         uint16_t cpl_num;
 620
 621         if (hw->cq_sq_head >= hw->sq_head)
 622                 cpl_num = hw->cq_sq_head - hw->sq_head;
 623         else
 624                 cpl_num = hw->sq_depth_mask + 1 - hw->sq_head + hw->cq_sq_head;
 625
 626         if (cpl_num > nb_cpls)
 627                 cpl_num = nb_cpls;
 628
 629         return cpl_num;
 630 }
 631
 632 static uint16_t
 633 hisi_dma_completed(void *dev_private,
 634                    uint16_t vchan, const uint16_t nb_cpls,
 635                    uint16_t *last_idx, bool *has_error)
 636 {
 637         struct hisi_dma_dev *hw = dev_private;
 638         uint16_t sq_head = hw->sq_head;
 639         uint16_t cpl_num, i;
 640
 641         RTE_SET_USED(vchan);
 642         hisi_dma_scan_cq(hw);
 643
 644         cpl_num = hisi_dma_calc_cpls(hw, nb_cpls);
 645         for (i = 0; i < cpl_num; i++) {
 646                 if (hw->status[sq_head]) {
 647                         *has_error = true;
 648                         break;
 649                 }
 650                 sq_head = (sq_head + 1) & hw->sq_depth_mask;
 651         }
 652         if (i > 0) {
 653                 hw->cridx += i;
 654                 *last_idx = hw->cridx - 1;
 655                 hw->sq_head = sq_head;
 656         }
 657         hw->completed += i;
 658
 659         return i;
 660 }
 661
 662 static enum rte_dma_status_code
 663 hisi_dma_convert_status(uint16_t status)
 664 {
 665         switch (status) {
 666         case HISI_DMA_STATUS_SUCCESS:
 667                 return RTE_DMA_STATUS_SUCCESSFUL;
 668         case HISI_DMA_STATUS_INVALID_OPCODE:
 669                 return RTE_DMA_STATUS_INVALID_OPCODE;
 670         case HISI_DMA_STATUS_INVALID_LENGTH:
 671                 return RTE_DMA_STATUS_INVALID_LENGTH;
 672         case HISI_DMA_STATUS_USER_ABORT:
 673                 return RTE_DMA_STATUS_USER_ABORT;
 674         case HISI_DMA_STATUS_REMOTE_READ_ERROR:
 675         case HISI_DMA_STATUS_AXI_READ_ERROR:
 676                 return RTE_DMA_STATUS_BUS_READ_ERROR;
 677         case HISI_DMA_STATUS_AXI_WRITE_ERROR:
 678                 return RTE_DMA_STATUS_BUS_WRITE_ERROR;
 679         case HISI_DMA_STATUS_DATA_POISON:
 680         case HISI_DMA_STATUS_REMOTE_DATA_POISION:
 681                 return RTE_DMA_STATUS_DATA_POISION;
 682         case HISI_DMA_STATUS_SQE_READ_ERROR:
 683         case HISI_DMA_STATUS_SQE_READ_POISION:
 684                 return RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR;
 685         case HISI_DMA_STATUS_LINK_DOWN_ERROR:
 686                 return RTE_DMA_STATUS_DEV_LINK_ERROR;
 687         default:
 688                 return RTE_DMA_STATUS_ERROR_UNKNOWN;
 689         }
 690 }
 691
 692 static uint16_t
 693 hisi_dma_completed_status(void *dev_private,
 694                           uint16_t vchan, const uint16_t nb_cpls,
 695                           uint16_t *last_idx, enum rte_dma_status_code *status)
 696 {
 697         struct hisi_dma_dev *hw = dev_private;
 698         uint16_t sq_head = hw->sq_head;
 699         uint16_t cpl_num, i;
 700
 701         RTE_SET_USED(vchan);
 702         hisi_dma_scan_cq(hw);
 703
 704         cpl_num = hisi_dma_calc_cpls(hw, nb_cpls);
 705         for (i = 0; i < cpl_num; i++) {
 706                 status[i] = hisi_dma_convert_status(hw->status[sq_head]);
 707                 hw->errors += !!status[i];
 708                 hw->status[sq_head] = HISI_DMA_STATUS_SUCCESS;
 709                 sq_head = (sq_head + 1) & hw->sq_depth_mask;
 710         }
 711         if (likely(cpl_num > 0)) {
 712                 hw->cridx += cpl_num;
 713                 *last_idx = hw->cridx - 1;
 714                 hw->sq_head = sq_head;
 715         }
 716         hw->completed += cpl_num;
 717
 718         return cpl_num;
 719 }
 720
 721 static uint16_t
 722 hisi_dma_burst_capacity(const void *dev_private, uint16_t vchan)
 723 {
 724         const struct hisi_dma_dev *hw = dev_private;
 725         uint16_t sq_head = hw->sq_head;
 726         uint16_t sq_tail = hw->sq_tail;
 727
 728         RTE_SET_USED(vchan);
 729
 730         return (sq_tail >= sq_head) ? hw->sq_depth_mask - sq_tail + sq_head :
 731                                       sq_head - 1 - sq_tail;
 732 }
 733
 734 static void
 735 hisi_dma_gen_pci_device_name(const struct rte_pci_device *pci_dev,
 736                              char *name, size_t size)
 737 {
 738         memset(name, 0, size);
 739         (void)snprintf(name, size, "%x:%x.%x",
 740                  pci_dev->addr.bus, pci_dev->addr.devid,
 741                  pci_dev->addr.function);
 742 }
 743
 744 static void
 745 hisi_dma_gen_dev_name(const struct rte_pci_device *pci_dev,
 746                       uint8_t queue_id, char *name, size_t size)
 747 {
 748         memset(name, 0, size);
 749         (void)snprintf(name, size, "%x:%x.%x-ch%u",
 750                  pci_dev->addr.bus, pci_dev->addr.devid,
 751                  pci_dev->addr.function, queue_id);
 752 }
 753
 754 /**
 755  * Hardware queue state machine:
 756  *
 757  *   -----------  dmadev_create   ------------------
 758  *   | Unknown | ---------------> |      IDLE      |
 759  *   -----------                  ------------------
 760  *                                   ^          |
 761  *                                   |          |dev_start
 762  *                           dev_stop|          |
 763  *                                   |          v
 764  *                                ------------------
 765  *                                |      RUN       |
 766  *                                ------------------
 767  *
 768  */
 769 static const struct rte_dma_dev_ops hisi_dmadev_ops = {
 770         .dev_info_get     = hisi_dma_info_get,
 771         .dev_configure    = hisi_dma_configure,
 772         .dev_start        = hisi_dma_start,
 773         .dev_stop         = hisi_dma_stop,
 774         .dev_close        = hisi_dma_close,
 775         .vchan_setup      = hisi_dma_vchan_setup,
 776         .stats_get        = hisi_dma_stats_get,
 777         .stats_reset      = hisi_dma_stats_reset,
 778         .dev_dump         = hisi_dma_dump,
 779 };
 780
 781 static int
 782 hisi_dma_create(struct rte_pci_device *pci_dev, uint8_t queue_id,
 783                 uint8_t revision)
 784 {
 785 #define REG_PCI_BAR_INDEX       2
 786
 787         char name[RTE_DEV_NAME_MAX_LEN];
 788         struct rte_dma_dev *dev;
 789         struct hisi_dma_dev *hw;
 790         int ret;
 791
 792         hisi_dma_gen_dev_name(pci_dev, queue_id, name, sizeof(name));
 793         dev = rte_dma_pmd_allocate(name, pci_dev->device.numa_node,
 794                                    sizeof(*hw));
 795         if (dev == NULL) {
 796                 HISI_DMA_LOG(ERR, "%s allocate dmadev fail!", name);
 797                 return -EINVAL;
 798         }
 799
 800         dev->device = &pci_dev->device;
 801         dev->dev_ops = &hisi_dmadev_ops;
 802         dev->fp_obj->dev_private = dev->data->dev_private;
 803         dev->fp_obj->copy = hisi_dma_copy;
 804         dev->fp_obj->submit = hisi_dma_submit;
 805         dev->fp_obj->completed = hisi_dma_completed;
 806         dev->fp_obj->completed_status = hisi_dma_completed_status;
 807         dev->fp_obj->burst_capacity = hisi_dma_burst_capacity;
 808
 809         hw = dev->data->dev_private;
 810         hw->data = dev->data;
 811         hw->revision = revision;
 812         hw->reg_layout = hisi_dma_reg_layout(revision);
 813         hw->io_base = pci_dev->mem_resource[REG_PCI_BAR_INDEX].addr;
 814         hw->queue_id = queue_id;
 815         hw->sq_tail_reg = hisi_dma_queue_regaddr(hw,
 816                                                  HISI_DMA_QUEUE_SQ_TAIL_REG);
 817         hw->cq_head_reg = hisi_dma_queue_regaddr(hw,
 818                                                  HISI_DMA_QUEUE_CQ_HEAD_REG);
 819
 820         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 821                 ret = hisi_dma_reset_hw(hw);
 822                 if (ret) {
 823                         HISI_DMA_LOG(ERR, "%s init device fail!", name);
 824                         (void)rte_dma_pmd_release(name);
 825                         return -EIO;
 826                 }
 827         }
 828
 829         dev->state = RTE_DMA_DEV_READY;
 830         HISI_DMA_LOG(DEBUG, "%s create dmadev success!", name);
 831
 832         return 0;
 833 }
 834
 835 static int
 836 hisi_dma_check_revision(struct rte_pci_device *pci_dev, const char *name,
 837                         uint8_t *out_revision)
 838 {
 839         uint8_t revision;
 840         int ret;
 841
 842         ret = rte_pci_read_config(pci_dev, &revision, 1,
 843                                   HISI_DMA_PCI_REVISION_ID_REG);
 844         if (ret != 1) {
 845                 HISI_DMA_LOG(ERR, "%s read PCI revision failed!", name);
 846                 return -EINVAL;
 847         }
 848         if (hisi_dma_reg_layout(revision) == HISI_DMA_REG_LAYOUT_INVALID) {
 849                 HISI_DMA_LOG(ERR, "%s revision: 0x%x not supported!",
 850                              name, revision);
 851                 return -EINVAL;
 852         }
 853
 854         *out_revision = revision;
 855         return 0;
 856 }
 857
 858 static int
 859 hisi_dma_probe(struct rte_pci_driver *pci_drv __rte_unused,
 860                struct rte_pci_device *pci_dev)
 861 {
 862         char name[RTE_DEV_NAME_MAX_LEN] = { 0 };
 863         uint8_t revision;
 864         uint8_t i;
 865         int ret;
 866
 867         hisi_dma_gen_pci_device_name(pci_dev, name, sizeof(name));
 868
 869         if (pci_dev->mem_resource[2].addr == NULL) {
 870                 HISI_DMA_LOG(ERR, "%s BAR2 is NULL!\n", name);
 871                 return -ENODEV;
 872         }
 873
 874         ret = hisi_dma_check_revision(pci_dev, name, &revision);
 875         if (ret)
 876                 return ret;
 877         HISI_DMA_LOG(DEBUG, "%s read PCI revision: 0x%x", name, revision);
 878
 879         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 880                 hisi_dma_init_gbl(pci_dev->mem_resource[2].addr, revision);
 881
 882         for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
 883                 ret = hisi_dma_create(pci_dev, i, revision);
 884                 if (ret) {
 885                         HISI_DMA_LOG(ERR, "%s create dmadev %u failed!",
 886                                      name, i);
 887                         break;
 888                 }
 889         }
 890
 891         return ret;
 892 }
 893
 894 static int
 895 hisi_dma_remove(struct rte_pci_device *pci_dev)
 896 {
 897         char name[RTE_DEV_NAME_MAX_LEN];
 898         uint8_t i;
 899         int ret;
 900
 901         for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
 902                 hisi_dma_gen_dev_name(pci_dev, i, name, sizeof(name));
 903                 ret = rte_dma_pmd_release(name);
 904                 if (ret)
 905                         return ret;
 906         }
 907
 908         return 0;
 909 }
 910
 911 static const struct rte_pci_id pci_id_hisi_dma_map[] = {
 912         { RTE_PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HISI_DMA_DEVICE_ID) },
 913         { .vendor_id = 0, }, /* sentinel */
 914 };
 915
 916 static struct rte_pci_driver hisi_dma_pmd_drv = {
 917         .id_table  = pci_id_hisi_dma_map,
 918         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 919         .probe     = hisi_dma_probe,
 920         .remove    = hisi_dma_remove,
 921 };
 922
 923 RTE_PMD_REGISTER_PCI(dma_hisilicon, hisi_dma_pmd_drv);
 924 RTE_PMD_REGISTER_PCI_TABLE(dma_hisilicon, pci_id_hisi_dma_map);
 925 RTE_PMD_REGISTER_KMOD_DEP(dma_hisilicon, "vfio-pci");