drivers/dma/hisilicon/hisi_dmadev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 HiSilicon Limited
   3  */
   4
   5 #include <inttypes.h>
   6 #include <string.h>
   7
   8 #include <rte_bus_pci.h>
   9 #include <rte_cycles.h>
  10 #include <rte_eal.h>
  11 #include <rte_io.h>
  12 #include <rte_log.h>
  13 #include <rte_malloc.h>
  14 #include <rte_memzone.h>
  15 #include <rte_pci.h>
  16 #include <rte_dmadev_pmd.h>
  17
  18 #include "hisi_dmadev.h"
  19
  20 RTE_LOG_REGISTER_DEFAULT(hisi_dma_logtype, INFO);
  21 #define HISI_DMA_LOG(level, fmt, args...) \
  22                 rte_log(RTE_LOG_ ## level, hisi_dma_logtype, \
  23                 "%s(): " fmt "\n", __func__, ##args)
  24 #define HISI_DMA_LOG_RAW(hw, level, fmt, args...) \
  25                 rte_log(RTE_LOG_ ## level, hisi_dma_logtype, \
  26                 "%s %s(): " fmt "\n", (hw)->data->dev_name, \
  27                 __func__, ##args)
  28 #define HISI_DMA_DEBUG(hw, fmt, args...) \
  29                 HISI_DMA_LOG_RAW(hw, DEBUG, fmt, ## args)
  30 #define HISI_DMA_INFO(hw, fmt, args...) \
  31                 HISI_DMA_LOG_RAW(hw, INFO, fmt, ## args)
  32 #define HISI_DMA_WARN(hw, fmt, args...) \
  33                 HISI_DMA_LOG_RAW(hw, WARNING, fmt, ## args)
  34 #define HISI_DMA_ERR(hw, fmt, args...) \
  35                 HISI_DMA_LOG_RAW(hw, ERR, fmt, ## args)
  36
  37 static uint32_t
  38 hisi_dma_queue_base(struct hisi_dma_dev *hw)
  39 {
  40         if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
  41                 return HISI_DMA_HIP08_QUEUE_BASE;
  42         else if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP09)
  43                 return HISI_DMA_HIP09_QUEUE_BASE;
  44         else
  45                 return 0;
  46 }
  47
  48 static volatile void *
  49 hisi_dma_queue_regaddr(struct hisi_dma_dev *hw, uint32_t qoff)
  50 {
  51         uint32_t off = hisi_dma_queue_base(hw) +
  52                         hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
  53         return (volatile void *)((char *)hw->io_base + off);
  54 }
  55
  56 static void
  57 hisi_dma_write_reg(void *base, uint32_t off, uint32_t val)
  58 {
  59         rte_write32(rte_cpu_to_le_32(val),
  60                     (volatile void *)((char *)base + off));
  61 }
  62
  63 static void
  64 hisi_dma_write_dev(struct hisi_dma_dev *hw, uint32_t off, uint32_t val)
  65 {
  66         hisi_dma_write_reg(hw->io_base, off, val);
  67 }
  68
  69 static void
  70 hisi_dma_write_queue(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t val)
  71 {
  72         uint32_t off = hisi_dma_queue_base(hw) +
  73                         hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
  74         hisi_dma_write_dev(hw, off, val);
  75 }
  76
  77 static uint32_t
  78 hisi_dma_read_reg(void *base, uint32_t off)
  79 {
  80         uint32_t val = rte_read32((volatile void *)((char *)base + off));
  81         return rte_le_to_cpu_32(val);
  82 }
  83
  84 static uint32_t
  85 hisi_dma_read_dev(struct hisi_dma_dev *hw, uint32_t off)
  86 {
  87         return hisi_dma_read_reg(hw->io_base, off);
  88 }
  89
  90 static uint32_t
  91 hisi_dma_read_queue(struct hisi_dma_dev *hw, uint32_t qoff)
  92 {
  93         uint32_t off = hisi_dma_queue_base(hw) +
  94                         hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
  95         return hisi_dma_read_dev(hw, off);
  96 }
  97
  98 static void
  99 hisi_dma_update_bit(struct hisi_dma_dev *hw, uint32_t off, uint32_t pos,
 100                     bool set)
 101 {
 102         uint32_t tmp = hisi_dma_read_dev(hw, off);
 103         uint32_t mask = 1u << pos;
 104         tmp = set ? tmp | mask : tmp & ~mask;
 105         hisi_dma_write_dev(hw, off, tmp);
 106 }
 107
 108 static void
 109 hisi_dma_update_queue_bit(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t pos,
 110                           bool set)
 111 {
 112         uint32_t tmp = hisi_dma_read_queue(hw, qoff);
 113         uint32_t mask = 1u << pos;
 114         tmp = set ? tmp | mask : tmp & ~mask;
 115         hisi_dma_write_queue(hw, qoff, tmp);
 116 }
 117
 118 static void
 119 hisi_dma_update_queue_mbit(struct hisi_dma_dev *hw, uint32_t qoff,
 120                            uint32_t mask, bool set)
 121 {
 122         uint32_t tmp = hisi_dma_read_queue(hw, qoff);
 123         tmp = set ? tmp | mask : tmp & ~mask;
 124         hisi_dma_write_queue(hw, qoff, tmp);
 125 }
 126
 127 #define hisi_dma_poll_hw_state(hw, val, cond, sleep_us, timeout_us) ({ \
 128         uint32_t timeout = 0; \
 129         while (timeout++ <= (timeout_us)) { \
 130                 (val) = hisi_dma_read_queue(hw, HISI_DMA_QUEUE_FSM_REG); \
 131                 if (cond) \
 132                         break; \
 133                 rte_delay_us(sleep_us); \
 134         } \
 135         (cond) ? 0 : -ETIME; \
 136 })
 137
 138 static int
 139 hisi_dma_reset_hw(struct hisi_dma_dev *hw)
 140 {
 141 #define POLL_SLEEP_US   100
 142 #define POLL_TIMEOUT_US 10000
 143
 144         uint32_t tmp;
 145         int ret;
 146
 147         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 148                                   HISI_DMA_QUEUE_CTRL0_PAUSE_B, true);
 149         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 150                                   HISI_DMA_QUEUE_CTRL0_EN_B, false);
 151
 152         ret = hisi_dma_poll_hw_state(hw, tmp,
 153                 FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) != HISI_DMA_STATE_RUN,
 154                 POLL_SLEEP_US, POLL_TIMEOUT_US);
 155         if (ret) {
 156                 HISI_DMA_ERR(hw, "disable dma timeout!");
 157                 return ret;
 158         }
 159
 160         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL1_REG,
 161                                   HISI_DMA_QUEUE_CTRL1_RESET_B, true);
 162         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_TAIL_REG, 0);
 163         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_HEAD_REG, 0);
 164         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 165                                   HISI_DMA_QUEUE_CTRL0_PAUSE_B, false);
 166
 167         ret = hisi_dma_poll_hw_state(hw, tmp,
 168                 FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) == HISI_DMA_STATE_IDLE,
 169                 POLL_SLEEP_US, POLL_TIMEOUT_US);
 170         if (ret) {
 171                 HISI_DMA_ERR(hw, "reset dma timeout!");
 172                 return ret;
 173         }
 174
 175         return 0;
 176 }
 177
 178 static void
 179 hisi_dma_init_common(struct hisi_dma_dev *hw)
 180 {
 181         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_BASE_L_REG,
 182                              lower_32_bits(hw->sqe_iova));
 183         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_BASE_H_REG,
 184                              upper_32_bits(hw->sqe_iova));
 185         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_BASE_L_REG,
 186                              lower_32_bits(hw->cqe_iova));
 187         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_BASE_H_REG,
 188                              upper_32_bits(hw->cqe_iova));
 189         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_DEPTH_REG,
 190                              hw->sq_depth_mask);
 191         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_DEPTH_REG, hw->cq_depth - 1);
 192         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_TAIL_REG, 0);
 193         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_HEAD_REG, 0);
 194         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM0_REG, 0);
 195         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM1_REG, 0);
 196         hisi_dma_write_queue(hw, HISI_DMA_QUEUE_ERR_INT_NUM2_REG, 0);
 197 }
 198
 199 static void
 200 hisi_dma_init_hw(struct hisi_dma_dev *hw)
 201 {
 202         hisi_dma_init_common(hw);
 203
 204         if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
 205                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM3_REG,
 206                                      0);
 207                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM4_REG,
 208                                      0);
 209                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM5_REG,
 210                                      0);
 211                 hisi_dma_write_queue(hw, HISI_DMA_HIP08_QUEUE_ERR_INT_NUM6_REG,
 212                                      0);
 213                 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 214                                 HISI_DMA_HIP08_QUEUE_CTRL0_ERR_ABORT_B, false);
 215                 hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_STATUS_REG,
 216                                 HISI_DMA_HIP08_QUEUE_INT_MASK_M, true);
 217                 hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_MASK_REG,
 218                                 HISI_DMA_HIP08_QUEUE_INT_MASK_M, true);
 219         } else if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) {
 220                 hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 221                                 HISI_DMA_HIP09_QUEUE_CTRL0_ERR_ABORT_M, false);
 222                 hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_STATUS_REG,
 223                                 HISI_DMA_HIP09_QUEUE_INT_MASK_M, true);
 224                 hisi_dma_update_queue_mbit(hw, HISI_DMA_QUEUE_INT_MASK_REG,
 225                                 HISI_DMA_HIP09_QUEUE_INT_MASK_M, true);
 226                 hisi_dma_update_queue_mbit(hw,
 227                                 HISI_DMA_HIP09_QUEUE_ERR_INT_STATUS_REG,
 228                                 HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_M, true);
 229                 hisi_dma_update_queue_mbit(hw,
 230                                 HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_REG,
 231                                 HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_M, true);
 232                 hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL1_REG,
 233                                 HISI_DMA_HIP09_QUEUE_CTRL1_VA_ENABLE_B, true);
 234                 hisi_dma_update_bit(hw,
 235                                 HISI_DMA_HIP09_QUEUE_CFG_REG(hw->queue_id),
 236                                 HISI_DMA_HIP09_QUEUE_CFG_LINK_DOWN_MASK_B,
 237                                 true);
 238         }
 239 }
 240
 241 static void
 242 hisi_dma_init_gbl(void *pci_bar, uint8_t revision)
 243 {
 244         struct hisi_dma_dev hw;
 245
 246         memset(&hw, 0, sizeof(hw));
 247         hw.io_base = pci_bar;
 248
 249         if (revision == HISI_DMA_REVISION_HIP08B)
 250                 hisi_dma_update_bit(&hw, HISI_DMA_HIP08_MODE_REG,
 251                                     HISI_DMA_HIP08_MODE_SEL_B, true);
 252 }
 253
 254 static uint8_t
 255 hisi_dma_reg_layout(uint8_t revision)
 256 {
 257         if (revision == HISI_DMA_REVISION_HIP08B)
 258                 return HISI_DMA_REG_LAYOUT_HIP08;
 259         else if (revision >= HISI_DMA_REVISION_HIP09A)
 260                 return HISI_DMA_REG_LAYOUT_HIP09;
 261         else
 262                 return HISI_DMA_REG_LAYOUT_INVALID;
 263 }
 264
 265 static void
 266 hisi_dma_zero_iomem(struct hisi_dma_dev *hw)
 267 {
 268         memset(hw->iomz->addr, 0, hw->iomz_sz);
 269 }
 270
 271 static int
 272 hisi_dma_alloc_iomem(struct hisi_dma_dev *hw, uint16_t ring_size,
 273                      const char *dev_name)
 274 {
 275         uint32_t sq_size = sizeof(struct hisi_dma_sqe) * ring_size;
 276         uint32_t cq_size = sizeof(struct hisi_dma_cqe) *
 277                            (ring_size + HISI_DMA_CQ_RESERVED);
 278         uint32_t status_size = sizeof(uint16_t) * ring_size;
 279         char mz_name[RTE_MEMZONE_NAMESIZE];
 280         const struct rte_memzone *iomz;
 281         uint32_t total_size;
 282
 283         sq_size = RTE_CACHE_LINE_ROUNDUP(sq_size);
 284         cq_size = RTE_CACHE_LINE_ROUNDUP(cq_size);
 285         status_size = RTE_CACHE_LINE_ROUNDUP(status_size);
 286         total_size = sq_size + cq_size + status_size;
 287
 288         (void)snprintf(mz_name, sizeof(mz_name), "hisi_dma:%s", dev_name);
 289         iomz = rte_memzone_reserve(mz_name, total_size, hw->data->numa_node,
 290                                    RTE_MEMZONE_IOVA_CONTIG);
 291         if (iomz == NULL) {
 292                 HISI_DMA_ERR(hw, "malloc %s iomem fail!", mz_name);
 293                 return -ENOMEM;
 294         }
 295
 296         hw->iomz = iomz;
 297         hw->iomz_sz = total_size;
 298         hw->sqe = iomz->addr;
 299         hw->cqe = (void *)((char *)iomz->addr + sq_size);
 300         hw->status = (void *)((char *)iomz->addr + sq_size + cq_size);
 301         hw->sqe_iova = iomz->iova;
 302         hw->cqe_iova = iomz->iova + sq_size;
 303         hw->sq_depth_mask = ring_size - 1;
 304         hw->cq_depth = ring_size + HISI_DMA_CQ_RESERVED;
 305         hisi_dma_zero_iomem(hw);
 306
 307         return 0;
 308 }
 309
 310 static void
 311 hisi_dma_free_iomem(struct hisi_dma_dev *hw)
 312 {
 313         if (hw->iomz != NULL)
 314                 rte_memzone_free(hw->iomz);
 315
 316         hw->iomz = NULL;
 317         hw->sqe = NULL;
 318         hw->cqe = NULL;
 319         hw->status = NULL;
 320         hw->sqe_iova = 0;
 321         hw->cqe_iova = 0;
 322         hw->sq_depth_mask = 0;
 323         hw->cq_depth = 0;
 324 }
 325
 326 static int
 327 hisi_dma_info_get(const struct rte_dma_dev *dev,
 328                   struct rte_dma_info *dev_info,
 329                   uint32_t info_sz)
 330 {
 331         struct hisi_dma_dev *hw = dev->data->dev_private;
 332         RTE_SET_USED(info_sz);
 333
 334         dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
 335                              RTE_DMA_CAPA_OPS_COPY;
 336         if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP09)
 337                 dev_info->dev_capa |= RTE_DMA_CAPA_HANDLES_ERRORS;
 338
 339         dev_info->max_vchans = 1;
 340         dev_info->max_desc = HISI_DMA_MAX_DESC_NUM;
 341         dev_info->min_desc = HISI_DMA_MIN_DESC_NUM;
 342
 343         return 0;
 344 }
 345
 346 static int
 347 hisi_dma_configure(struct rte_dma_dev *dev,
 348                    const struct rte_dma_conf *conf,
 349                    uint32_t conf_sz)
 350 {
 351         RTE_SET_USED(dev);
 352         RTE_SET_USED(conf);
 353         RTE_SET_USED(conf_sz);
 354         return 0;
 355 }
 356
 357 static int
 358 hisi_dma_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
 359                      const struct rte_dma_vchan_conf *conf,
 360                      uint32_t conf_sz)
 361 {
 362         struct hisi_dma_dev *hw = dev->data->dev_private;
 363         int ret;
 364
 365         RTE_SET_USED(vchan);
 366         RTE_SET_USED(conf_sz);
 367
 368         if (!rte_is_power_of_2(conf->nb_desc)) {
 369                 HISI_DMA_ERR(hw, "Number of desc must be power of 2!");
 370                 return -EINVAL;
 371         }
 372
 373         hisi_dma_free_iomem(hw);
 374         ret = hisi_dma_alloc_iomem(hw, conf->nb_desc, dev->data->dev_name);
 375         if (ret)
 376                 return ret;
 377
 378         return 0;
 379 }
 380
 381 static int
 382 hisi_dma_start(struct rte_dma_dev *dev)
 383 {
 384         struct hisi_dma_dev *hw = dev->data->dev_private;
 385
 386         if (hw->iomz == NULL) {
 387                 HISI_DMA_ERR(hw, "Vchan was not setup, start fail!\n");
 388                 return -EINVAL;
 389         }
 390
 391         /* Reset the dmadev to a known state, include:
 392          *   1) zero iomem, also include status fields.
 393          *   2) init hardware register.
 394          *   3) init index values to zero.
 395          *   4) init running statistics.
 396          */
 397         hisi_dma_zero_iomem(hw);
 398         hisi_dma_init_hw(hw);
 399         hw->ridx = 0;
 400         hw->cridx = 0;
 401         hw->sq_head = 0;
 402         hw->sq_tail = 0;
 403         hw->cq_sq_head = 0;
 404         hw->cq_head = 0;
 405         hw->cqs_completed = 0;
 406         hw->cqe_vld = 1;
 407         hw->submitted = 0;
 408         hw->completed = 0;
 409         hw->errors = 0;
 410         hw->qfulls = 0;
 411
 412         hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
 413                                   HISI_DMA_QUEUE_CTRL0_EN_B, true);
 414
 415         return 0;
 416 }
 417
 418 static int
 419 hisi_dma_stop(struct rte_dma_dev *dev)
 420 {
 421         return hisi_dma_reset_hw(dev->data->dev_private);
 422 }
 423
 424 static int
 425 hisi_dma_close(struct rte_dma_dev *dev)
 426 {
 427         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 428                 /* The dmadev already stopped */
 429                 hisi_dma_free_iomem(dev->data->dev_private);
 430         }
 431         return 0;
 432 }
 433
 434 static int
 435 hisi_dma_stats_get(const struct rte_dma_dev *dev, uint16_t vchan,
 436                    struct rte_dma_stats *stats,
 437                    uint32_t stats_sz)
 438 {
 439         struct hisi_dma_dev *hw = dev->data->dev_private;
 440
 441         RTE_SET_USED(vchan);
 442         RTE_SET_USED(stats_sz);
 443         stats->submitted = hw->submitted;
 444         stats->completed = hw->completed;
 445         stats->errors = hw->errors;
 446
 447         return 0;
 448 }
 449
 450 static int
 451 hisi_dma_stats_reset(struct rte_dma_dev *dev, uint16_t vchan)
 452 {
 453         struct hisi_dma_dev *hw = dev->data->dev_private;
 454
 455         RTE_SET_USED(vchan);
 456         hw->submitted = 0;
 457         hw->completed = 0;
 458         hw->errors = 0;
 459         hw->qfulls = 0;
 460
 461         return 0;
 462 }
 463
 464 static int
 465 hisi_dma_vchan_status(const struct rte_dma_dev *dev, uint16_t vchan,
 466                       enum rte_dma_vchan_status *status)
 467 {
 468         struct hisi_dma_dev *hw = dev->data->dev_private;
 469         uint32_t val;
 470
 471         RTE_SET_USED(vchan);
 472
 473         val = hisi_dma_read_queue(hw, HISI_DMA_QUEUE_FSM_REG);
 474         val = FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, val);
 475         if (val == HISI_DMA_STATE_RUN)
 476                 *status = RTE_DMA_VCHAN_ACTIVE;
 477         else if (val == HISI_DMA_STATE_CPL)
 478                 *status = RTE_DMA_VCHAN_IDLE;
 479         else
 480                 *status = RTE_DMA_VCHAN_HALTED_ERROR;
 481
 482         return 0;
 483 }
 484
 485 static void
 486 hisi_dma_dump_range(struct hisi_dma_dev *hw, FILE *f, uint32_t start,
 487                     uint32_t end)
 488 {
 489 #define DUMP_REGNUM_PER_LINE    4
 490
 491         uint32_t cnt, i;
 492
 493         cnt = 0;
 494         for (i = start; i <= end; i += sizeof(uint32_t)) {
 495                 if (cnt % DUMP_REGNUM_PER_LINE == 0)
 496                         (void)fprintf(f, "      [%4x]:", i);
 497                 (void)fprintf(f, " 0x%08x", hisi_dma_read_dev(hw, i));
 498                 cnt++;
 499                 if (cnt % DUMP_REGNUM_PER_LINE == 0)
 500                         (void)fprintf(f, "\n");
 501         }
 502         if (cnt % DUMP_REGNUM_PER_LINE)
 503                 (void)fprintf(f, "\n");
 504 }
 505
 506 static void
 507 hisi_dma_dump_common(struct hisi_dma_dev *hw, FILE *f)
 508 {
 509         struct {
 510                 uint8_t reg_layout;
 511                 uint32_t start;
 512                 uint32_t end;
 513         } reg_info[] = {
 514                 { HISI_DMA_REG_LAYOUT_HIP08,
 515                   HISI_DMA_HIP08_DUMP_START_REG,
 516                   HISI_DMA_HIP08_DUMP_END_REG },
 517                 { HISI_DMA_REG_LAYOUT_HIP09,
 518                   HISI_DMA_HIP09_DUMP_REGION_A_START_REG,
 519                   HISI_DMA_HIP09_DUMP_REGION_A_END_REG },
 520                 { HISI_DMA_REG_LAYOUT_HIP09,
 521                   HISI_DMA_HIP09_DUMP_REGION_B_START_REG,
 522                   HISI_DMA_HIP09_DUMP_REGION_B_END_REG },
 523                 { HISI_DMA_REG_LAYOUT_HIP09,
 524                   HISI_DMA_HIP09_DUMP_REGION_C_START_REG,
 525                   HISI_DMA_HIP09_DUMP_REGION_C_END_REG },
 526                 { HISI_DMA_REG_LAYOUT_HIP09,
 527                   HISI_DMA_HIP09_DUMP_REGION_D_START_REG,
 528                   HISI_DMA_HIP09_DUMP_REGION_D_END_REG },
 529         };
 530         uint32_t i;
 531
 532         (void)fprintf(f, "    common-register:\n");
 533         for (i = 0; i < RTE_DIM(reg_info); i++) {
 534                 if (hw->reg_layout != reg_info[i].reg_layout)
 535                         continue;
 536                 hisi_dma_dump_range(hw, f, reg_info[i].start, reg_info[i].end);
 537         }
 538 }
 539
 540 static void
 541 hisi_dma_dump_read_queue(struct hisi_dma_dev *hw, uint32_t qoff,
 542                          char *buffer, int max_sz)
 543 {
 544         memset(buffer, 0, max_sz);
 545
 546         /* Address-related registers are not printed for security reasons. */
 547         if (qoff == HISI_DMA_QUEUE_SQ_BASE_L_REG ||
 548             qoff == HISI_DMA_QUEUE_SQ_BASE_H_REG ||
 549             qoff == HISI_DMA_QUEUE_CQ_BASE_L_REG ||
 550             qoff == HISI_DMA_QUEUE_CQ_BASE_H_REG) {
 551                 (void)snprintf(buffer, max_sz, "**********");
 552                 return;
 553         }
 554
 555         (void)snprintf(buffer, max_sz, "0x%08x", hisi_dma_read_queue(hw, qoff));
 556 }
 557
 558 static void
 559 hisi_dma_dump_queue(struct hisi_dma_dev *hw, FILE *f)
 560 {
 561 #define REG_FMT_LEN     32
 562         char buf[REG_FMT_LEN] = { 0 };
 563         uint32_t i;
 564
 565         (void)fprintf(f, "    queue-register:\n");
 566         for (i = 0; i < HISI_DMA_QUEUE_REGION_SIZE; ) {
 567                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
 568                 (void)fprintf(f, "      [%2x]: %s", i, buf);
 569                 i += sizeof(uint32_t);
 570                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
 571                 (void)fprintf(f, " %s", buf);
 572                 i += sizeof(uint32_t);
 573                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
 574                 (void)fprintf(f, " %s", buf);
 575                 i += sizeof(uint32_t);
 576                 hisi_dma_dump_read_queue(hw, i, buf, sizeof(buf));
 577                 (void)fprintf(f, " %s\n", buf);
 578                 i += sizeof(uint32_t);
 579         }
 580 }
 581
 582 static int
 583 hisi_dma_dump(const struct rte_dma_dev *dev, FILE *f)
 584 {
 585         struct hisi_dma_dev *hw = dev->data->dev_private;
 586
 587         (void)fprintf(f,
 588                 "    revision: 0x%x queue_id: %u ring_size: %u\n"
 589                 "    ridx: %u cridx: %u\n"
 590                 "    sq_head: %u sq_tail: %u cq_sq_head: %u\n"
 591                 "    cq_head: %u cqs_completed: %u cqe_vld: %u\n"
 592                 "    submitted: %" PRIu64 " completed: %" PRIu64 " errors: %"
 593                 PRIu64 " qfulls: %" PRIu64 "\n",
 594                 hw->revision, hw->queue_id,
 595                 hw->sq_depth_mask > 0 ? hw->sq_depth_mask + 1 : 0,
 596                 hw->ridx, hw->cridx,
 597                 hw->sq_head, hw->sq_tail, hw->cq_sq_head,
 598                 hw->cq_head, hw->cqs_completed, hw->cqe_vld,
 599                 hw->submitted, hw->completed, hw->errors, hw->qfulls);
 600         hisi_dma_dump_queue(hw, f);
 601         hisi_dma_dump_common(hw, f);
 602
 603         return 0;
 604 }
 605
 606 static int
 607 hisi_dma_copy(void *dev_private, uint16_t vchan,
 608                  rte_iova_t src, rte_iova_t dst,
 609                  uint32_t length, uint64_t flags)
 610 {
 611         struct hisi_dma_dev *hw = dev_private;
 612         struct hisi_dma_sqe *sqe = &hw->sqe[hw->sq_tail];
 613
 614         RTE_SET_USED(vchan);
 615
 616         if (((hw->sq_tail + 1) & hw->sq_depth_mask) == hw->sq_head) {
 617                 hw->qfulls++;
 618                 return -ENOSPC;
 619         }
 620
 621         sqe->dw0 = rte_cpu_to_le_32(SQE_OPCODE_M2M);
 622         sqe->dw1 = 0;
 623         sqe->dw2 = 0;
 624         sqe->length = rte_cpu_to_le_32(length);
 625         sqe->src_addr = rte_cpu_to_le_64(src);
 626         sqe->dst_addr = rte_cpu_to_le_64(dst);
 627         hw->sq_tail = (hw->sq_tail + 1) & hw->sq_depth_mask;
 628         hw->submitted++;
 629
 630         if (flags & RTE_DMA_OP_FLAG_FENCE)
 631                 sqe->dw0 |= rte_cpu_to_le_32(SQE_FENCE_FLAG);
 632         if (flags & RTE_DMA_OP_FLAG_SUBMIT)
 633                 rte_write32(rte_cpu_to_le_32(hw->sq_tail), hw->sq_tail_reg);
 634
 635         return hw->ridx++;
 636 }
 637
 638 static int
 639 hisi_dma_submit(void *dev_private, uint16_t vchan)
 640 {
 641         struct hisi_dma_dev *hw = dev_private;
 642
 643         RTE_SET_USED(vchan);
 644         rte_write32(rte_cpu_to_le_32(hw->sq_tail), hw->sq_tail_reg);
 645
 646         return 0;
 647 }
 648
 649 static inline void
 650 hisi_dma_scan_cq(struct hisi_dma_dev *hw)
 651 {
 652         volatile struct hisi_dma_cqe *cqe;
 653         uint16_t csq_head = hw->cq_sq_head;
 654         uint16_t cq_head = hw->cq_head;
 655         uint16_t count = 0;
 656         uint64_t misc;
 657
 658         while (count < hw->cq_depth) {
 659                 cqe = &hw->cqe[cq_head];
 660                 misc = cqe->misc;
 661                 misc = rte_le_to_cpu_64(misc);
 662                 if (FIELD_GET(CQE_VALID_B, misc) != hw->cqe_vld)
 663                         break;
 664
 665                 csq_head = FIELD_GET(CQE_SQ_HEAD_MASK, misc);
 666                 if (unlikely(csq_head > hw->sq_depth_mask)) {
 667                         /**
 668                          * Defensive programming to prevent overflow of the
 669                          * status array indexed by csq_head. Only error logs
 670                          * are used for prompting.
 671                          */
 672                         HISI_DMA_ERR(hw, "invalid csq_head:%u!\n", csq_head);
 673                         count = 0;
 674                         break;
 675                 }
 676                 if (unlikely(misc & CQE_STATUS_MASK))
 677                         hw->status[csq_head] = FIELD_GET(CQE_STATUS_MASK,
 678                                                          misc);
 679
 680                 count++;
 681                 cq_head++;
 682                 if (cq_head == hw->cq_depth) {
 683                         hw->cqe_vld = !hw->cqe_vld;
 684                         cq_head = 0;
 685                 }
 686         }
 687
 688         if (count == 0)
 689                 return;
 690
 691         hw->cq_head = cq_head;
 692         hw->cq_sq_head = (csq_head + 1) & hw->sq_depth_mask;
 693         hw->cqs_completed += count;
 694         if (hw->cqs_completed >= HISI_DMA_CQ_RESERVED) {
 695                 rte_write32(rte_cpu_to_le_32(cq_head), hw->cq_head_reg);
 696                 hw->cqs_completed = 0;
 697         }
 698 }
 699
 700 static inline uint16_t
 701 hisi_dma_calc_cpls(struct hisi_dma_dev *hw, const uint16_t nb_cpls)
 702 {
 703         uint16_t cpl_num;
 704
 705         if (hw->cq_sq_head >= hw->sq_head)
 706                 cpl_num = hw->cq_sq_head - hw->sq_head;
 707         else
 708                 cpl_num = hw->sq_depth_mask + 1 - hw->sq_head + hw->cq_sq_head;
 709
 710         if (cpl_num > nb_cpls)
 711                 cpl_num = nb_cpls;
 712
 713         return cpl_num;
 714 }
 715
 716 static uint16_t
 717 hisi_dma_completed(void *dev_private,
 718                    uint16_t vchan, const uint16_t nb_cpls,
 719                    uint16_t *last_idx, bool *has_error)
 720 {
 721         struct hisi_dma_dev *hw = dev_private;
 722         uint16_t sq_head = hw->sq_head;
 723         uint16_t cpl_num, i;
 724
 725         RTE_SET_USED(vchan);
 726         hisi_dma_scan_cq(hw);
 727
 728         cpl_num = hisi_dma_calc_cpls(hw, nb_cpls);
 729         for (i = 0; i < cpl_num; i++) {
 730                 if (hw->status[sq_head]) {
 731                         *has_error = true;
 732                         break;
 733                 }
 734                 sq_head = (sq_head + 1) & hw->sq_depth_mask;
 735         }
 736         *last_idx = hw->cridx + i - 1;
 737         if (i > 0) {
 738                 hw->cridx += i;
 739                 hw->sq_head = sq_head;
 740                 hw->completed += i;
 741         }
 742
 743         return i;
 744 }
 745
 746 static enum rte_dma_status_code
 747 hisi_dma_convert_status(uint16_t status)
 748 {
 749         switch (status) {
 750         case HISI_DMA_STATUS_SUCCESS:
 751                 return RTE_DMA_STATUS_SUCCESSFUL;
 752         case HISI_DMA_STATUS_INVALID_OPCODE:
 753                 return RTE_DMA_STATUS_INVALID_OPCODE;
 754         case HISI_DMA_STATUS_INVALID_LENGTH:
 755                 return RTE_DMA_STATUS_INVALID_LENGTH;
 756         case HISI_DMA_STATUS_USER_ABORT:
 757                 return RTE_DMA_STATUS_USER_ABORT;
 758         case HISI_DMA_STATUS_REMOTE_READ_ERROR:
 759         case HISI_DMA_STATUS_AXI_READ_ERROR:
 760                 return RTE_DMA_STATUS_BUS_READ_ERROR;
 761         case HISI_DMA_STATUS_AXI_WRITE_ERROR:
 762                 return RTE_DMA_STATUS_BUS_WRITE_ERROR;
 763         case HISI_DMA_STATUS_DATA_POISON:
 764         case HISI_DMA_STATUS_REMOTE_DATA_POISION:
 765                 return RTE_DMA_STATUS_DATA_POISION;
 766         case HISI_DMA_STATUS_SQE_READ_ERROR:
 767         case HISI_DMA_STATUS_SQE_READ_POISION:
 768                 return RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR;
 769         case HISI_DMA_STATUS_LINK_DOWN_ERROR:
 770                 return RTE_DMA_STATUS_DEV_LINK_ERROR;
 771         default:
 772                 return RTE_DMA_STATUS_ERROR_UNKNOWN;
 773         }
 774 }
 775
 776 static uint16_t
 777 hisi_dma_completed_status(void *dev_private,
 778                           uint16_t vchan, const uint16_t nb_cpls,
 779                           uint16_t *last_idx, enum rte_dma_status_code *status)
 780 {
 781         struct hisi_dma_dev *hw = dev_private;
 782         uint16_t sq_head = hw->sq_head;
 783         uint16_t cpl_num, i;
 784
 785         RTE_SET_USED(vchan);
 786         hisi_dma_scan_cq(hw);
 787
 788         cpl_num = hisi_dma_calc_cpls(hw, nb_cpls);
 789         for (i = 0; i < cpl_num; i++) {
 790                 status[i] = hisi_dma_convert_status(hw->status[sq_head]);
 791                 hw->errors += !!status[i];
 792                 hw->status[sq_head] = HISI_DMA_STATUS_SUCCESS;
 793                 sq_head = (sq_head + 1) & hw->sq_depth_mask;
 794         }
 795         *last_idx = hw->cridx + cpl_num - 1;
 796         if (likely(cpl_num > 0)) {
 797                 hw->cridx += cpl_num;
 798                 hw->sq_head = sq_head;
 799                 hw->completed += cpl_num;
 800         }
 801
 802         return cpl_num;
 803 }
 804
 805 static uint16_t
 806 hisi_dma_burst_capacity(const void *dev_private, uint16_t vchan)
 807 {
 808         const struct hisi_dma_dev *hw = dev_private;
 809         uint16_t sq_head = hw->sq_head;
 810         uint16_t sq_tail = hw->sq_tail;
 811
 812         RTE_SET_USED(vchan);
 813
 814         return (sq_tail >= sq_head) ? hw->sq_depth_mask - sq_tail + sq_head :
 815                                       sq_head - 1 - sq_tail;
 816 }
 817
 818 static void
 819 hisi_dma_gen_dev_name(const struct rte_pci_device *pci_dev,
 820                       uint8_t queue_id, char *dev_name, size_t size)
 821 {
 822         char name[RTE_DEV_NAME_MAX_LEN] = { 0 };
 823
 824         memset(dev_name, 0, size);
 825         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 826         (void)snprintf(dev_name, size, "%s-ch%u", name, queue_id);
 827 }
 828
 829 /**
 830  * Hardware queue state machine:
 831  *
 832  *   -----------  dmadev_create   ------------------
 833  *   | Unknown | ---------------> |      IDLE      |
 834  *   -----------                  ------------------
 835  *                                   ^          |
 836  *                                   |          |dev_start
 837  *                           dev_stop|          |
 838  *                                   |          v
 839  *                                ------------------
 840  *                                |      CPL       |
 841  *                                ------------------
 842  *                                   ^          |
 843  *                      hardware     |          |
 844  *                      completed all|          |dev_submit
 845  *                      descriptors  |          |
 846  *                                   |          |
 847  *                                ------------------
 848  *                                |      RUN       |
 849  *                                ------------------
 850  *
 851  */
 852 static const struct rte_dma_dev_ops hisi_dmadev_ops = {
 853         .dev_info_get     = hisi_dma_info_get,
 854         .dev_configure    = hisi_dma_configure,
 855         .dev_start        = hisi_dma_start,
 856         .dev_stop         = hisi_dma_stop,
 857         .dev_close        = hisi_dma_close,
 858         .vchan_setup      = hisi_dma_vchan_setup,
 859         .stats_get        = hisi_dma_stats_get,
 860         .stats_reset      = hisi_dma_stats_reset,
 861         .vchan_status     = hisi_dma_vchan_status,
 862         .dev_dump         = hisi_dma_dump,
 863 };
 864
 865 static int
 866 hisi_dma_create(struct rte_pci_device *pci_dev, uint8_t queue_id,
 867                 uint8_t revision)
 868 {
 869 #define REG_PCI_BAR_INDEX       2
 870
 871         char name[RTE_DEV_NAME_MAX_LEN];
 872         struct rte_dma_dev *dev;
 873         struct hisi_dma_dev *hw;
 874         int ret;
 875
 876         hisi_dma_gen_dev_name(pci_dev, queue_id, name, sizeof(name));
 877         dev = rte_dma_pmd_allocate(name, pci_dev->device.numa_node,
 878                                    sizeof(*hw));
 879         if (dev == NULL) {
 880                 HISI_DMA_LOG(ERR, "%s allocate dmadev fail!", name);
 881                 return -EINVAL;
 882         }
 883
 884         dev->device = &pci_dev->device;
 885         dev->dev_ops = &hisi_dmadev_ops;
 886         dev->fp_obj->dev_private = dev->data->dev_private;
 887         dev->fp_obj->copy = hisi_dma_copy;
 888         dev->fp_obj->submit = hisi_dma_submit;
 889         dev->fp_obj->completed = hisi_dma_completed;
 890         dev->fp_obj->completed_status = hisi_dma_completed_status;
 891         dev->fp_obj->burst_capacity = hisi_dma_burst_capacity;
 892
 893         hw = dev->data->dev_private;
 894         hw->data = dev->data;
 895         hw->revision = revision;
 896         hw->reg_layout = hisi_dma_reg_layout(revision);
 897         hw->io_base = pci_dev->mem_resource[REG_PCI_BAR_INDEX].addr;
 898         hw->queue_id = queue_id;
 899         hw->sq_tail_reg = hisi_dma_queue_regaddr(hw,
 900                                                  HISI_DMA_QUEUE_SQ_TAIL_REG);
 901         hw->cq_head_reg = hisi_dma_queue_regaddr(hw,
 902                                                  HISI_DMA_QUEUE_CQ_HEAD_REG);
 903
 904         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 905                 ret = hisi_dma_reset_hw(hw);
 906                 if (ret) {
 907                         HISI_DMA_LOG(ERR, "%s init device fail!", name);
 908                         (void)rte_dma_pmd_release(name);
 909                         return -EIO;
 910                 }
 911         }
 912
 913         dev->state = RTE_DMA_DEV_READY;
 914         HISI_DMA_LOG(DEBUG, "%s create dmadev success!", name);
 915
 916         return 0;
 917 }
 918
 919 static int
 920 hisi_dma_check_revision(struct rte_pci_device *pci_dev, const char *name,
 921                         uint8_t *out_revision)
 922 {
 923         uint8_t revision;
 924         int ret;
 925
 926         ret = rte_pci_read_config(pci_dev, &revision, 1,
 927                                   HISI_DMA_PCI_REVISION_ID_REG);
 928         if (ret != 1) {
 929                 HISI_DMA_LOG(ERR, "%s read PCI revision failed!", name);
 930                 return -EINVAL;
 931         }
 932         if (hisi_dma_reg_layout(revision) == HISI_DMA_REG_LAYOUT_INVALID) {
 933                 HISI_DMA_LOG(ERR, "%s revision: 0x%x not supported!",
 934                              name, revision);
 935                 return -EINVAL;
 936         }
 937
 938         *out_revision = revision;
 939         return 0;
 940 }
 941
 942 static int
 943 hisi_dma_probe(struct rte_pci_driver *pci_drv __rte_unused,
 944                struct rte_pci_device *pci_dev)
 945 {
 946         char name[RTE_DEV_NAME_MAX_LEN] = { 0 };
 947         uint8_t revision;
 948         uint8_t i;
 949         int ret;
 950
 951         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 952
 953         if (pci_dev->mem_resource[2].addr == NULL) {
 954                 HISI_DMA_LOG(ERR, "%s BAR2 is NULL!\n", name);
 955                 return -ENODEV;
 956         }
 957
 958         ret = hisi_dma_check_revision(pci_dev, name, &revision);
 959         if (ret)
 960                 return ret;
 961         HISI_DMA_LOG(DEBUG, "%s read PCI revision: 0x%x", name, revision);
 962
 963         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 964                 hisi_dma_init_gbl(pci_dev->mem_resource[2].addr, revision);
 965
 966         for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
 967                 ret = hisi_dma_create(pci_dev, i, revision);
 968                 if (ret) {
 969                         HISI_DMA_LOG(ERR, "%s create dmadev %u failed!",
 970                                      name, i);
 971                         break;
 972                 }
 973         }
 974
 975         return ret;
 976 }
 977
 978 static int
 979 hisi_dma_remove(struct rte_pci_device *pci_dev)
 980 {
 981         char name[RTE_DEV_NAME_MAX_LEN];
 982         uint8_t i;
 983         int ret;
 984
 985         for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
 986                 hisi_dma_gen_dev_name(pci_dev, i, name, sizeof(name));
 987                 ret = rte_dma_pmd_release(name);
 988                 if (ret)
 989                         return ret;
 990         }
 991
 992         return 0;
 993 }
 994
 995 static const struct rte_pci_id pci_id_hisi_dma_map[] = {
 996         { RTE_PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, HISI_DMA_DEVICE_ID) },
 997         { .vendor_id = 0, }, /* sentinel */
 998 };
 999
1000 static struct rte_pci_driver hisi_dma_pmd_drv = {
1001         .id_table  = pci_id_hisi_dma_map,
1002         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1003         .probe     = hisi_dma_probe,
1004         .remove    = hisi_dma_remove,
1005 };
1006
1007 RTE_PMD_REGISTER_PCI(dma_hisilicon, hisi_dma_pmd_drv);
1008 RTE_PMD_REGISTER_PCI_TABLE(dma_hisilicon, pci_id_hisi_dma_map);
1009 RTE_PMD_REGISTER_KMOD_DEP(dma_hisilicon, "vfio-pci");