drivers/dma/hisilicon/hisi_dmadev.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2021 HiSilicon Limited
   3  */
   4
   5 #ifndef HISI_DMADEV_H
   6 #define HISI_DMADEV_H
   7
   8 #include <rte_byteorder.h>
   9 #include <rte_common.h>
  10
  11 #define BIT(x)  (1ul << (x))
  12 #define BITS_PER_LONG   (__SIZEOF_LONG__ * 8)
  13 #define GENMASK(h, l) \
  14                 (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
  15 #define BF_SHF(x) (__builtin_ffsll(x) - 1)
  16 #define FIELD_GET(mask, reg) \
  17                 ((typeof(mask))(((reg) & (mask)) >> BF_SHF(mask)))
  18
  19 #define lower_32_bits(x) ((uint32_t)(x))
  20 #define upper_32_bits(x) ((uint32_t)(((x) >> 16) >> 16))
  21
  22 #define PCI_VENDOR_ID_HUAWEI                    0x19e5
  23 #define HISI_DMA_DEVICE_ID                      0xA122
  24 #define HISI_DMA_PCI_REVISION_ID_REG            0x08
  25 #define HISI_DMA_REVISION_HIP08B                0x21
  26 #define HISI_DMA_REVISION_HIP09A                0x30
  27
  28 #define HISI_DMA_MAX_HW_QUEUES                  4
  29 #define HISI_DMA_MAX_DESC_NUM                   8192
  30 #define HISI_DMA_MIN_DESC_NUM                   32
  31
  32 /**
  33  * The HIP08B(HiSilicon IP08) and HIP09B(HiSilicon IP09) are DMA iEPs, they
  34  * have the same pci device id but different pci revision.
  35  * Unfortunately, they have different register layouts, so two layout
  36  * enumerations are defined.
  37  */
  38 enum {
  39         HISI_DMA_REG_LAYOUT_INVALID = 0,
  40         HISI_DMA_REG_LAYOUT_HIP08,
  41         HISI_DMA_REG_LAYOUT_HIP09
  42 };
  43
  44 /**
  45  * Hardware PCI bar register MAP:
  46  *
  47  *     --------------
  48  *     | Misc-reg-0 |
  49  *     |            |
  50  *     --------------   -> Queue base
  51  *     |            |
  52  *     | Queue-0    |
  53  *     |            |
  54  *     --------------   ---
  55  *     |            |    ^
  56  *     | Queue-1    |   Queue region
  57  *     |            |    v
  58  *     --------------   ---
  59  *     | ...        |
  60  *     | Queue-x    |
  61  *     | ...        |
  62  *     --------------
  63  *     | Misc-reg-1 |
  64  *     --------------
  65  *
  66  * As described above, a single queue register is continuous and occupies the
  67  * length of queue-region. The global offset for a single queue register is
  68  * calculated by:
  69  *     offset = queue-base + (queue-id * queue-region) + reg-offset-in-region.
  70  *
  71  * The first part of queue region is basically the same for HIP08 and HIP09
  72  * register layouts, therefore, HISI_QUEUE_* registers are defined for it.
  73  */
  74 #define HISI_DMA_QUEUE_SQ_BASE_L_REG            0x0
  75 #define HISI_DMA_QUEUE_SQ_BASE_H_REG            0x4
  76 #define HISI_DMA_QUEUE_SQ_DEPTH_REG             0x8
  77 #define HISI_DMA_QUEUE_SQ_TAIL_REG              0xC
  78 #define HISI_DMA_QUEUE_CQ_BASE_L_REG            0x10
  79 #define HISI_DMA_QUEUE_CQ_BASE_H_REG            0x14
  80 #define HISI_DMA_QUEUE_CQ_DEPTH_REG             0x18
  81 #define HISI_DMA_QUEUE_CQ_HEAD_REG              0x1C
  82 #define HISI_DMA_QUEUE_CTRL0_REG                0x20
  83 #define HISI_DMA_QUEUE_CTRL0_EN_B               0
  84 #define HISI_DMA_QUEUE_CTRL0_PAUSE_B            4
  85 #define HISI_DMA_QUEUE_CTRL1_REG                0x24
  86 #define HISI_DMA_QUEUE_CTRL1_RESET_B            0
  87 #define HISI_DMA_QUEUE_FSM_REG                  0x30
  88 #define HISI_DMA_QUEUE_FSM_STS_M                GENMASK(3, 0)
  89 #define HISI_DMA_QUEUE_INT_STATUS_REG           0x40
  90 #define HISI_DMA_QUEUE_INT_MASK_REG             0x44
  91 #define HISI_DMA_QUEUE_ERR_INT_NUM0_REG         0x84
  92 #define HISI_DMA_QUEUE_ERR_INT_NUM1_REG         0x88
  93 #define HISI_DMA_QUEUE_ERR_INT_NUM2_REG         0x8C
  94 #define HISI_DMA_QUEUE_REGION_SIZE              0x100
  95
  96 /**
  97  * HiSilicon IP08 DMA register and field define:
  98  */
  99 #define HISI_DMA_HIP08_QUEUE_BASE                       0x0
 100 #define HISI_DMA_HIP08_QUEUE_CTRL0_ERR_ABORT_B          2
 101 #define HISI_DMA_HIP08_QUEUE_INT_MASK_M                 GENMASK(14, 0)
 102 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM3_REG           0x90
 103 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM4_REG           0x94
 104 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM5_REG           0x98
 105 #define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM6_REG           0x48
 106 #define HISI_DMA_HIP08_MODE_REG                         0x217C
 107 #define HISI_DMA_HIP08_MODE_SEL_B                       0
 108 #define HISI_DMA_HIP08_DUMP_START_REG                   0x2000
 109 #define HISI_DMA_HIP08_DUMP_END_REG                     0x2280
 110
 111 /**
 112  * HiSilicon IP09 DMA register and field define:
 113  */
 114 #define HISI_DMA_HIP09_QUEUE_BASE                       0x2000
 115 #define HISI_DMA_HIP09_QUEUE_CTRL0_ERR_ABORT_M          GENMASK(31, 28)
 116 #define HISI_DMA_HIP09_QUEUE_CTRL1_VA_ENABLE_B          2
 117 #define HISI_DMA_HIP09_QUEUE_INT_MASK_M                 0x1
 118 #define HISI_DMA_HIP09_QUEUE_ERR_INT_STATUS_REG         0x48
 119 #define HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_REG           0x4C
 120 #define HISI_DMA_HIP09_QUEUE_ERR_INT_MASK_M             GENMASK(18, 1)
 121 #define HISI_DMA_HIP09_QUEUE_CFG_REG(queue_id)          (0x800 + \
 122                                                          (queue_id) * 0x20)
 123 #define HISI_DMA_HIP09_QUEUE_CFG_LINK_DOWN_MASK_B       16
 124 #define HISI_DMA_HIP09_DUMP_REGION_A_START_REG          0x0
 125 #define HISI_DMA_HIP09_DUMP_REGION_A_END_REG            0x368
 126 #define HISI_DMA_HIP09_DUMP_REGION_B_START_REG          0x800
 127 #define HISI_DMA_HIP09_DUMP_REGION_B_END_REG            0xA08
 128 #define HISI_DMA_HIP09_DUMP_REGION_C_START_REG          0x1800
 129 #define HISI_DMA_HIP09_DUMP_REGION_C_END_REG            0x1A4C
 130 #define HISI_DMA_HIP09_DUMP_REGION_D_START_REG          0x1C00
 131 #define HISI_DMA_HIP09_DUMP_REGION_D_END_REG            0x1CC4
 132
 133 /**
 134  * In fact, there are multiple states, but it need to pay attention to
 135  * the following three states for the driver:
 136  */
 137 enum {
 138         HISI_DMA_STATE_IDLE = 0,
 139         HISI_DMA_STATE_RUN,
 140         /**
 141          * All of the submitted descriptor are finished, and the queue
 142          * is waiting for new descriptors.
 143          */
 144         HISI_DMA_STATE_CPL,
 145 };
 146
 147 /**
 148  * Hardware complete status define:
 149  */
 150 #define HISI_DMA_STATUS_SUCCESS                 0x0
 151 #define HISI_DMA_STATUS_INVALID_OPCODE          0x1
 152 #define HISI_DMA_STATUS_INVALID_LENGTH          0x2
 153 #define HISI_DMA_STATUS_USER_ABORT              0x4
 154 #define HISI_DMA_STATUS_REMOTE_READ_ERROR       0x10
 155 #define HISI_DMA_STATUS_AXI_READ_ERROR          0x20
 156 #define HISI_DMA_STATUS_AXI_WRITE_ERROR         0x40
 157 #define HISI_DMA_STATUS_DATA_POISON             0x80
 158 #define HISI_DMA_STATUS_SQE_READ_ERROR          0x100
 159 #define HISI_DMA_STATUS_SQE_READ_POISION        0x200
 160 #define HISI_DMA_STATUS_REMOTE_DATA_POISION     0x400
 161 #define HISI_DMA_STATUS_LINK_DOWN_ERROR         0x800
 162
 163 /**
 164  * After scanning the CQ array, the CQ head register needs to be updated.
 165  * Updating the register involves write memory barrier operations.
 166  * Here use the following method to reduce WMB operations:
 167  *   a) malloc more CQEs, which correspond to the macro HISI_DMA_CQ_RESERVED.
 168  *   b) update the CQ head register after accumulated number of completed CQs
 169  *      is greater than or equal to HISI_DMA_CQ_RESERVED.
 170  */
 171 #define HISI_DMA_CQ_RESERVED            64
 172
 173 struct hisi_dma_sqe {
 174         uint32_t dw0;
 175 #define SQE_FENCE_FLAG  BIT(10)
 176 #define SQE_OPCODE_M2M  0x4
 177         uint32_t dw1;
 178         uint32_t dw2;
 179         uint32_t length;
 180         uint64_t src_addr;
 181         uint64_t dst_addr;
 182 };
 183
 184 struct hisi_dma_cqe {
 185         uint64_t rsv;
 186         uint64_t misc;
 187 #define CQE_SQ_HEAD_MASK        GENMASK(15, 0)
 188 #define CQE_VALID_B             BIT(48)
 189 #define CQE_STATUS_MASK         GENMASK(63, 49)
 190 };
 191
 192 struct hisi_dma_dev {
 193         struct hisi_dma_sqe *sqe;
 194         volatile struct hisi_dma_cqe *cqe;
 195         uint16_t *status; /* the completion status array of SQEs. */
 196
 197         volatile void *sq_tail_reg; /**< register address for doorbell. */
 198         volatile void *cq_head_reg; /**< register address for answer CQ. */
 199
 200         uint16_t sq_depth_mask; /**< SQ depth - 1, the SQ depth is power of 2 */
 201         uint16_t cq_depth; /* CQ depth */
 202
 203         uint16_t ridx; /**< ring index which will assign to the next request. */
 204         /** ring index which returned by hisi_dmadev_completed APIs. */
 205         uint16_t cridx;
 206
 207         /**
 208          * SQE array management fields:
 209          *
 210          *  -----------------------------------------------------
 211          *  | SQE0 | SQE1 | SQE2 |   ...  | SQEx | ... | SQEn-1 |
 212          *  -----------------------------------------------------
 213          *     ^             ^               ^
 214          *     |             |               |
 215          *   sq_head     cq_sq_head       sq_tail
 216          *
 217          *  sq_head: index to the oldest completed request, this filed was
 218          *           updated by hisi_dmadev_completed* APIs.
 219          *  sq_tail: index of the next new request, this field was updated by
 220          *           hisi_dmadev_copy API.
 221          *  cq_sq_head: next index of index that has been completed by hardware,
 222          *              this filed was updated by hisi_dmadev_completed* APIs.
 223          *
 224          *  [sq_head, cq_sq_head): the SQEs that hardware already completed.
 225          *  [cq_sq_head, sq_tail): the SQEs that hardware processing.
 226          */
 227         uint16_t sq_head;
 228         uint16_t sq_tail;
 229         uint16_t cq_sq_head;
 230         /**
 231          * The driver scans the CQE array, if the valid bit changes, the CQE is
 232          * considered valid.
 233          * Note: One CQE is corresponding to one or several SQEs, e.g. app
 234          *       submits two copy requests, the hardware processes the two SQEs,
 235          *       but it may write back only one CQE and the CQE's sq_head field
 236          *       indicates the index of the second copy request in the SQE
 237          *       array.
 238          */
 239         uint16_t cq_head; /**< CQ index for next scans. */
 240         /** accumulated number of completed CQs
 241          * @see HISI_DMA_CQ_RESERVED
 242          */
 243         uint16_t cqs_completed;
 244         uint8_t cqe_vld; /**< valid bit for CQE, will change for every round. */
 245
 246         uint64_t submitted;
 247         uint64_t completed;
 248         uint64_t errors;
 249         uint64_t qfulls;
 250
 251         /**
 252          * The following fields are not accessed in the I/O path, so they are
 253          * placed at the end.
 254          */
 255         struct rte_dma_dev_data *data;
 256         uint8_t revision; /**< PCI revision. */
 257         uint8_t reg_layout; /**< hardware register layout. */
 258         void *io_base;
 259         uint8_t queue_id; /**< hardware DMA queue index. */
 260         const struct rte_memzone *iomz;
 261         uint32_t iomz_sz;
 262         rte_iova_t sqe_iova;
 263         rte_iova_t cqe_iova;
 264 };
 265
 266 #endif /* HISI_DMADEV_H */