#include <string.h>
#include <rte_bus_pci.h>
+#include <rte_cycles.h>
#include <rte_eal.h>
+#include <rte_io.h>
#include <rte_log.h>
#include <rte_pci.h>
#include <rte_dmadev_pmd.h>
#define HISI_DMA_ERR(hw, fmt, args...) \
HISI_DMA_LOG_RAW(hw, ERR, fmt, ## args)
+static uint32_t
+hisi_dma_queue_base(struct hisi_dma_dev *hw)
+{
+ if (hw->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
+ return HISI_DMA_HIP08_QUEUE_BASE;
+ else
+ return 0;
+}
+
+static void
+hisi_dma_write_reg(void *base, uint32_t off, uint32_t val)
+{
+ rte_write32(rte_cpu_to_le_32(val),
+ (volatile void *)((char *)base + off));
+}
+
+static void
+hisi_dma_write_dev(struct hisi_dma_dev *hw, uint32_t off, uint32_t val)
+{
+ hisi_dma_write_reg(hw->io_base, off, val);
+}
+
+static void
+hisi_dma_write_queue(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t val)
+{
+ uint32_t off = hisi_dma_queue_base(hw) +
+ hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
+ hisi_dma_write_dev(hw, off, val);
+}
+
+static uint32_t
+hisi_dma_read_reg(void *base, uint32_t off)
+{
+ uint32_t val = rte_read32((volatile void *)((char *)base + off));
+ return rte_le_to_cpu_32(val);
+}
+
+static uint32_t
+hisi_dma_read_dev(struct hisi_dma_dev *hw, uint32_t off)
+{
+ return hisi_dma_read_reg(hw->io_base, off);
+}
+
+static uint32_t
+hisi_dma_read_queue(struct hisi_dma_dev *hw, uint32_t qoff)
+{
+ uint32_t off = hisi_dma_queue_base(hw) +
+ hw->queue_id * HISI_DMA_QUEUE_REGION_SIZE + qoff;
+ return hisi_dma_read_dev(hw, off);
+}
+
+static void
+hisi_dma_update_bit(struct hisi_dma_dev *hw, uint32_t off, uint32_t pos,
+ bool set)
+{
+ uint32_t tmp = hisi_dma_read_dev(hw, off);
+ uint32_t mask = 1u << pos;
+ tmp = set ? tmp | mask : tmp & ~mask;
+ hisi_dma_write_dev(hw, off, tmp);
+}
+
+static void
+hisi_dma_update_queue_bit(struct hisi_dma_dev *hw, uint32_t qoff, uint32_t pos,
+ bool set)
+{
+ uint32_t tmp = hisi_dma_read_queue(hw, qoff);
+ uint32_t mask = 1u << pos;
+ tmp = set ? tmp | mask : tmp & ~mask;
+ hisi_dma_write_queue(hw, qoff, tmp);
+}
+
+#define hisi_dma_poll_hw_state(hw, val, cond, sleep_us, timeout_us) ({ \
+ uint32_t timeout = 0; \
+ while (timeout++ <= (timeout_us)) { \
+ (val) = hisi_dma_read_queue(hw, HISI_DMA_QUEUE_FSM_REG); \
+ if (cond) \
+ break; \
+ rte_delay_us(sleep_us); \
+ } \
+ (cond) ? 0 : -ETIME; \
+})
+
+static int
+hisi_dma_reset_hw(struct hisi_dma_dev *hw)
+{
+#define POLL_SLEEP_US 100
+#define POLL_TIMEOUT_US 10000
+
+ uint32_t tmp;
+ int ret;
+
+ hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
+ HISI_DMA_QUEUE_CTRL0_PAUSE_B, true);
+ hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
+ HISI_DMA_QUEUE_CTRL0_EN_B, false);
+
+ ret = hisi_dma_poll_hw_state(hw, tmp,
+ FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) != HISI_DMA_STATE_RUN,
+ POLL_SLEEP_US, POLL_TIMEOUT_US);
+ if (ret) {
+ HISI_DMA_ERR(hw, "disable dma timeout!");
+ return ret;
+ }
+
+ hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL1_REG,
+ HISI_DMA_QUEUE_CTRL1_RESET_B, true);
+ hisi_dma_write_queue(hw, HISI_DMA_QUEUE_SQ_TAIL_REG, 0);
+ hisi_dma_write_queue(hw, HISI_DMA_QUEUE_CQ_HEAD_REG, 0);
+ hisi_dma_update_queue_bit(hw, HISI_DMA_QUEUE_CTRL0_REG,
+ HISI_DMA_QUEUE_CTRL0_PAUSE_B, false);
+
+ ret = hisi_dma_poll_hw_state(hw, tmp,
+ FIELD_GET(HISI_DMA_QUEUE_FSM_STS_M, tmp) == HISI_DMA_STATE_IDLE,
+ POLL_SLEEP_US, POLL_TIMEOUT_US);
+ if (ret) {
+ HISI_DMA_ERR(hw, "reset dma timeout!");
+ return ret;
+ }
+
+ return 0;
+}
+
+static void
+hisi_dma_init_gbl(void *pci_bar, uint8_t revision)
+{
+ struct hisi_dma_dev hw;
+
+ memset(&hw, 0, sizeof(hw));
+ hw.io_base = pci_bar;
+
+ if (revision == HISI_DMA_REVISION_HIP08B)
+ hisi_dma_update_bit(&hw, HISI_DMA_HIP08_MODE_REG,
+ HISI_DMA_HIP08_MODE_SEL_B, true);
+}
+
static uint8_t
hisi_dma_reg_layout(uint8_t revision)
{
pci_dev->addr.function);
}
+static void
+hisi_dma_gen_dev_name(const struct rte_pci_device *pci_dev,
+ uint8_t queue_id, char *name, size_t size)
+{
+ memset(name, 0, size);
+ (void)snprintf(name, size, "%x:%x.%x-ch%u",
+ pci_dev->addr.bus, pci_dev->addr.devid,
+ pci_dev->addr.function, queue_id);
+}
+
+static int
+hisi_dma_create(struct rte_pci_device *pci_dev, uint8_t queue_id,
+ uint8_t revision)
+{
+#define REG_PCI_BAR_INDEX 2
+
+ char name[RTE_DEV_NAME_MAX_LEN];
+ struct rte_dma_dev *dev;
+ struct hisi_dma_dev *hw;
+ int ret;
+
+ hisi_dma_gen_dev_name(pci_dev, queue_id, name, sizeof(name));
+ dev = rte_dma_pmd_allocate(name, pci_dev->device.numa_node,
+ sizeof(*hw));
+ if (dev == NULL) {
+ HISI_DMA_LOG(ERR, "%s allocate dmadev fail!", name);
+ return -EINVAL;
+ }
+
+ dev->device = &pci_dev->device;
+
+ hw = dev->data->dev_private;
+ hw->data = dev->data;
+ hw->revision = revision;
+ hw->reg_layout = hisi_dma_reg_layout(revision);
+ hw->io_base = pci_dev->mem_resource[REG_PCI_BAR_INDEX].addr;
+ hw->queue_id = queue_id;
+
+ ret = hisi_dma_reset_hw(hw);
+ if (ret) {
+ HISI_DMA_LOG(ERR, "%s init device fail!", name);
+ (void)rte_dma_pmd_release(name);
+ return -EIO;
+ }
+
+ dev->state = RTE_DMA_DEV_READY;
+ HISI_DMA_LOG(DEBUG, "%s create dmadev success!", name);
+
+ return 0;
+}
+
static int
hisi_dma_check_revision(struct rte_pci_device *pci_dev, const char *name,
uint8_t *out_revision)
{
char name[RTE_DEV_NAME_MAX_LEN] = { 0 };
uint8_t revision;
+ uint8_t i;
int ret;
hisi_dma_gen_pci_device_name(pci_dev, name, sizeof(name));
return ret;
HISI_DMA_LOG(DEBUG, "%s read PCI revision: 0x%x", name, revision);
+ hisi_dma_init_gbl(pci_dev->mem_resource[2].addr, revision);
+
+ for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
+ ret = hisi_dma_create(pci_dev, i, revision);
+ if (ret) {
+ HISI_DMA_LOG(ERR, "%s create dmadev %u failed!",
+ name, i);
+ break;
+ }
+ }
+
return ret;
}
static int
hisi_dma_remove(struct rte_pci_device *pci_dev)
{
- RTE_SET_USED(pci_dev);
+ char name[RTE_DEV_NAME_MAX_LEN];
+ uint8_t i;
+ int ret;
+
+ for (i = 0; i < HISI_DMA_MAX_HW_QUEUES; i++) {
+ hisi_dma_gen_dev_name(pci_dev, i, name, sizeof(name));
+ ret = rte_dma_pmd_release(name);
+ if (ret)
+ return ret;
+ }
+
return 0;
}
#ifndef HISI_DMADEV_H
#define HISI_DMADEV_H
+#include <rte_byteorder.h>
+#include <rte_common.h>
+
+#define BIT(x) (1ul << (x))
+#define BITS_PER_LONG (__SIZEOF_LONG__ * 8)
+#define GENMASK(h, l) \
+ (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
+#define BF_SHF(x) (__builtin_ffsll(x) - 1)
+#define FIELD_GET(mask, reg) \
+ ((typeof(mask))(((reg) & (mask)) >> BF_SHF(mask)))
+
#define PCI_VENDOR_ID_HUAWEI 0x19e5
#define HISI_DMA_DEVICE_ID 0xA122
#define HISI_DMA_PCI_REVISION_ID_REG 0x08
#define HISI_DMA_REVISION_HIP08B 0x21
+#define HISI_DMA_MAX_HW_QUEUES 4
+
/**
* The HIP08B(HiSilicon IP08) and later Chip(e.g. HiSilicon IP09) are DMA iEPs,
* they have the same pci device id but with different pci revision.
HISI_DMA_REG_LAYOUT_HIP08
};
+/**
+ * Hardware PCI bar register MAP:
+ *
+ * --------------
+ * | Misc-reg-0 |
+ * | |
+ * -------------- -> Queue base
+ * | |
+ * | Queue-0 |
+ * | |
+ * -------------- ---
+ * | | ^
+ * | Queue-1 | Queue region
+ * | | v
+ * -------------- ---
+ * | ... |
+ * | Queue-x |
+ * | ... |
+ * --------------
+ * | Misc-reg-1 |
+ * --------------
+ *
+ * As described above, a single queue register is continuous and occupies the
+ * length of queue-region. The global offset for a single queue register is
+ * calculated by:
+ * offset = queue-base + (queue-id * queue-region) + reg-offset-in-region.
+ *
+ * The first part of queue region is basically the same for HIP08 and later chip
+ * register layouts, therefore, HISI_QUEUE_* registers are defined for it.
+ */
+#define HISI_DMA_QUEUE_SQ_BASE_L_REG 0x0
+#define HISI_DMA_QUEUE_SQ_BASE_H_REG 0x4
+#define HISI_DMA_QUEUE_SQ_DEPTH_REG 0x8
+#define HISI_DMA_QUEUE_SQ_TAIL_REG 0xC
+#define HISI_DMA_QUEUE_CQ_BASE_L_REG 0x10
+#define HISI_DMA_QUEUE_CQ_BASE_H_REG 0x14
+#define HISI_DMA_QUEUE_CQ_DEPTH_REG 0x18
+#define HISI_DMA_QUEUE_CQ_HEAD_REG 0x1C
+#define HISI_DMA_QUEUE_CTRL0_REG 0x20
+#define HISI_DMA_QUEUE_CTRL0_EN_B 0
+#define HISI_DMA_QUEUE_CTRL0_PAUSE_B 4
+#define HISI_DMA_QUEUE_CTRL1_REG 0x24
+#define HISI_DMA_QUEUE_CTRL1_RESET_B 0
+#define HISI_DMA_QUEUE_FSM_REG 0x30
+#define HISI_DMA_QUEUE_FSM_STS_M GENMASK(3, 0)
+#define HISI_DMA_QUEUE_INT_STATUS_REG 0x40
+#define HISI_DMA_QUEUE_ERR_INT_NUM0_REG 0x84
+#define HISI_DMA_QUEUE_ERR_INT_NUM1_REG 0x88
+#define HISI_DMA_QUEUE_ERR_INT_NUM2_REG 0x8C
+#define HISI_DMA_QUEUE_REGION_SIZE 0x100
+
+/**
+ * HiSilicon IP08 DMA register and field define:
+ */
+#define HISI_DMA_HIP08_QUEUE_BASE 0x0
+#define HISI_DMA_HIP08_QUEUE_CTRL0_ERR_ABORT_B 2
+#define HISI_DMA_HIP08_QUEUE_INT_MASK_REG 0x44
+#define HISI_DMA_HIP08_QUEUE_INT_MASK_M GENMASK(14, 0)
+#define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM3_REG 0x90
+#define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM4_REG 0x94
+#define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM5_REG 0x98
+#define HISI_DMA_HIP08_QUEUE_ERR_INT_NUM6_REG 0x48
+#define HISI_DMA_HIP08_MODE_REG 0x217C
+#define HISI_DMA_HIP08_MODE_SEL_B 0
+#define HISI_DMA_HIP08_DUMP_START_REG 0x2000
+#define HISI_DMA_HIP08_DUMP_END_REG 0x2280
+
+/**
+ * In fact, there are multiple states, but it need to pay attention to
+ * the following two states for the driver:
+ */
+enum {
+ HISI_DMA_STATE_IDLE = 0,
+ HISI_DMA_STATE_RUN,
+};
+
+struct hisi_dma_dev {
+ struct rte_dma_dev_data *data;
+ uint8_t revision; /**< PCI revision. */
+ uint8_t reg_layout; /**< hardware register layout. */
+ void *io_base;
+ uint8_t queue_id; /**< hardware DMA queue index. */
+};
+
#endif /* HISI_DMADEV_H */
git.droids-corp.org / dpdk.git / commitdiff