#include <rte_rawdev.h>
#include <rte_memzone.h>
#include <rte_prefetch.h>
-#include "rte_ioat_spec.h"
/**
* @internal
- * Structure representing a device instance
+ * Structure representing a device descriptor
+ */
+struct rte_ioat_generic_hw_desc {
+ uint32_t size;
+ union {
+ uint32_t control_raw;
+ struct {
+ uint32_t int_enable: 1;
+ uint32_t src_snoop_disable: 1;
+ uint32_t dest_snoop_disable: 1;
+ uint32_t completion_update: 1;
+ uint32_t fence: 1;
+ uint32_t reserved2: 1;
+ uint32_t src_page_break: 1;
+ uint32_t dest_page_break: 1;
+ uint32_t bundle: 1;
+ uint32_t dest_dca: 1;
+ uint32_t hint: 1;
+ uint32_t reserved: 13;
+ uint32_t op: 8;
+ } control;
+ } u;
+ uint64_t src_addr;
+ uint64_t dest_addr;
+ uint64_t next;
+ uint64_t op_specific[4];
+};
+
+/**
+ * @internal
+ * Identify the data path to use.
+ * Must be first field of rte_ioat_rawdev and rte_idxd_rawdev structs
+ */
+enum rte_ioat_dev_type {
+ RTE_IOAT_DEV,
+ RTE_IDXD_DEV,
+};
+
+/**
+ * @internal
+ * Structure representing an IOAT device instance
*/
struct rte_ioat_rawdev {
+ enum rte_ioat_dev_type type;
struct rte_rawdev *rawdev;
const struct rte_memzone *mz;
const struct rte_memzone *desc_mz;
- volatile struct rte_ioat_registers *regs;
+ volatile uint16_t *doorbell;
phys_addr_t status_addr;
phys_addr_t ring_addr;
/* to report completions, the device will write status back here */
volatile uint64_t status __rte_cache_aligned;
+
+ /* pointer to the register bar */
+ volatile struct rte_ioat_registers *regs;
+};
+
+#define RTE_IOAT_CHANSTS_IDLE 0x1
+#define RTE_IOAT_CHANSTS_SUSPENDED 0x2
+#define RTE_IOAT_CHANSTS_HALTED 0x3
+#define RTE_IOAT_CHANSTS_ARMED 0x4
+
+/*
+ * Defines used in the data path for interacting with hardware.
+ */
+#define IDXD_CMD_OP_SHIFT 24
+enum rte_idxd_ops {
+ idxd_op_nop = 0,
+ idxd_op_batch,
+ idxd_op_drain,
+ idxd_op_memmove,
+ idxd_op_fill
+};
+
+#define IDXD_FLAG_FENCE (1 << 0)
+#define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2)
+#define IDXD_FLAG_REQUEST_COMPLETION (1 << 3)
+#define IDXD_FLAG_CACHE_CONTROL (1 << 8)
+
+/**
+ * Hardware descriptor used by DSA hardware, for both bursts and
+ * for individual operations.
+ */
+struct rte_idxd_hw_desc {
+ uint32_t pasid;
+ uint32_t op_flags;
+ rte_iova_t completion;
+
+ RTE_STD_C11
+ union {
+ rte_iova_t src; /* source address for copy ops etc. */
+ rte_iova_t desc_addr; /* descriptor pointer for batch */
+ };
+ rte_iova_t dst;
+
+ uint32_t size; /* length of data for op, or batch size */
+
+ /* 28 bytes of padding here */
+} __rte_aligned(64);
+
+/**
+ * Completion record structure written back by DSA
+ */
+struct rte_idxd_completion {
+ uint8_t status;
+ uint8_t result;
+ /* 16-bits pad here */
+ uint32_t completed_size; /* data length, or descriptors for batch */
+
+ rte_iova_t fault_address;
+ uint32_t invalid_flags;
+} __rte_aligned(32);
+
+#define BATCH_SIZE 64
+
+/**
+ * Structure used inside the driver for building up and submitting
+ * a batch of operations to the DSA hardware.
+ */
+struct rte_idxd_desc_batch {
+ struct rte_idxd_completion comp; /* the completion record for batch */
+
+ uint16_t submitted;
+ uint16_t op_count;
+ uint16_t hdl_end;
+
+ struct rte_idxd_hw_desc batch_desc;
+
+ /* batches must always have 2 descriptors, so put a null at the start */
+ struct rte_idxd_hw_desc null_desc;
+ struct rte_idxd_hw_desc ops[BATCH_SIZE];
+};
+
+/**
+ * structure used to save the "handles" provided by the user to be
+ * returned to the user on job completion.
+ */
+struct rte_idxd_user_hdl {
+ uint64_t src;
+ uint64_t dst;
+};
+
+/**
+ * @internal
+ * Structure representing an IDXD device instance
+ */
+struct rte_idxd_rawdev {
+ enum rte_ioat_dev_type type;
+ void *portal; /* address to write the batch descriptor */
+
+ /* counters to track the batches and the individual op handles */
+ uint16_t batch_ring_sz; /* size of batch ring */
+ uint16_t hdl_ring_sz; /* size of the user hdl ring */
+
+ uint16_t next_batch; /* where we write descriptor ops */
+ uint16_t next_completed; /* batch where we read completions */
+ uint16_t next_ret_hdl; /* the next user hdl to return */
+ uint16_t last_completed_hdl; /* the last user hdl that has completed */
+ uint16_t next_free_hdl; /* where the handle for next op will go */
+ uint16_t hdls_disable; /* disable tracking completion handles */
+
+ struct rte_idxd_user_hdl *hdl_ring;
+ struct rte_idxd_desc_batch *batch_ring;
};
/*
*/
static inline int
rte_ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
- unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl,
- int fence)
+ unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
{
struct rte_ioat_rawdev *ioat =
(struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
desc = &ioat->desc_ring[write];
desc->size = length;
/* set descriptor write-back every 16th descriptor */
- desc->u.control_raw = (uint32_t)((!!fence << 4) | (!(write & 0xF)) << 3);
+ desc->u.control_raw = (uint32_t)((!(write & 0xF)) << 3);
desc->src_addr = src;
desc->dest_addr = dst;
return 1;
}
+/* add fence to last written descriptor */
+static inline int
+rte_ioat_fence(int dev_id)
+{
+ struct rte_ioat_rawdev *ioat =
+ (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
+ unsigned short write = ioat->next_write;
+ unsigned short mask = ioat->ring_size - 1;
+ struct rte_ioat_generic_hw_desc *desc;
+
+ write = (write - 1) & mask;
+ desc = &ioat->desc_ring[write];
+
+ desc->u.control.fence = 1;
+ return 0;
+}
+
/*
- * Trigger hardware to begin performing enqueued copy operations
+ * Trigger hardware to begin performing enqueued operations
*/
static inline void
-rte_ioat_do_copies(int dev_id)
+rte_ioat_perform_ops(int dev_id)
{
struct rte_ioat_rawdev *ioat =
(struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
ioat->desc_ring[(ioat->next_write - 1) & (ioat->ring_size - 1)].u
.control.completion_update = 1;
rte_compiler_barrier();
- ioat->regs->dmacount = ioat->next_write;
+ *ioat->doorbell = ioat->next_write;
ioat->started = ioat->enqueued;
}
}
/*
- * Returns details of copy operations that have been completed
+ * Returns details of operations that have been completed
*/
static inline int
-rte_ioat_completed_copies(int dev_id, uint8_t max_copies,
+rte_ioat_completed_ops(int dev_id, uint8_t max_copies,
uintptr_t *src_hdls, uintptr_t *dst_hdls)
{
struct rte_ioat_rawdev *ioat =
return count;
}
+static inline void
+__rte_deprecated_msg("use rte_ioat_perform_ops() instead")
+rte_ioat_do_copies(int dev_id) { rte_ioat_perform_ops(dev_id); }
+
+static inline int
+__rte_deprecated_msg("use rte_ioat_completed_ops() instead")
+rte_ioat_completed_copies(int dev_id, uint8_t max_copies,
+ uintptr_t *src_hdls, uintptr_t *dst_hdls)
+{
+ return rte_ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls);
+}
+
#endif /* _RTE_IOAT_RAWDEV_FNS_H_ */
git.droids-corp.org / dpdk.git / blobdiff