X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fraw%2Fioat%2Frte_ioat_rawdev_fns.h;h=6049e3bd8be7644a1884f40fa630ff6bf363f5e6;hb=675ba7069148c7b18f3c74a66938dd34ee8ca653;hp=89bfc8d21a1276583a87752caedf1c6d0aead057;hpb=8636b9a18e19a48fc7c133900b476ac581f02aaa;p=dpdk.git diff --git a/drivers/raw/ioat/rte_ioat_rawdev_fns.h b/drivers/raw/ioat/rte_ioat_rawdev_fns.h index 89bfc8d21a..6049e3bd8b 100644 --- a/drivers/raw/ioat/rte_ioat_rawdev_fns.h +++ b/drivers/raw/ioat/rte_ioat_rawdev_fns.h @@ -4,11 +4,45 @@ #ifndef _RTE_IOAT_RAWDEV_FNS_H_ #define _RTE_IOAT_RAWDEV_FNS_H_ +/** + * @file + * This header file contains the implementation of the various ioat + * rawdev functions for IOAT/CBDMA hardware. The API specification and key + * public structures are defined in "rte_ioat_rawdev.h". + * + * This file should not be included directly, but instead applications should + * include "rte_ioat_rawdev.h", which then includes this file - and the IDXD/DSA + * equivalent header - in turn. + */ + #include #include #include #include +/** + * @internal + * Identify the data path to use. + * Must be first field of rte_ioat_rawdev and rte_idxd_rawdev structs + */ +enum rte_ioat_dev_type { + RTE_IOAT_DEV, + RTE_IDXD_DEV, +}; + +/** + * @internal + * some statistics for tracking, if added/changed update xstats fns + */ +struct rte_ioat_xstats { + uint64_t enqueue_failed; + uint64_t enqueued; + uint64_t started; + uint64_t completed; +}; + +#include "rte_idxd_rawdev_fns.h" + /** * @internal * Structure representing a device descriptor @@ -39,27 +73,6 @@ struct rte_ioat_generic_hw_desc { uint64_t op_specific[4]; }; -/** - * @internal - * Identify the data path to use. - * Must be first field of rte_ioat_rawdev and rte_idxd_rawdev structs - */ -enum rte_ioat_dev_type { - RTE_IOAT_DEV, - RTE_IDXD_DEV, -}; - -/** - * @internal - * some statistics for tracking, if added/changed update xstats fns - */ -struct rte_ioat_xstats { - uint64_t enqueue_failed; - uint64_t enqueued; - uint64_t started; - uint64_t completed; -}; - /** * @internal * Structure representing an IOAT device instance @@ -98,114 +111,21 @@ struct rte_ioat_rawdev { #define RTE_IOAT_CHANSTS_HALTED 0x3 #define RTE_IOAT_CHANSTS_ARMED 0x4 -/* - * Defines used in the data path for interacting with hardware. - */ -#define IDXD_CMD_OP_SHIFT 24 -enum rte_idxd_ops { - idxd_op_nop = 0, - idxd_op_batch, - idxd_op_drain, - idxd_op_memmove, - idxd_op_fill -}; - -#define IDXD_FLAG_FENCE (1 << 0) -#define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2) -#define IDXD_FLAG_REQUEST_COMPLETION (1 << 3) -#define IDXD_FLAG_CACHE_CONTROL (1 << 8) - -/** - * Hardware descriptor used by DSA hardware, for both bursts and - * for individual operations. - */ -struct rte_idxd_hw_desc { - uint32_t pasid; - uint32_t op_flags; - rte_iova_t completion; - - RTE_STD_C11 - union { - rte_iova_t src; /* source address for copy ops etc. */ - rte_iova_t desc_addr; /* descriptor pointer for batch */ - }; - rte_iova_t dst; - - uint32_t size; /* length of data for op, or batch size */ - - /* 28 bytes of padding here */ -} __rte_aligned(64); - -/** - * Completion record structure written back by DSA - */ -struct rte_idxd_completion { - uint8_t status; - uint8_t result; - /* 16-bits pad here */ - uint32_t completed_size; /* data length, or descriptors for batch */ - - rte_iova_t fault_address; - uint32_t invalid_flags; -} __rte_aligned(32); - -#define BATCH_SIZE 64 - -/** - * Structure used inside the driver for building up and submitting - * a batch of operations to the DSA hardware. - */ -struct rte_idxd_desc_batch { - struct rte_idxd_completion comp; /* the completion record for batch */ - - uint16_t submitted; - uint16_t op_count; - uint16_t hdl_end; - - struct rte_idxd_hw_desc batch_desc; - - /* batches must always have 2 descriptors, so put a null at the start */ - struct rte_idxd_hw_desc null_desc; - struct rte_idxd_hw_desc ops[BATCH_SIZE]; -}; - -/** - * structure used to save the "handles" provided by the user to be - * returned to the user on job completion. - */ -struct rte_idxd_user_hdl { - uint64_t src; - uint64_t dst; -}; - -/** - * @internal - * Structure representing an IDXD device instance - */ -struct rte_idxd_rawdev { - enum rte_ioat_dev_type type; - void *portal; /* address to write the batch descriptor */ - - /* counters to track the batches and the individual op handles */ - uint16_t batch_ring_sz; /* size of batch ring */ - uint16_t hdl_ring_sz; /* size of the user hdl ring */ - - uint16_t next_batch; /* where we write descriptor ops */ - uint16_t next_completed; /* batch where we read completions */ - uint16_t next_ret_hdl; /* the next user hdl to return */ - uint16_t last_completed_hdl; /* the last user hdl that has completed */ - uint16_t next_free_hdl; /* where the handle for next op will go */ - uint16_t hdls_disable; /* disable tracking completion handles */ +static __rte_always_inline uint16_t +__ioat_burst_capacity(int dev_id) +{ + struct rte_ioat_rawdev *ioat = + (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private; + unsigned short size = ioat->ring_size - 1; + unsigned short read = ioat->next_read; + unsigned short write = ioat->next_write; + unsigned short space = size - (write - read); - struct rte_idxd_user_hdl *hdl_ring; - struct rte_idxd_desc_batch *batch_ring; -}; + return space; +} -/* - * Enqueue a copy operation onto the ioat device - */ static __rte_always_inline int -__ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst, +__ioat_write_desc(int dev_id, uint32_t op, uint64_t src, phys_addr_t dst, unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl) { struct rte_ioat_rawdev *ioat = @@ -227,7 +147,8 @@ __ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst, desc = &ioat->desc_ring[write]; desc->size = length; /* set descriptor write-back every 16th descriptor */ - desc->u.control_raw = (uint32_t)((!(write & 0xF)) << 3); + desc->u.control_raw = (uint32_t)((op << IOAT_CMD_OP_SHIFT) | + (!(write & 0xF) << IOAT_COMP_UPDATE_SHIFT)); desc->src_addr = src; desc->dest_addr = dst; @@ -240,6 +161,27 @@ __ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst, return 1; } +static __rte_always_inline int +__ioat_enqueue_fill(int dev_id, uint64_t pattern, phys_addr_t dst, + unsigned int length, uintptr_t dst_hdl) +{ + static const uintptr_t null_hdl; + + return __ioat_write_desc(dev_id, ioat_op_fill, pattern, dst, length, + null_hdl, dst_hdl); +} + +/* + * Enqueue a copy operation onto the ioat device + */ +static __rte_always_inline int +__ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst, + unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl) +{ + return __ioat_write_desc(dev_id, ioat_op_copy, src, dst, length, + src_hdl, dst_hdl); +} + /* add fence to last written descriptor */ static __rte_always_inline int __ioat_fence(int dev_id) @@ -260,7 +202,7 @@ __ioat_fence(int dev_id) /* * Trigger hardware to begin performing enqueued operations */ -static __rte_always_inline void +static __rte_always_inline int __ioat_perform_ops(int dev_id) { struct rte_ioat_rawdev *ioat = @@ -270,6 +212,8 @@ __ioat_perform_ops(int dev_id) rte_compiler_barrier(); *ioat->doorbell = ioat->next_write; ioat->xstats.started = ioat->xstats.enqueued; + + return 0; } /** @@ -340,133 +284,27 @@ end: return count; } -static __rte_always_inline int -__idxd_write_desc(int dev_id, const struct rte_idxd_hw_desc *desc, - const struct rte_idxd_user_hdl *hdl) +static inline uint16_t +rte_ioat_burst_capacity(int dev_id) { - struct rte_idxd_rawdev *idxd = - (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private; - struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch]; - - /* check for room in the handle ring */ - if (((idxd->next_free_hdl + 1) & (idxd->hdl_ring_sz - 1)) == idxd->next_ret_hdl) - goto failed; - - /* check for space in current batch */ - if (b->op_count >= BATCH_SIZE) - goto failed; - - /* check that we can actually use the current batch */ - if (b->submitted) - goto failed; - - /* write the descriptor */ - b->ops[b->op_count++] = *desc; - - /* store the completion details */ - if (!idxd->hdls_disable) - idxd->hdl_ring[idxd->next_free_hdl] = *hdl; - if (++idxd->next_free_hdl == idxd->hdl_ring_sz) - idxd->next_free_hdl = 0; - - return 1; - -failed: - rte_errno = ENOSPC; - return 0; -} - -static __rte_always_inline int -__idxd_enqueue_copy(int dev_id, rte_iova_t src, rte_iova_t dst, - unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl) -{ - const struct rte_idxd_hw_desc desc = { - .op_flags = (idxd_op_memmove << IDXD_CMD_OP_SHIFT) | - IDXD_FLAG_CACHE_CONTROL, - .src = src, - .dst = dst, - .size = length - }; - const struct rte_idxd_user_hdl hdl = { - .src = src_hdl, - .dst = dst_hdl - }; - return __idxd_write_desc(dev_id, &desc, &hdl); -} - -static __rte_always_inline int -__idxd_fence(int dev_id) -{ - static const struct rte_idxd_hw_desc fence = { - .op_flags = IDXD_FLAG_FENCE - }; - static const struct rte_idxd_user_hdl null_hdl; - return __idxd_write_desc(dev_id, &fence, &null_hdl); -} - -static __rte_always_inline void -__idxd_movdir64b(volatile void *dst, const void *src) -{ - asm volatile (".byte 0x66, 0x0f, 0x38, 0xf8, 0x02" - : - : "a" (dst), "d" (src)); -} - -static __rte_always_inline void -__idxd_perform_ops(int dev_id) -{ - struct rte_idxd_rawdev *idxd = - (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private; - struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch]; - - if (b->submitted || b->op_count == 0) - return; - b->hdl_end = idxd->next_free_hdl; - b->comp.status = 0; - b->submitted = 1; - b->batch_desc.size = b->op_count + 1; - __idxd_movdir64b(idxd->portal, &b->batch_desc); - - if (++idxd->next_batch == idxd->batch_ring_sz) - idxd->next_batch = 0; + enum rte_ioat_dev_type *type = + (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private; + if (*type == RTE_IDXD_DEV) + return __idxd_burst_capacity(dev_id); + else + return __ioat_burst_capacity(dev_id); } -static __rte_always_inline int -__idxd_completed_ops(int dev_id, uint8_t max_ops, - uintptr_t *src_hdls, uintptr_t *dst_hdls) +static inline int +rte_ioat_enqueue_fill(int dev_id, uint64_t pattern, phys_addr_t dst, + unsigned int len, uintptr_t dst_hdl) { - struct rte_idxd_rawdev *idxd = - (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private; - struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_completed]; - uint16_t h_idx = idxd->next_ret_hdl; - int n = 0; - - while (b->submitted && b->comp.status != 0) { - idxd->last_completed_hdl = b->hdl_end; - b->submitted = 0; - b->op_count = 0; - if (++idxd->next_completed == idxd->batch_ring_sz) - idxd->next_completed = 0; - b = &idxd->batch_ring[idxd->next_completed]; - } - - if (!idxd->hdls_disable) - for (n = 0; n < max_ops && h_idx != idxd->last_completed_hdl; n++) { - src_hdls[n] = idxd->hdl_ring[h_idx].src; - dst_hdls[n] = idxd->hdl_ring[h_idx].dst; - if (++h_idx == idxd->hdl_ring_sz) - h_idx = 0; - } + enum rte_ioat_dev_type *type = + (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private; + if (*type == RTE_IDXD_DEV) + return __idxd_enqueue_fill(dev_id, pattern, dst, len, dst_hdl); else - while (h_idx != idxd->last_completed_hdl) { - n++; - if (++h_idx == idxd->hdl_ring_sz) - h_idx = 0; - } - - idxd->next_ret_hdl = h_idx; - - return n; + return __ioat_enqueue_fill(dev_id, pattern, dst, len, dst_hdl); } static inline int @@ -494,7 +332,7 @@ rte_ioat_fence(int dev_id) return __ioat_fence(dev_id); } -static inline void +static inline int rte_ioat_perform_ops(int dev_id) { enum rte_ioat_dev_type *type = @@ -507,16 +345,22 @@ rte_ioat_perform_ops(int dev_id) static inline int rte_ioat_completed_ops(int dev_id, uint8_t max_copies, + uint32_t *status, uint8_t *num_unsuccessful, uintptr_t *src_hdls, uintptr_t *dst_hdls) { enum rte_ioat_dev_type *type = (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private; + uint8_t tmp; /* used so functions don't need to check for null parameter */ + + if (num_unsuccessful == NULL) + num_unsuccessful = &tmp; + + *num_unsuccessful = 0; if (*type == RTE_IDXD_DEV) - return __idxd_completed_ops(dev_id, max_copies, + return __idxd_completed_ops(dev_id, max_copies, status, num_unsuccessful, src_hdls, dst_hdls); else - return __ioat_completed_ops(dev_id, max_copies, - src_hdls, dst_hdls); + return __ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls); } static inline void @@ -528,7 +372,8 @@ __rte_deprecated_msg("use rte_ioat_completed_ops() instead") rte_ioat_completed_copies(int dev_id, uint8_t max_copies, uintptr_t *src_hdls, uintptr_t *dst_hdls) { - return rte_ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls); + return rte_ioat_completed_ops(dev_id, max_copies, NULL, NULL, + src_hdls, dst_hdls); } #endif /* _RTE_IOAT_RAWDEV_FNS_H_ */