X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fraw%2Fioat%2Frte_ioat_rawdev.h;h=4b271f17443b88ab774deb82c1ab66c68c33a959;hb=85e26044cb8da437428dcf3a0755ceb1cdd38426;hp=f2cf98cd9367509fd664e44c46fa0ca7c8e4e3fe;hpb=849470d522ed63bee142de75f6f6cf174722b75b;p=dpdk.git diff --git a/drivers/raw/ioat/rte_ioat_rawdev.h b/drivers/raw/ioat/rte_ioat_rawdev.h index f2cf98cd93..4b271f1744 100644 --- a/drivers/raw/ioat/rte_ioat_rawdev.h +++ b/drivers/raw/ioat/rte_ioat_rawdev.h @@ -15,9 +15,11 @@ */ #include +#include #include #include -#include +#include +#include "rte_ioat_spec.h" /** Name of the device driver */ #define IOAT_PMD_RAWDEV_NAME rawdev_ioat @@ -54,8 +56,178 @@ struct rte_ioat_rawdev { struct rte_ioat_generic_hw_desc *desc_ring; __m128i *hdls; /* completion handles for returning to user */ + + unsigned short next_read; + unsigned short next_write; + + /* some statistics for tracking, if added/changed update xstats fns*/ + uint64_t enqueue_failed __rte_cache_aligned; + uint64_t enqueued; + uint64_t started; + uint64_t completed; + /* to report completions, the device will write status back here */ volatile uint64_t status __rte_cache_aligned; }; -#endif +/** + * Enqueue a copy operation onto the ioat device + * + * This queues up a copy operation to be performed by hardware, but does not + * trigger hardware to begin that operation. + * + * @param dev_id + * The rawdev device id of the ioat instance + * @param src + * The physical address of the source buffer + * @param dst + * The physical address of the destination buffer + * @param length + * The length of the data to be copied + * @param src_hdl + * An opaque handle for the source data, to be returned when this operation + * has been completed and the user polls for the completion details + * @param dst_hdl + * An opaque handle for the destination data, to be returned when this + * operation has been completed and the user polls for the completion details + * @param fence + * A flag parameter indicating that hardware should not begin to perform any + * subsequently enqueued copy operations until after this operation has + * completed + * @return + * Number of operations enqueued, either 0 or 1 + */ +static inline int +rte_ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst, + unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl, + int fence) +{ + struct rte_ioat_rawdev *ioat = rte_rawdevs[dev_id].dev_private; + unsigned short read = ioat->next_read; + unsigned short write = ioat->next_write; + unsigned short mask = ioat->ring_size - 1; + unsigned short space = mask + read - write; + struct rte_ioat_generic_hw_desc *desc; + + if (space == 0) { + ioat->enqueue_failed++; + return 0; + } + + ioat->next_write = write + 1; + write &= mask; + + desc = &ioat->desc_ring[write]; + desc->size = length; + /* set descriptor write-back every 16th descriptor */ + desc->u.control_raw = (uint32_t)((!!fence << 4) | (!(write & 0xF)) << 3); + desc->src_addr = src; + desc->dest_addr = dst; + + ioat->hdls[write] = _mm_set_epi64x((int64_t)dst_hdl, (int64_t)src_hdl); + rte_prefetch0(&ioat->desc_ring[ioat->next_write & mask]); + + ioat->enqueued++; + return 1; +} + +/** + * Trigger hardware to begin performing enqueued copy operations + * + * This API is used to write the "doorbell" to the hardware to trigger it + * to begin the copy operations previously enqueued by rte_ioat_enqueue_copy() + * + * @param dev_id + * The rawdev device id of the ioat instance + */ +static inline void +rte_ioat_do_copies(int dev_id) +{ + struct rte_ioat_rawdev *ioat = rte_rawdevs[dev_id].dev_private; + ioat->desc_ring[(ioat->next_write - 1) & (ioat->ring_size - 1)].u + .control.completion_update = 1; + rte_compiler_barrier(); + ioat->regs->dmacount = ioat->next_write; + ioat->started = ioat->enqueued; +} + +/** + * @internal + * Returns the index of the last completed operation. + */ +static inline int +rte_ioat_get_last_completed(struct rte_ioat_rawdev *ioat, int *error) +{ + uint64_t status = ioat->status; + + /* lower 3 bits indicate "transfer status" : active, idle, halted. + * We can ignore bit 0. + */ + *error = status & (RTE_IOAT_CHANSTS_SUSPENDED | RTE_IOAT_CHANSTS_ARMED); + return (status - ioat->ring_addr) >> 6; +} + +/** + * Returns details of copy operations that have been completed + * + * Returns to the caller the user-provided "handles" for the copy operations + * which have been completed by the hardware, and not already returned by + * a previous call to this API. + * + * @param dev_id + * The rawdev device id of the ioat instance + * @param max_copies + * The number of entries which can fit in the src_hdls and dst_hdls + * arrays, i.e. max number of completed operations to report + * @param src_hdls + * Array to hold the source handle parameters of the completed copies + * @param dst_hdls + * Array to hold the destination handle parameters of the completed copies + * @return + * -1 on error, with rte_errno set appropriately. + * Otherwise number of completed operations i.e. number of entries written + * to the src_hdls and dst_hdls array parameters. + */ +static inline int +rte_ioat_completed_copies(int dev_id, uint8_t max_copies, + uintptr_t *src_hdls, uintptr_t *dst_hdls) +{ + struct rte_ioat_rawdev *ioat = rte_rawdevs[dev_id].dev_private; + unsigned short mask = (ioat->ring_size - 1); + unsigned short read = ioat->next_read; + unsigned short end_read, count; + int error; + int i = 0; + + end_read = (rte_ioat_get_last_completed(ioat, &error) + 1) & mask; + count = (end_read - (read & mask)) & mask; + + if (error) { + rte_errno = EIO; + return -1; + } + + if (count > max_copies) + count = max_copies; + + for (; i < count - 1; i += 2, read += 2) { + __m128i hdls0 = _mm_load_si128(&ioat->hdls[read & mask]); + __m128i hdls1 = _mm_load_si128(&ioat->hdls[(read + 1) & mask]); + + _mm_storeu_si128((void *)&src_hdls[i], + _mm_unpacklo_epi64(hdls0, hdls1)); + _mm_storeu_si128((void *)&dst_hdls[i], + _mm_unpackhi_epi64(hdls0, hdls1)); + } + for (; i < count; i++, read++) { + uintptr_t *hdls = (void *)&ioat->hdls[read & mask]; + src_hdls[i] = hdls[0]; + dst_hdls[i] = hdls[1]; + } + + ioat->next_read = read; + ioat->completed += count; + return count; +} + +#endif /* _RTE_IOAT_RAWDEV_H_ */