X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fdma%2Fioat%2Fioat_dmadev.c;h=a230496b111d6f7115fee433de135f0bfcd7cd66;hb=9fda31c3229ca6e036cae80392578ed6e5a51119;hp=4d00fec5c8e23ecdca49716b24389e1864217c1e;hpb=3d4b027324be6acded76ef1b94ad862fe02189c9;p=dpdk.git diff --git a/drivers/dma/ioat/ioat_dmadev.c b/drivers/dma/ioat/ioat_dmadev.c index 4d00fec5c8..a230496b11 100644 --- a/drivers/dma/ioat/ioat_dmadev.c +++ b/drivers/dma/ioat/ioat_dmadev.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "ioat_internal.h" @@ -76,6 +77,9 @@ ioat_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan __rte_unused, ioat->offset = 0; ioat->failure = 0; + /* Reset Stats. */ + ioat->stats = (struct rte_dma_stats){0}; + /* Configure descriptor ring - each one points to next. */ for (i = 0; i < ioat->qcfg.nb_desc; i++) { ioat->desc_ring[i].next = ioat->ring_addr + @@ -221,6 +225,8 @@ __submit(struct ioat_dmadev *ioat) { *ioat->doorbell = ioat->next_write - ioat->offset; + ioat->stats.submitted += (uint16_t)(ioat->next_write - ioat->last_write); + ioat->last_write = ioat->next_write; } @@ -351,6 +357,10 @@ __dev_dump(void *dev_private, FILE *f) fprintf(f, " Dest: 0x%"PRIx64"\n", ioat->desc_ring[ioat->next_read & mask].dest_addr); fprintf(f, " Next: 0x%"PRIx64"\n", ioat->desc_ring[ioat->next_read & mask].next); fprintf(f, " }\n"); + fprintf(f, " Key Stats { submitted: %"PRIu64", comp: %"PRIu64", failed: %"PRIu64" }\n", + ioat->stats.submitted, + ioat->stats.completed, + ioat->stats.errors); return 0; } @@ -362,6 +372,209 @@ ioat_dev_dump(const struct rte_dma_dev *dev, FILE *f) return __dev_dump(dev->fp_obj->dev_private, f); } +/* Returns the index of the last completed operation. */ +static inline uint16_t +__get_last_completed(const struct ioat_dmadev *ioat, int *state) +{ + /* Status register contains the address of the completed operation */ + uint64_t status = ioat->status; + + /* lower 3 bits indicate "transfer status" : active, idle, halted. + * We can ignore bit 0. + */ + *state = status & IOAT_CHANSTS_STATUS; + + /* If we are just after recovering from an error the address returned by + * status will be 0, in this case we return the offset - 1 as the last + * completed. If not return the status value minus the chainaddr which + * gives us an offset into the ring. Right shifting by 6 (divide by 64) + * gives the index of the completion from the HW point of view and adding + * the offset translates the ring index from HW to SW point of view. + */ + if ((status & ~IOAT_CHANSTS_STATUS) == 0) + return ioat->offset - 1; + + return (status - ioat->ring_addr) >> 6; +} + +/* Translates IOAT ChanERRs to DMA error codes. */ +static inline enum rte_dma_status_code +__translate_status_ioat_to_dma(uint32_t chanerr) +{ + if (chanerr & IOAT_CHANERR_INVALID_SRC_ADDR_MASK) + return RTE_DMA_STATUS_INVALID_SRC_ADDR; + else if (chanerr & IOAT_CHANERR_INVALID_DST_ADDR_MASK) + return RTE_DMA_STATUS_INVALID_DST_ADDR; + else if (chanerr & IOAT_CHANERR_INVALID_LENGTH_MASK) + return RTE_DMA_STATUS_INVALID_LENGTH; + else if (chanerr & IOAT_CHANERR_DESCRIPTOR_READ_ERROR_MASK) + return RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR; + else + return RTE_DMA_STATUS_ERROR_UNKNOWN; +} + +/* Returns details of operations that have been completed. */ +static uint16_t +ioat_completed(void *dev_private, uint16_t qid __rte_unused, const uint16_t max_ops, + uint16_t *last_idx, bool *has_error) +{ + struct ioat_dmadev *ioat = dev_private; + + const unsigned short mask = (ioat->qcfg.nb_desc - 1); + const unsigned short read = ioat->next_read; + unsigned short last_completed, count; + int state, fails = 0; + + /* Do not do any work if there is an uncleared error. */ + if (ioat->failure != 0) { + *has_error = true; + *last_idx = ioat->next_read - 2; + return 0; + } + + last_completed = __get_last_completed(ioat, &state); + count = (last_completed + 1 - read) & mask; + + /* Cap count at max_ops or set as last run in batch. */ + if (count > max_ops) + count = max_ops; + + if (count == max_ops || state != IOAT_CHANSTS_HALTED) { + ioat->next_read = read + count; + *last_idx = ioat->next_read - 1; + } else { + *has_error = true; + rte_errno = EIO; + ioat->failure = ioat->regs->chanerr; + ioat->next_read = read + count + 1; + if (__ioat_recover(ioat) != 0) { + IOAT_PMD_ERR("Device HALTED and could not be recovered\n"); + __dev_dump(dev_private, stdout); + return 0; + } + __submit(ioat); + fails++; + *last_idx = ioat->next_read - 2; + } + + ioat->stats.completed += count; + ioat->stats.errors += fails; + + return count; +} + +/* Returns detailed status information about operations that have been completed. */ +static uint16_t +ioat_completed_status(void *dev_private, uint16_t qid __rte_unused, + uint16_t max_ops, uint16_t *last_idx, enum rte_dma_status_code *status) +{ + struct ioat_dmadev *ioat = dev_private; + + const unsigned short mask = (ioat->qcfg.nb_desc - 1); + const unsigned short read = ioat->next_read; + unsigned short count, last_completed; + uint64_t fails = 0; + int state, i; + + last_completed = __get_last_completed(ioat, &state); + count = (last_completed + 1 - read) & mask; + + for (i = 0; i < RTE_MIN(count + 1, max_ops); i++) + status[i] = RTE_DMA_STATUS_SUCCESSFUL; + + /* Cap count at max_ops or set as last run in batch. */ + if (count > max_ops) + count = max_ops; + + if (count == max_ops || state != IOAT_CHANSTS_HALTED) + ioat->next_read = read + count; + else { + rte_errno = EIO; + status[count] = __translate_status_ioat_to_dma(ioat->regs->chanerr); + count++; + ioat->next_read = read + count; + if (__ioat_recover(ioat) != 0) { + IOAT_PMD_ERR("Device HALTED and could not be recovered\n"); + __dev_dump(dev_private, stdout); + return 0; + } + __submit(ioat); + fails++; + } + + if (ioat->failure > 0) { + status[0] = __translate_status_ioat_to_dma(ioat->failure); + count = RTE_MIN(count + 1, max_ops); + ioat->failure = 0; + } + + *last_idx = ioat->next_read - 1; + + ioat->stats.completed += count; + ioat->stats.errors += fails; + + return count; +} + +/* Get the remaining capacity of the ring. */ +static uint16_t +ioat_burst_capacity(const void *dev_private, uint16_t vchan __rte_unused) +{ + const struct ioat_dmadev *ioat = dev_private; + unsigned short size = ioat->qcfg.nb_desc - 1; + unsigned short read = ioat->next_read; + unsigned short write = ioat->next_write; + unsigned short space = size - (write - read); + + return space; +} + +/* Retrieve the generic stats of a DMA device. */ +static int +ioat_stats_get(const struct rte_dma_dev *dev, uint16_t vchan __rte_unused, + struct rte_dma_stats *rte_stats, uint32_t size) +{ + struct rte_dma_stats *stats = (&((struct ioat_dmadev *)dev->fp_obj->dev_private)->stats); + + if (size < sizeof(rte_stats)) + return -EINVAL; + if (rte_stats == NULL) + return -EINVAL; + + *rte_stats = *stats; + return 0; +} + +/* Reset the generic stat counters for the DMA device. */ +static int +ioat_stats_reset(struct rte_dma_dev *dev, uint16_t vchan __rte_unused) +{ + struct ioat_dmadev *ioat = dev->fp_obj->dev_private; + + ioat->stats = (struct rte_dma_stats){0}; + return 0; +} + +/* Check if the IOAT device is idle. */ +static int +ioat_vchan_status(const struct rte_dma_dev *dev, uint16_t vchan __rte_unused, + enum rte_dma_vchan_status *status) +{ + int state = 0; + const struct ioat_dmadev *ioat = dev->fp_obj->dev_private; + const uint16_t mask = ioat->qcfg.nb_desc - 1; + const uint16_t last = __get_last_completed(ioat, &state); + + if (state == IOAT_CHANSTS_HALTED || state == IOAT_CHANSTS_SUSPENDED) + *status = RTE_DMA_VCHAN_HALTED_ERROR; + else if (last == ((ioat->next_write - 1) & mask)) + *status = RTE_DMA_VCHAN_IDLE; + else + *status = RTE_DMA_VCHAN_ACTIVE; + + return 0; +} + /* Create a DMA device. */ static int ioat_dmadev_create(const char *name, struct rte_pci_device *dev) @@ -373,6 +586,9 @@ ioat_dmadev_create(const char *name, struct rte_pci_device *dev) .dev_info_get = ioat_dev_info_get, .dev_start = ioat_dev_start, .dev_stop = ioat_dev_stop, + .stats_get = ioat_stats_get, + .stats_reset = ioat_stats_reset, + .vchan_status = ioat_vchan_status, .vchan_setup = ioat_vchan_setup, }; @@ -398,6 +614,9 @@ ioat_dmadev_create(const char *name, struct rte_pci_device *dev) dmadev->dev_ops = &ioat_dmadev_ops; + dmadev->fp_obj->burst_capacity = ioat_burst_capacity; + dmadev->fp_obj->completed = ioat_completed; + dmadev->fp_obj->completed_status = ioat_completed_status; dmadev->fp_obj->copy = ioat_enqueue_copy; dmadev->fp_obj->fill = ioat_enqueue_fill; dmadev->fp_obj->submit = ioat_submit;