drivers/raw/ioat/rte_ioat_rawdev_fns.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Intel Corporation
   3  */
   4 #ifndef _RTE_IOAT_RAWDEV_FNS_H_
   5 #define _RTE_IOAT_RAWDEV_FNS_H_
   6
   7 #include <x86intrin.h>
   8 #include <rte_rawdev.h>
   9 #include <rte_memzone.h>
  10 #include <rte_prefetch.h>
  11 #include "rte_ioat_spec.h"
  12
  13 /**
  14  * @internal
  15  * Structure representing a device instance
  16  */
  17 struct rte_ioat_rawdev {
  18         struct rte_rawdev *rawdev;
  19         const struct rte_memzone *mz;
  20         const struct rte_memzone *desc_mz;
  21
  22         volatile struct rte_ioat_registers *regs;
  23         phys_addr_t status_addr;
  24         phys_addr_t ring_addr;
  25
  26         unsigned short ring_size;
  27         bool hdls_disable;
  28         struct rte_ioat_generic_hw_desc *desc_ring;
  29         __m128i *hdls; /* completion handles for returning to user */
  30
  31
  32         unsigned short next_read;
  33         unsigned short next_write;
  34
  35         /* some statistics for tracking, if added/changed update xstats fns*/
  36         uint64_t enqueue_failed __rte_cache_aligned;
  37         uint64_t enqueued;
  38         uint64_t started;
  39         uint64_t completed;
  40
  41         /* to report completions, the device will write status back here */
  42         volatile uint64_t status __rte_cache_aligned;
  43 };
  44
  45 /*
  46  * Enqueue a copy operation onto the ioat device
  47  */
  48 static inline int
  49 rte_ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
  50                 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
  51 {
  52         struct rte_ioat_rawdev *ioat =
  53                         (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
  54         unsigned short read = ioat->next_read;
  55         unsigned short write = ioat->next_write;
  56         unsigned short mask = ioat->ring_size - 1;
  57         unsigned short space = mask + read - write;
  58         struct rte_ioat_generic_hw_desc *desc;
  59
  60         if (space == 0) {
  61                 ioat->enqueue_failed++;
  62                 return 0;
  63         }
  64
  65         ioat->next_write = write + 1;
  66         write &= mask;
  67
  68         desc = &ioat->desc_ring[write];
  69         desc->size = length;
  70         /* set descriptor write-back every 16th descriptor */
  71         desc->u.control_raw = (uint32_t)((!(write & 0xF)) << 3);
  72         desc->src_addr = src;
  73         desc->dest_addr = dst;
  74
  75         if (!ioat->hdls_disable)
  76                 ioat->hdls[write] = _mm_set_epi64x((int64_t)dst_hdl,
  77                                         (int64_t)src_hdl);
  78         rte_prefetch0(&ioat->desc_ring[ioat->next_write & mask]);
  79
  80         ioat->enqueued++;
  81         return 1;
  82 }
  83
  84 /* add fence to last written descriptor */
  85 static inline int
  86 rte_ioat_fence(int dev_id)
  87 {
  88         struct rte_ioat_rawdev *ioat =
  89                         (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
  90         unsigned short write = ioat->next_write;
  91         unsigned short mask = ioat->ring_size - 1;
  92         struct rte_ioat_generic_hw_desc *desc;
  93
  94         write = (write - 1) & mask;
  95         desc = &ioat->desc_ring[write];
  96
  97         desc->u.control.fence = 1;
  98         return 0;
  99 }
 100
 101 /*
 102  * Trigger hardware to begin performing enqueued operations
 103  */
 104 static inline void
 105 rte_ioat_perform_ops(int dev_id)
 106 {
 107         struct rte_ioat_rawdev *ioat =
 108                         (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
 109         ioat->desc_ring[(ioat->next_write - 1) & (ioat->ring_size - 1)].u
 110                         .control.completion_update = 1;
 111         rte_compiler_barrier();
 112         ioat->regs->dmacount = ioat->next_write;
 113         ioat->started = ioat->enqueued;
 114 }
 115
 116 /**
 117  * @internal
 118  * Returns the index of the last completed operation.
 119  */
 120 static inline int
 121 rte_ioat_get_last_completed(struct rte_ioat_rawdev *ioat, int *error)
 122 {
 123         uint64_t status = ioat->status;
 124
 125         /* lower 3 bits indicate "transfer status" : active, idle, halted.
 126          * We can ignore bit 0.
 127          */
 128         *error = status & (RTE_IOAT_CHANSTS_SUSPENDED | RTE_IOAT_CHANSTS_ARMED);
 129         return (status - ioat->ring_addr) >> 6;
 130 }
 131
 132 /*
 133  * Returns details of operations that have been completed
 134  */
 135 static inline int
 136 rte_ioat_completed_ops(int dev_id, uint8_t max_copies,
 137                 uintptr_t *src_hdls, uintptr_t *dst_hdls)
 138 {
 139         struct rte_ioat_rawdev *ioat =
 140                         (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
 141         unsigned short mask = (ioat->ring_size - 1);
 142         unsigned short read = ioat->next_read;
 143         unsigned short end_read, count;
 144         int error;
 145         int i = 0;
 146
 147         end_read = (rte_ioat_get_last_completed(ioat, &error) + 1) & mask;
 148         count = (end_read - (read & mask)) & mask;
 149
 150         if (error) {
 151                 rte_errno = EIO;
 152                 return -1;
 153         }
 154
 155         if (ioat->hdls_disable) {
 156                 read += count;
 157                 goto end;
 158         }
 159
 160         if (count > max_copies)
 161                 count = max_copies;
 162
 163         for (; i < count - 1; i += 2, read += 2) {
 164                 __m128i hdls0 = _mm_load_si128(&ioat->hdls[read & mask]);
 165                 __m128i hdls1 = _mm_load_si128(&ioat->hdls[(read + 1) & mask]);
 166
 167                 _mm_storeu_si128((__m128i *)&src_hdls[i],
 168                                 _mm_unpacklo_epi64(hdls0, hdls1));
 169                 _mm_storeu_si128((__m128i *)&dst_hdls[i],
 170                                 _mm_unpackhi_epi64(hdls0, hdls1));
 171         }
 172         for (; i < count; i++, read++) {
 173                 uintptr_t *hdls = (uintptr_t *)&ioat->hdls[read & mask];
 174                 src_hdls[i] = hdls[0];
 175                 dst_hdls[i] = hdls[1];
 176         }
 177
 178 end:
 179         ioat->next_read = read;
 180         ioat->completed += count;
 181         return count;
 182 }
 183
 184 static inline void
 185 __rte_deprecated_msg("use rte_ioat_perform_ops() instead")
 186 rte_ioat_do_copies(int dev_id) { rte_ioat_perform_ops(dev_id); }
 187
 188 static inline int
 189 __rte_deprecated_msg("use rte_ioat_completed_ops() instead")
 190 rte_ioat_completed_copies(int dev_id, uint8_t max_copies,
 191                 uintptr_t *src_hdls, uintptr_t *dst_hdls)
 192 {
 193         return rte_ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls);
 194 }
 195
 196 #endif /* _RTE_IOAT_RAWDEV_FNS_H_ */