1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019-2020 Intel Corporation
4 #ifndef _RTE_IOAT_RAWDEV_FNS_H_
5 #define _RTE_IOAT_RAWDEV_FNS_H_
9 * This header file contains the implementation of the various ioat
10 * rawdev functions for IOAT/CBDMA hardware. The API specification and key
11 * public structures are defined in "rte_ioat_rawdev.h".
13 * This file should not be included directly, but instead applications should
14 * include "rte_ioat_rawdev.h", which then includes this file - and the IDXD/DSA
15 * equivalent header - in turn.
18 #include <x86intrin.h>
19 #include <rte_rawdev.h>
20 #include <rte_memzone.h>
21 #include <rte_prefetch.h>
25 * Identify the data path to use.
26 * Must be first field of rte_ioat_rawdev and rte_idxd_rawdev structs
28 enum rte_ioat_dev_type {
35 * some statistics for tracking, if added/changed update xstats fns
37 struct rte_ioat_xstats {
38 uint64_t enqueue_failed;
44 #include "rte_idxd_rawdev_fns.h"
48 * Structure representing a device descriptor
50 struct rte_ioat_generic_hw_desc {
55 uint32_t int_enable: 1;
56 uint32_t src_snoop_disable: 1;
57 uint32_t dest_snoop_disable: 1;
58 uint32_t completion_update: 1;
60 uint32_t reserved2: 1;
61 uint32_t src_page_break: 1;
62 uint32_t dest_page_break: 1;
66 uint32_t reserved: 13;
73 uint64_t op_specific[4];
78 * Structure representing an IOAT device instance
80 struct rte_ioat_rawdev {
81 /* common fields at the top - match those in rte_idxd_rawdev */
82 enum rte_ioat_dev_type type;
83 struct rte_ioat_xstats xstats;
85 struct rte_rawdev *rawdev;
86 const struct rte_memzone *mz;
87 const struct rte_memzone *desc_mz;
89 volatile uint16_t *doorbell __rte_cache_aligned;
90 phys_addr_t status_addr;
91 phys_addr_t ring_addr;
93 unsigned short ring_size;
95 struct rte_ioat_generic_hw_desc *desc_ring;
96 __m128i *hdls; /* completion handles for returning to user */
99 unsigned short next_read;
100 unsigned short next_write;
102 /* to report completions, the device will write status back here */
103 volatile uint64_t status __rte_cache_aligned;
105 /* pointer to the register bar */
106 volatile struct rte_ioat_registers *regs;
109 #define RTE_IOAT_CHANSTS_IDLE 0x1
110 #define RTE_IOAT_CHANSTS_SUSPENDED 0x2
111 #define RTE_IOAT_CHANSTS_HALTED 0x3
112 #define RTE_IOAT_CHANSTS_ARMED 0x4
114 static __rte_always_inline int
115 __ioat_write_desc(int dev_id, uint32_t op, uint64_t src, phys_addr_t dst,
116 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
118 struct rte_ioat_rawdev *ioat =
119 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
120 unsigned short read = ioat->next_read;
121 unsigned short write = ioat->next_write;
122 unsigned short mask = ioat->ring_size - 1;
123 unsigned short space = mask + read - write;
124 struct rte_ioat_generic_hw_desc *desc;
127 ioat->xstats.enqueue_failed++;
131 ioat->next_write = write + 1;
134 desc = &ioat->desc_ring[write];
136 /* set descriptor write-back every 16th descriptor */
137 desc->u.control_raw = (uint32_t)((op << IOAT_CMD_OP_SHIFT) |
138 (!(write & 0xF) << IOAT_COMP_UPDATE_SHIFT));
139 desc->src_addr = src;
140 desc->dest_addr = dst;
142 if (!ioat->hdls_disable)
143 ioat->hdls[write] = _mm_set_epi64x((int64_t)dst_hdl,
145 rte_prefetch0(&ioat->desc_ring[ioat->next_write & mask]);
147 ioat->xstats.enqueued++;
151 static __rte_always_inline int
152 __ioat_enqueue_fill(int dev_id, uint64_t pattern, phys_addr_t dst,
153 unsigned int length, uintptr_t dst_hdl)
155 static const uintptr_t null_hdl;
157 return __ioat_write_desc(dev_id, ioat_op_fill, pattern, dst, length,
162 * Enqueue a copy operation onto the ioat device
164 static __rte_always_inline int
165 __ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
166 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
168 return __ioat_write_desc(dev_id, ioat_op_copy, src, dst, length,
172 /* add fence to last written descriptor */
173 static __rte_always_inline int
174 __ioat_fence(int dev_id)
176 struct rte_ioat_rawdev *ioat =
177 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
178 unsigned short write = ioat->next_write;
179 unsigned short mask = ioat->ring_size - 1;
180 struct rte_ioat_generic_hw_desc *desc;
182 write = (write - 1) & mask;
183 desc = &ioat->desc_ring[write];
185 desc->u.control.fence = 1;
190 * Trigger hardware to begin performing enqueued operations
192 static __rte_always_inline int
193 __ioat_perform_ops(int dev_id)
195 struct rte_ioat_rawdev *ioat =
196 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
197 ioat->desc_ring[(ioat->next_write - 1) & (ioat->ring_size - 1)].u
198 .control.completion_update = 1;
199 rte_compiler_barrier();
200 *ioat->doorbell = ioat->next_write;
201 ioat->xstats.started = ioat->xstats.enqueued;
208 * Returns the index of the last completed operation.
210 static __rte_always_inline int
211 __ioat_get_last_completed(struct rte_ioat_rawdev *ioat, int *error)
213 uint64_t status = ioat->status;
215 /* lower 3 bits indicate "transfer status" : active, idle, halted.
216 * We can ignore bit 0.
218 *error = status & (RTE_IOAT_CHANSTS_SUSPENDED | RTE_IOAT_CHANSTS_ARMED);
219 return (status - ioat->ring_addr) >> 6;
223 * Returns details of operations that have been completed
225 static __rte_always_inline int
226 __ioat_completed_ops(int dev_id, uint8_t max_copies,
227 uintptr_t *src_hdls, uintptr_t *dst_hdls)
229 struct rte_ioat_rawdev *ioat =
230 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
231 unsigned short mask = (ioat->ring_size - 1);
232 unsigned short read = ioat->next_read;
233 unsigned short end_read, count;
237 end_read = (__ioat_get_last_completed(ioat, &error) + 1) & mask;
238 count = (end_read - (read & mask)) & mask;
245 if (ioat->hdls_disable) {
250 if (count > max_copies)
253 for (; i < count - 1; i += 2, read += 2) {
254 __m128i hdls0 = _mm_load_si128(&ioat->hdls[read & mask]);
255 __m128i hdls1 = _mm_load_si128(&ioat->hdls[(read + 1) & mask]);
257 _mm_storeu_si128((__m128i *)&src_hdls[i],
258 _mm_unpacklo_epi64(hdls0, hdls1));
259 _mm_storeu_si128((__m128i *)&dst_hdls[i],
260 _mm_unpackhi_epi64(hdls0, hdls1));
262 for (; i < count; i++, read++) {
263 uintptr_t *hdls = (uintptr_t *)&ioat->hdls[read & mask];
264 src_hdls[i] = hdls[0];
265 dst_hdls[i] = hdls[1];
269 ioat->next_read = read;
270 ioat->xstats.completed += count;
275 rte_ioat_enqueue_fill(int dev_id, uint64_t pattern, phys_addr_t dst,
276 unsigned int len, uintptr_t dst_hdl)
278 enum rte_ioat_dev_type *type =
279 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
280 if (*type == RTE_IDXD_DEV)
281 return __idxd_enqueue_fill(dev_id, pattern, dst, len, dst_hdl);
283 return __ioat_enqueue_fill(dev_id, pattern, dst, len, dst_hdl);
287 rte_ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
288 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
290 enum rte_ioat_dev_type *type =
291 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
292 if (*type == RTE_IDXD_DEV)
293 return __idxd_enqueue_copy(dev_id, src, dst, length,
296 return __ioat_enqueue_copy(dev_id, src, dst, length,
301 rte_ioat_fence(int dev_id)
303 enum rte_ioat_dev_type *type =
304 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
305 if (*type == RTE_IDXD_DEV)
306 return __idxd_fence(dev_id);
308 return __ioat_fence(dev_id);
312 rte_ioat_perform_ops(int dev_id)
314 enum rte_ioat_dev_type *type =
315 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
316 if (*type == RTE_IDXD_DEV)
317 return __idxd_perform_ops(dev_id);
319 return __ioat_perform_ops(dev_id);
323 rte_ioat_completed_ops(int dev_id, uint8_t max_copies,
324 uintptr_t *src_hdls, uintptr_t *dst_hdls)
326 enum rte_ioat_dev_type *type =
327 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
328 if (*type == RTE_IDXD_DEV)
329 return __idxd_completed_ops(dev_id, max_copies,
332 return __ioat_completed_ops(dev_id, max_copies,
337 __rte_deprecated_msg("use rte_ioat_perform_ops() instead")
338 rte_ioat_do_copies(int dev_id) { rte_ioat_perform_ops(dev_id); }
341 __rte_deprecated_msg("use rte_ioat_completed_ops() instead")
342 rte_ioat_completed_copies(int dev_id, uint8_t max_copies,
343 uintptr_t *src_hdls, uintptr_t *dst_hdls)
345 return rte_ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls);
348 #endif /* _RTE_IOAT_RAWDEV_FNS_H_ */