1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019-2020 Intel Corporation
4 #ifndef _RTE_IOAT_RAWDEV_FNS_H_
5 #define _RTE_IOAT_RAWDEV_FNS_H_
8 #include <rte_rawdev.h>
9 #include <rte_memzone.h>
10 #include <rte_prefetch.h>
14 * Structure representing a device descriptor
16 struct rte_ioat_generic_hw_desc {
21 uint32_t int_enable: 1;
22 uint32_t src_snoop_disable: 1;
23 uint32_t dest_snoop_disable: 1;
24 uint32_t completion_update: 1;
26 uint32_t reserved2: 1;
27 uint32_t src_page_break: 1;
28 uint32_t dest_page_break: 1;
32 uint32_t reserved: 13;
39 uint64_t op_specific[4];
44 * Identify the data path to use.
45 * Must be first field of rte_ioat_rawdev and rte_idxd_rawdev structs
47 enum rte_ioat_dev_type {
54 * some statistics for tracking, if added/changed update xstats fns
56 struct rte_ioat_xstats {
57 uint64_t enqueue_failed;
65 * Structure representing an IOAT device instance
67 struct rte_ioat_rawdev {
68 /* common fields at the top - match those in rte_idxd_rawdev */
69 enum rte_ioat_dev_type type;
70 struct rte_ioat_xstats xstats;
72 struct rte_rawdev *rawdev;
73 const struct rte_memzone *mz;
74 const struct rte_memzone *desc_mz;
76 volatile uint16_t *doorbell __rte_cache_aligned;
77 phys_addr_t status_addr;
78 phys_addr_t ring_addr;
80 unsigned short ring_size;
82 struct rte_ioat_generic_hw_desc *desc_ring;
83 __m128i *hdls; /* completion handles for returning to user */
86 unsigned short next_read;
87 unsigned short next_write;
89 /* to report completions, the device will write status back here */
90 volatile uint64_t status __rte_cache_aligned;
92 /* pointer to the register bar */
93 volatile struct rte_ioat_registers *regs;
96 #define RTE_IOAT_CHANSTS_IDLE 0x1
97 #define RTE_IOAT_CHANSTS_SUSPENDED 0x2
98 #define RTE_IOAT_CHANSTS_HALTED 0x3
99 #define RTE_IOAT_CHANSTS_ARMED 0x4
102 * Defines used in the data path for interacting with hardware.
104 #define IDXD_CMD_OP_SHIFT 24
113 #define IDXD_FLAG_FENCE (1 << 0)
114 #define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2)
115 #define IDXD_FLAG_REQUEST_COMPLETION (1 << 3)
116 #define IDXD_FLAG_CACHE_CONTROL (1 << 8)
118 #define IOAT_COMP_UPDATE_SHIFT 3
119 #define IOAT_CMD_OP_SHIFT 24
121 ioat_op_copy = 0, /* Standard DMA Operation */
122 ioat_op_fill /* Block Fill */
126 * Hardware descriptor used by DSA hardware, for both bursts and
127 * for individual operations.
129 struct rte_idxd_hw_desc {
132 rte_iova_t completion;
136 rte_iova_t src; /* source address for copy ops etc. */
137 rte_iova_t desc_addr; /* descriptor pointer for batch */
141 uint32_t size; /* length of data for op, or batch size */
143 /* 28 bytes of padding here */
147 * Completion record structure written back by DSA
149 struct rte_idxd_completion {
152 /* 16-bits pad here */
153 uint32_t completed_size; /* data length, or descriptors for batch */
155 rte_iova_t fault_address;
156 uint32_t invalid_flags;
159 #define BATCH_SIZE 64
162 * Structure used inside the driver for building up and submitting
163 * a batch of operations to the DSA hardware.
165 struct rte_idxd_desc_batch {
166 struct rte_idxd_completion comp; /* the completion record for batch */
172 struct rte_idxd_hw_desc batch_desc;
174 /* batches must always have 2 descriptors, so put a null at the start */
175 struct rte_idxd_hw_desc null_desc;
176 struct rte_idxd_hw_desc ops[BATCH_SIZE];
180 * structure used to save the "handles" provided by the user to be
181 * returned to the user on job completion.
183 struct rte_idxd_user_hdl {
190 * Structure representing an IDXD device instance
192 struct rte_idxd_rawdev {
193 enum rte_ioat_dev_type type;
194 struct rte_ioat_xstats xstats;
196 void *portal; /* address to write the batch descriptor */
198 /* counters to track the batches and the individual op handles */
199 uint16_t batch_ring_sz; /* size of batch ring */
200 uint16_t hdl_ring_sz; /* size of the user hdl ring */
202 uint16_t next_batch; /* where we write descriptor ops */
203 uint16_t next_completed; /* batch where we read completions */
204 uint16_t next_ret_hdl; /* the next user hdl to return */
205 uint16_t last_completed_hdl; /* the last user hdl that has completed */
206 uint16_t next_free_hdl; /* where the handle for next op will go */
207 uint16_t hdls_disable; /* disable tracking completion handles */
209 struct rte_idxd_user_hdl *hdl_ring;
210 struct rte_idxd_desc_batch *batch_ring;
213 static __rte_always_inline int
214 __ioat_write_desc(int dev_id, uint32_t op, uint64_t src, phys_addr_t dst,
215 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
217 struct rte_ioat_rawdev *ioat =
218 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
219 unsigned short read = ioat->next_read;
220 unsigned short write = ioat->next_write;
221 unsigned short mask = ioat->ring_size - 1;
222 unsigned short space = mask + read - write;
223 struct rte_ioat_generic_hw_desc *desc;
226 ioat->xstats.enqueue_failed++;
230 ioat->next_write = write + 1;
233 desc = &ioat->desc_ring[write];
235 /* set descriptor write-back every 16th descriptor */
236 desc->u.control_raw = (uint32_t)((op << IOAT_CMD_OP_SHIFT) |
237 (!(write & 0xF) << IOAT_COMP_UPDATE_SHIFT));
238 desc->src_addr = src;
239 desc->dest_addr = dst;
241 if (!ioat->hdls_disable)
242 ioat->hdls[write] = _mm_set_epi64x((int64_t)dst_hdl,
244 rte_prefetch0(&ioat->desc_ring[ioat->next_write & mask]);
246 ioat->xstats.enqueued++;
250 static __rte_always_inline int
251 __ioat_enqueue_fill(int dev_id, uint64_t pattern, phys_addr_t dst,
252 unsigned int length, uintptr_t dst_hdl)
254 static const uintptr_t null_hdl;
256 return __ioat_write_desc(dev_id, ioat_op_fill, pattern, dst, length,
261 * Enqueue a copy operation onto the ioat device
263 static __rte_always_inline int
264 __ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
265 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
267 return __ioat_write_desc(dev_id, ioat_op_copy, src, dst, length,
271 /* add fence to last written descriptor */
272 static __rte_always_inline int
273 __ioat_fence(int dev_id)
275 struct rte_ioat_rawdev *ioat =
276 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
277 unsigned short write = ioat->next_write;
278 unsigned short mask = ioat->ring_size - 1;
279 struct rte_ioat_generic_hw_desc *desc;
281 write = (write - 1) & mask;
282 desc = &ioat->desc_ring[write];
284 desc->u.control.fence = 1;
289 * Trigger hardware to begin performing enqueued operations
291 static __rte_always_inline void
292 __ioat_perform_ops(int dev_id)
294 struct rte_ioat_rawdev *ioat =
295 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
296 ioat->desc_ring[(ioat->next_write - 1) & (ioat->ring_size - 1)].u
297 .control.completion_update = 1;
298 rte_compiler_barrier();
299 *ioat->doorbell = ioat->next_write;
300 ioat->xstats.started = ioat->xstats.enqueued;
305 * Returns the index of the last completed operation.
307 static __rte_always_inline int
308 __ioat_get_last_completed(struct rte_ioat_rawdev *ioat, int *error)
310 uint64_t status = ioat->status;
312 /* lower 3 bits indicate "transfer status" : active, idle, halted.
313 * We can ignore bit 0.
315 *error = status & (RTE_IOAT_CHANSTS_SUSPENDED | RTE_IOAT_CHANSTS_ARMED);
316 return (status - ioat->ring_addr) >> 6;
320 * Returns details of operations that have been completed
322 static __rte_always_inline int
323 __ioat_completed_ops(int dev_id, uint8_t max_copies,
324 uintptr_t *src_hdls, uintptr_t *dst_hdls)
326 struct rte_ioat_rawdev *ioat =
327 (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
328 unsigned short mask = (ioat->ring_size - 1);
329 unsigned short read = ioat->next_read;
330 unsigned short end_read, count;
334 end_read = (__ioat_get_last_completed(ioat, &error) + 1) & mask;
335 count = (end_read - (read & mask)) & mask;
342 if (ioat->hdls_disable) {
347 if (count > max_copies)
350 for (; i < count - 1; i += 2, read += 2) {
351 __m128i hdls0 = _mm_load_si128(&ioat->hdls[read & mask]);
352 __m128i hdls1 = _mm_load_si128(&ioat->hdls[(read + 1) & mask]);
354 _mm_storeu_si128((__m128i *)&src_hdls[i],
355 _mm_unpacklo_epi64(hdls0, hdls1));
356 _mm_storeu_si128((__m128i *)&dst_hdls[i],
357 _mm_unpackhi_epi64(hdls0, hdls1));
359 for (; i < count; i++, read++) {
360 uintptr_t *hdls = (uintptr_t *)&ioat->hdls[read & mask];
361 src_hdls[i] = hdls[0];
362 dst_hdls[i] = hdls[1];
366 ioat->next_read = read;
367 ioat->xstats.completed += count;
371 static __rte_always_inline int
372 __idxd_write_desc(int dev_id, const struct rte_idxd_hw_desc *desc,
373 const struct rte_idxd_user_hdl *hdl)
375 struct rte_idxd_rawdev *idxd =
376 (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private;
377 struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch];
379 /* check for room in the handle ring */
380 if (((idxd->next_free_hdl + 1) & (idxd->hdl_ring_sz - 1)) == idxd->next_ret_hdl)
383 /* check for space in current batch */
384 if (b->op_count >= BATCH_SIZE)
387 /* check that we can actually use the current batch */
391 /* write the descriptor */
392 b->ops[b->op_count++] = *desc;
394 /* store the completion details */
395 if (!idxd->hdls_disable)
396 idxd->hdl_ring[idxd->next_free_hdl] = *hdl;
397 if (++idxd->next_free_hdl == idxd->hdl_ring_sz)
398 idxd->next_free_hdl = 0;
400 idxd->xstats.enqueued++;
404 idxd->xstats.enqueue_failed++;
409 static __rte_always_inline int
410 __idxd_enqueue_fill(int dev_id, uint64_t pattern, rte_iova_t dst,
411 unsigned int length, uintptr_t dst_hdl)
413 const struct rte_idxd_hw_desc desc = {
414 .op_flags = (idxd_op_fill << IDXD_CMD_OP_SHIFT) |
415 IDXD_FLAG_CACHE_CONTROL,
420 const struct rte_idxd_user_hdl hdl = {
423 return __idxd_write_desc(dev_id, &desc, &hdl);
426 static __rte_always_inline int
427 __idxd_enqueue_copy(int dev_id, rte_iova_t src, rte_iova_t dst,
428 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
430 const struct rte_idxd_hw_desc desc = {
431 .op_flags = (idxd_op_memmove << IDXD_CMD_OP_SHIFT) |
432 IDXD_FLAG_CACHE_CONTROL,
437 const struct rte_idxd_user_hdl hdl = {
441 return __idxd_write_desc(dev_id, &desc, &hdl);
444 static __rte_always_inline int
445 __idxd_fence(int dev_id)
447 static const struct rte_idxd_hw_desc fence = {
448 .op_flags = IDXD_FLAG_FENCE
450 static const struct rte_idxd_user_hdl null_hdl;
451 return __idxd_write_desc(dev_id, &fence, &null_hdl);
454 static __rte_always_inline void
455 __idxd_movdir64b(volatile void *dst, const void *src)
457 asm volatile (".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
459 : "a" (dst), "d" (src));
462 static __rte_always_inline void
463 __idxd_perform_ops(int dev_id)
465 struct rte_idxd_rawdev *idxd =
466 (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private;
467 struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch];
469 if (b->submitted || b->op_count == 0)
471 b->hdl_end = idxd->next_free_hdl;
474 b->batch_desc.size = b->op_count + 1;
475 __idxd_movdir64b(idxd->portal, &b->batch_desc);
477 if (++idxd->next_batch == idxd->batch_ring_sz)
478 idxd->next_batch = 0;
479 idxd->xstats.started = idxd->xstats.enqueued;
482 static __rte_always_inline int
483 __idxd_completed_ops(int dev_id, uint8_t max_ops,
484 uintptr_t *src_hdls, uintptr_t *dst_hdls)
486 struct rte_idxd_rawdev *idxd =
487 (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private;
488 struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_completed];
489 uint16_t h_idx = idxd->next_ret_hdl;
492 while (b->submitted && b->comp.status != 0) {
493 idxd->last_completed_hdl = b->hdl_end;
496 if (++idxd->next_completed == idxd->batch_ring_sz)
497 idxd->next_completed = 0;
498 b = &idxd->batch_ring[idxd->next_completed];
501 if (!idxd->hdls_disable)
502 for (n = 0; n < max_ops && h_idx != idxd->last_completed_hdl; n++) {
503 src_hdls[n] = idxd->hdl_ring[h_idx].src;
504 dst_hdls[n] = idxd->hdl_ring[h_idx].dst;
505 if (++h_idx == idxd->hdl_ring_sz)
509 while (h_idx != idxd->last_completed_hdl) {
511 if (++h_idx == idxd->hdl_ring_sz)
515 idxd->next_ret_hdl = h_idx;
517 idxd->xstats.completed += n;
522 rte_ioat_enqueue_fill(int dev_id, uint64_t pattern, phys_addr_t dst,
523 unsigned int len, uintptr_t dst_hdl)
525 enum rte_ioat_dev_type *type =
526 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
527 if (*type == RTE_IDXD_DEV)
528 return __idxd_enqueue_fill(dev_id, pattern, dst, len, dst_hdl);
530 return __ioat_enqueue_fill(dev_id, pattern, dst, len, dst_hdl);
534 rte_ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
535 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
537 enum rte_ioat_dev_type *type =
538 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
539 if (*type == RTE_IDXD_DEV)
540 return __idxd_enqueue_copy(dev_id, src, dst, length,
543 return __ioat_enqueue_copy(dev_id, src, dst, length,
548 rte_ioat_fence(int dev_id)
550 enum rte_ioat_dev_type *type =
551 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
552 if (*type == RTE_IDXD_DEV)
553 return __idxd_fence(dev_id);
555 return __ioat_fence(dev_id);
559 rte_ioat_perform_ops(int dev_id)
561 enum rte_ioat_dev_type *type =
562 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
563 if (*type == RTE_IDXD_DEV)
564 return __idxd_perform_ops(dev_id);
566 return __ioat_perform_ops(dev_id);
570 rte_ioat_completed_ops(int dev_id, uint8_t max_copies,
571 uintptr_t *src_hdls, uintptr_t *dst_hdls)
573 enum rte_ioat_dev_type *type =
574 (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
575 if (*type == RTE_IDXD_DEV)
576 return __idxd_completed_ops(dev_id, max_copies,
579 return __ioat_completed_ops(dev_id, max_copies,
584 __rte_deprecated_msg("use rte_ioat_perform_ops() instead")
585 rte_ioat_do_copies(int dev_id) { rte_ioat_perform_ops(dev_id); }
588 __rte_deprecated_msg("use rte_ioat_completed_ops() instead")
589 rte_ioat_completed_copies(int dev_id, uint8_t max_copies,
590 uintptr_t *src_hdls, uintptr_t *dst_hdls)
592 return rte_ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls);
595 #endif /* _RTE_IOAT_RAWDEV_FNS_H_ */