drivers/raw/ioat/rte_ioat_rawdev_fns.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019-2020 Intel Corporation
   3  */
   4 #ifndef _RTE_IOAT_RAWDEV_FNS_H_
   5 #define _RTE_IOAT_RAWDEV_FNS_H_
   6
   7 #include <x86intrin.h>
   8 #include <rte_rawdev.h>
   9 #include <rte_memzone.h>
  10 #include <rte_prefetch.h>
  11
  12 /**
  13  * @internal
  14  * Structure representing a device descriptor
  15  */
  16 struct rte_ioat_generic_hw_desc {
  17         uint32_t size;
  18         union {
  19                 uint32_t control_raw;
  20                 struct {
  21                         uint32_t int_enable: 1;
  22                         uint32_t src_snoop_disable: 1;
  23                         uint32_t dest_snoop_disable: 1;
  24                         uint32_t completion_update: 1;
  25                         uint32_t fence: 1;
  26                         uint32_t reserved2: 1;
  27                         uint32_t src_page_break: 1;
  28                         uint32_t dest_page_break: 1;
  29                         uint32_t bundle: 1;
  30                         uint32_t dest_dca: 1;
  31                         uint32_t hint: 1;
  32                         uint32_t reserved: 13;
  33                         uint32_t op: 8;
  34                 } control;
  35         } u;
  36         uint64_t src_addr;
  37         uint64_t dest_addr;
  38         uint64_t next;
  39         uint64_t op_specific[4];
  40 };
  41
  42 /**
  43  * @internal
  44  * Identify the data path to use.
  45  * Must be first field of rte_ioat_rawdev and rte_idxd_rawdev structs
  46  */
  47 enum rte_ioat_dev_type {
  48         RTE_IOAT_DEV,
  49         RTE_IDXD_DEV,
  50 };
  51
  52 /**
  53  * @internal
  54  * some statistics for tracking, if added/changed update xstats fns
  55  */
  56 struct rte_ioat_xstats {
  57         uint64_t enqueue_failed;
  58         uint64_t enqueued;
  59         uint64_t started;
  60         uint64_t completed;
  61 };
  62
  63 /**
  64  * @internal
  65  * Structure representing an IOAT device instance
  66  */
  67 struct rte_ioat_rawdev {
  68         /* common fields at the top - match those in rte_idxd_rawdev */
  69         enum rte_ioat_dev_type type;
  70         struct rte_ioat_xstats xstats;
  71
  72         struct rte_rawdev *rawdev;
  73         const struct rte_memzone *mz;
  74         const struct rte_memzone *desc_mz;
  75
  76         volatile uint16_t *doorbell __rte_cache_aligned;
  77         phys_addr_t status_addr;
  78         phys_addr_t ring_addr;
  79
  80         unsigned short ring_size;
  81         bool hdls_disable;
  82         struct rte_ioat_generic_hw_desc *desc_ring;
  83         __m128i *hdls; /* completion handles for returning to user */
  84
  85
  86         unsigned short next_read;
  87         unsigned short next_write;
  88
  89         /* to report completions, the device will write status back here */
  90         volatile uint64_t status __rte_cache_aligned;
  91
  92         /* pointer to the register bar */
  93         volatile struct rte_ioat_registers *regs;
  94 };
  95
  96 #define RTE_IOAT_CHANSTS_IDLE                   0x1
  97 #define RTE_IOAT_CHANSTS_SUSPENDED              0x2
  98 #define RTE_IOAT_CHANSTS_HALTED                 0x3
  99 #define RTE_IOAT_CHANSTS_ARMED                  0x4
 100
 101 /*
 102  * Defines used in the data path for interacting with hardware.
 103  */
 104 #define IDXD_CMD_OP_SHIFT 24
 105 enum rte_idxd_ops {
 106         idxd_op_nop = 0,
 107         idxd_op_batch,
 108         idxd_op_drain,
 109         idxd_op_memmove,
 110         idxd_op_fill
 111 };
 112
 113 #define IDXD_FLAG_FENCE                 (1 << 0)
 114 #define IDXD_FLAG_COMPLETION_ADDR_VALID (1 << 2)
 115 #define IDXD_FLAG_REQUEST_COMPLETION    (1 << 3)
 116 #define IDXD_FLAG_CACHE_CONTROL         (1 << 8)
 117
 118 /**
 119  * Hardware descriptor used by DSA hardware, for both bursts and
 120  * for individual operations.
 121  */
 122 struct rte_idxd_hw_desc {
 123         uint32_t pasid;
 124         uint32_t op_flags;
 125         rte_iova_t completion;
 126
 127         RTE_STD_C11
 128         union {
 129                 rte_iova_t src;      /* source address for copy ops etc. */
 130                 rte_iova_t desc_addr; /* descriptor pointer for batch */
 131         };
 132         rte_iova_t dst;
 133
 134         uint32_t size;    /* length of data for op, or batch size */
 135
 136         /* 28 bytes of padding here */
 137 } __rte_aligned(64);
 138
 139 /**
 140  * Completion record structure written back by DSA
 141  */
 142 struct rte_idxd_completion {
 143         uint8_t status;
 144         uint8_t result;
 145         /* 16-bits pad here */
 146         uint32_t completed_size; /* data length, or descriptors for batch */
 147
 148         rte_iova_t fault_address;
 149         uint32_t invalid_flags;
 150 } __rte_aligned(32);
 151
 152 #define BATCH_SIZE 64
 153
 154 /**
 155  * Structure used inside the driver for building up and submitting
 156  * a batch of operations to the DSA hardware.
 157  */
 158 struct rte_idxd_desc_batch {
 159         struct rte_idxd_completion comp; /* the completion record for batch */
 160
 161         uint16_t submitted;
 162         uint16_t op_count;
 163         uint16_t hdl_end;
 164
 165         struct rte_idxd_hw_desc batch_desc;
 166
 167         /* batches must always have 2 descriptors, so put a null at the start */
 168         struct rte_idxd_hw_desc null_desc;
 169         struct rte_idxd_hw_desc ops[BATCH_SIZE];
 170 };
 171
 172 /**
 173  * structure used to save the "handles" provided by the user to be
 174  * returned to the user on job completion.
 175  */
 176 struct rte_idxd_user_hdl {
 177         uint64_t src;
 178         uint64_t dst;
 179 };
 180
 181 /**
 182  * @internal
 183  * Structure representing an IDXD device instance
 184  */
 185 struct rte_idxd_rawdev {
 186         enum rte_ioat_dev_type type;
 187         void *portal; /* address to write the batch descriptor */
 188
 189         /* counters to track the batches and the individual op handles */
 190         uint16_t batch_ring_sz;  /* size of batch ring */
 191         uint16_t hdl_ring_sz;    /* size of the user hdl ring */
 192
 193         uint16_t next_batch;     /* where we write descriptor ops */
 194         uint16_t next_completed; /* batch where we read completions */
 195         uint16_t next_ret_hdl;   /* the next user hdl to return */
 196         uint16_t last_completed_hdl; /* the last user hdl that has completed */
 197         uint16_t next_free_hdl;  /* where the handle for next op will go */
 198         uint16_t hdls_disable;   /* disable tracking completion handles */
 199
 200         struct rte_idxd_user_hdl *hdl_ring;
 201         struct rte_idxd_desc_batch *batch_ring;
 202 };
 203
 204 /*
 205  * Enqueue a copy operation onto the ioat device
 206  */
 207 static __rte_always_inline int
 208 __ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
 209                 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
 210 {
 211         struct rte_ioat_rawdev *ioat =
 212                         (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
 213         unsigned short read = ioat->next_read;
 214         unsigned short write = ioat->next_write;
 215         unsigned short mask = ioat->ring_size - 1;
 216         unsigned short space = mask + read - write;
 217         struct rte_ioat_generic_hw_desc *desc;
 218
 219         if (space == 0) {
 220                 ioat->xstats.enqueue_failed++;
 221                 return 0;
 222         }
 223
 224         ioat->next_write = write + 1;
 225         write &= mask;
 226
 227         desc = &ioat->desc_ring[write];
 228         desc->size = length;
 229         /* set descriptor write-back every 16th descriptor */
 230         desc->u.control_raw = (uint32_t)((!(write & 0xF)) << 3);
 231         desc->src_addr = src;
 232         desc->dest_addr = dst;
 233
 234         if (!ioat->hdls_disable)
 235                 ioat->hdls[write] = _mm_set_epi64x((int64_t)dst_hdl,
 236                                         (int64_t)src_hdl);
 237         rte_prefetch0(&ioat->desc_ring[ioat->next_write & mask]);
 238
 239         ioat->xstats.enqueued++;
 240         return 1;
 241 }
 242
 243 /* add fence to last written descriptor */
 244 static __rte_always_inline int
 245 __ioat_fence(int dev_id)
 246 {
 247         struct rte_ioat_rawdev *ioat =
 248                         (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
 249         unsigned short write = ioat->next_write;
 250         unsigned short mask = ioat->ring_size - 1;
 251         struct rte_ioat_generic_hw_desc *desc;
 252
 253         write = (write - 1) & mask;
 254         desc = &ioat->desc_ring[write];
 255
 256         desc->u.control.fence = 1;
 257         return 0;
 258 }
 259
 260 /*
 261  * Trigger hardware to begin performing enqueued operations
 262  */
 263 static __rte_always_inline void
 264 __ioat_perform_ops(int dev_id)
 265 {
 266         struct rte_ioat_rawdev *ioat =
 267                         (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
 268         ioat->desc_ring[(ioat->next_write - 1) & (ioat->ring_size - 1)].u
 269                         .control.completion_update = 1;
 270         rte_compiler_barrier();
 271         *ioat->doorbell = ioat->next_write;
 272         ioat->xstats.started = ioat->xstats.enqueued;
 273 }
 274
 275 /**
 276  * @internal
 277  * Returns the index of the last completed operation.
 278  */
 279 static __rte_always_inline int
 280 __ioat_get_last_completed(struct rte_ioat_rawdev *ioat, int *error)
 281 {
 282         uint64_t status = ioat->status;
 283
 284         /* lower 3 bits indicate "transfer status" : active, idle, halted.
 285          * We can ignore bit 0.
 286          */
 287         *error = status & (RTE_IOAT_CHANSTS_SUSPENDED | RTE_IOAT_CHANSTS_ARMED);
 288         return (status - ioat->ring_addr) >> 6;
 289 }
 290
 291 /*
 292  * Returns details of operations that have been completed
 293  */
 294 static __rte_always_inline int
 295 __ioat_completed_ops(int dev_id, uint8_t max_copies,
 296                 uintptr_t *src_hdls, uintptr_t *dst_hdls)
 297 {
 298         struct rte_ioat_rawdev *ioat =
 299                         (struct rte_ioat_rawdev *)rte_rawdevs[dev_id].dev_private;
 300         unsigned short mask = (ioat->ring_size - 1);
 301         unsigned short read = ioat->next_read;
 302         unsigned short end_read, count;
 303         int error;
 304         int i = 0;
 305
 306         end_read = (__ioat_get_last_completed(ioat, &error) + 1) & mask;
 307         count = (end_read - (read & mask)) & mask;
 308
 309         if (error) {
 310                 rte_errno = EIO;
 311                 return -1;
 312         }
 313
 314         if (ioat->hdls_disable) {
 315                 read += count;
 316                 goto end;
 317         }
 318
 319         if (count > max_copies)
 320                 count = max_copies;
 321
 322         for (; i < count - 1; i += 2, read += 2) {
 323                 __m128i hdls0 = _mm_load_si128(&ioat->hdls[read & mask]);
 324                 __m128i hdls1 = _mm_load_si128(&ioat->hdls[(read + 1) & mask]);
 325
 326                 _mm_storeu_si128((__m128i *)&src_hdls[i],
 327                                 _mm_unpacklo_epi64(hdls0, hdls1));
 328                 _mm_storeu_si128((__m128i *)&dst_hdls[i],
 329                                 _mm_unpackhi_epi64(hdls0, hdls1));
 330         }
 331         for (; i < count; i++, read++) {
 332                 uintptr_t *hdls = (uintptr_t *)&ioat->hdls[read & mask];
 333                 src_hdls[i] = hdls[0];
 334                 dst_hdls[i] = hdls[1];
 335         }
 336
 337 end:
 338         ioat->next_read = read;
 339         ioat->xstats.completed += count;
 340         return count;
 341 }
 342
 343 static __rte_always_inline int
 344 __idxd_write_desc(int dev_id, const struct rte_idxd_hw_desc *desc,
 345                 const struct rte_idxd_user_hdl *hdl)
 346 {
 347         struct rte_idxd_rawdev *idxd =
 348                         (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private;
 349         struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch];
 350
 351         /* check for room in the handle ring */
 352         if (((idxd->next_free_hdl + 1) & (idxd->hdl_ring_sz - 1)) == idxd->next_ret_hdl)
 353                 goto failed;
 354
 355         /* check for space in current batch */
 356         if (b->op_count >= BATCH_SIZE)
 357                 goto failed;
 358
 359         /* check that we can actually use the current batch */
 360         if (b->submitted)
 361                 goto failed;
 362
 363         /* write the descriptor */
 364         b->ops[b->op_count++] = *desc;
 365
 366         /* store the completion details */
 367         if (!idxd->hdls_disable)
 368                 idxd->hdl_ring[idxd->next_free_hdl] = *hdl;
 369         if (++idxd->next_free_hdl == idxd->hdl_ring_sz)
 370                 idxd->next_free_hdl = 0;
 371
 372         return 1;
 373
 374 failed:
 375         rte_errno = ENOSPC;
 376         return 0;
 377 }
 378
 379 static __rte_always_inline int
 380 __idxd_enqueue_copy(int dev_id, rte_iova_t src, rte_iova_t dst,
 381                 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
 382 {
 383         const struct rte_idxd_hw_desc desc = {
 384                         .op_flags =  (idxd_op_memmove << IDXD_CMD_OP_SHIFT) |
 385                                 IDXD_FLAG_CACHE_CONTROL,
 386                         .src = src,
 387                         .dst = dst,
 388                         .size = length
 389         };
 390         const struct rte_idxd_user_hdl hdl = {
 391                         .src = src_hdl,
 392                         .dst = dst_hdl
 393         };
 394         return __idxd_write_desc(dev_id, &desc, &hdl);
 395 }
 396
 397 static __rte_always_inline int
 398 __idxd_fence(int dev_id)
 399 {
 400         static const struct rte_idxd_hw_desc fence = {
 401                         .op_flags = IDXD_FLAG_FENCE
 402         };
 403         static const struct rte_idxd_user_hdl null_hdl;
 404         return __idxd_write_desc(dev_id, &fence, &null_hdl);
 405 }
 406
 407 static __rte_always_inline void
 408 __idxd_movdir64b(volatile void *dst, const void *src)
 409 {
 410         asm volatile (".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
 411                         :
 412                         : "a" (dst), "d" (src));
 413 }
 414
 415 static __rte_always_inline void
 416 __idxd_perform_ops(int dev_id)
 417 {
 418         struct rte_idxd_rawdev *idxd =
 419                         (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private;
 420         struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_batch];
 421
 422         if (b->submitted || b->op_count == 0)
 423                 return;
 424         b->hdl_end = idxd->next_free_hdl;
 425         b->comp.status = 0;
 426         b->submitted = 1;
 427         b->batch_desc.size = b->op_count + 1;
 428         __idxd_movdir64b(idxd->portal, &b->batch_desc);
 429
 430         if (++idxd->next_batch == idxd->batch_ring_sz)
 431                 idxd->next_batch = 0;
 432 }
 433
 434 static __rte_always_inline int
 435 __idxd_completed_ops(int dev_id, uint8_t max_ops,
 436                 uintptr_t *src_hdls, uintptr_t *dst_hdls)
 437 {
 438         struct rte_idxd_rawdev *idxd =
 439                         (struct rte_idxd_rawdev *)rte_rawdevs[dev_id].dev_private;
 440         struct rte_idxd_desc_batch *b = &idxd->batch_ring[idxd->next_completed];
 441         uint16_t h_idx = idxd->next_ret_hdl;
 442         int n = 0;
 443
 444         while (b->submitted && b->comp.status != 0) {
 445                 idxd->last_completed_hdl = b->hdl_end;
 446                 b->submitted = 0;
 447                 b->op_count = 0;
 448                 if (++idxd->next_completed == idxd->batch_ring_sz)
 449                         idxd->next_completed = 0;
 450                 b = &idxd->batch_ring[idxd->next_completed];
 451         }
 452
 453         if (!idxd->hdls_disable)
 454                 for (n = 0; n < max_ops && h_idx != idxd->last_completed_hdl; n++) {
 455                         src_hdls[n] = idxd->hdl_ring[h_idx].src;
 456                         dst_hdls[n] = idxd->hdl_ring[h_idx].dst;
 457                         if (++h_idx == idxd->hdl_ring_sz)
 458                                 h_idx = 0;
 459                 }
 460         else
 461                 while (h_idx != idxd->last_completed_hdl) {
 462                         n++;
 463                         if (++h_idx == idxd->hdl_ring_sz)
 464                                 h_idx = 0;
 465                 }
 466
 467         idxd->next_ret_hdl = h_idx;
 468
 469         return n;
 470 }
 471
 472 static inline int
 473 rte_ioat_enqueue_copy(int dev_id, phys_addr_t src, phys_addr_t dst,
 474                 unsigned int length, uintptr_t src_hdl, uintptr_t dst_hdl)
 475 {
 476         enum rte_ioat_dev_type *type =
 477                         (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
 478         if (*type == RTE_IDXD_DEV)
 479                 return __idxd_enqueue_copy(dev_id, src, dst, length,
 480                                 src_hdl, dst_hdl);
 481         else
 482                 return __ioat_enqueue_copy(dev_id, src, dst, length,
 483                                 src_hdl, dst_hdl);
 484 }
 485
 486 static inline int
 487 rte_ioat_fence(int dev_id)
 488 {
 489         enum rte_ioat_dev_type *type =
 490                         (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
 491         if (*type == RTE_IDXD_DEV)
 492                 return __idxd_fence(dev_id);
 493         else
 494                 return __ioat_fence(dev_id);
 495 }
 496
 497 static inline void
 498 rte_ioat_perform_ops(int dev_id)
 499 {
 500         enum rte_ioat_dev_type *type =
 501                         (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
 502         if (*type == RTE_IDXD_DEV)
 503                 return __idxd_perform_ops(dev_id);
 504         else
 505                 return __ioat_perform_ops(dev_id);
 506 }
 507
 508 static inline int
 509 rte_ioat_completed_ops(int dev_id, uint8_t max_copies,
 510                 uintptr_t *src_hdls, uintptr_t *dst_hdls)
 511 {
 512         enum rte_ioat_dev_type *type =
 513                         (enum rte_ioat_dev_type *)rte_rawdevs[dev_id].dev_private;
 514         if (*type == RTE_IDXD_DEV)
 515                 return __idxd_completed_ops(dev_id, max_copies,
 516                                 src_hdls, dst_hdls);
 517         else
 518                 return __ioat_completed_ops(dev_id,  max_copies,
 519                                 src_hdls, dst_hdls);
 520 }
 521
 522 static inline void
 523 __rte_deprecated_msg("use rte_ioat_perform_ops() instead")
 524 rte_ioat_do_copies(int dev_id) { rte_ioat_perform_ops(dev_id); }
 525
 526 static inline int
 527 __rte_deprecated_msg("use rte_ioat_completed_ops() instead")
 528 rte_ioat_completed_copies(int dev_id, uint8_t max_copies,
 529                 uintptr_t *src_hdls, uintptr_t *dst_hdls)
 530 {
 531         return rte_ioat_completed_ops(dev_id, max_copies, src_hdls, dst_hdls);
 532 }
 533
 534 #endif /* _RTE_IOAT_RAWDEV_FNS_H_ */