lib/gpudev/gpudev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_eal.h>
   6 #include <rte_tailq.h>
   7 #include <rte_string_fns.h>
   8 #include <rte_memzone.h>
   9 #include <rte_malloc.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12 #include <rte_eal_paging.h>
  13
  14 #include "rte_gpudev.h"
  15 #include "gpudev_driver.h"
  16
  17 /* Logging */
  18 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE);
  19 #define GPU_LOG(level, ...) \
  20         rte_log(RTE_LOG_ ## level, gpu_logtype, RTE_FMT("gpu: " \
  21                 RTE_FMT_HEAD(__VA_ARGS__, ) "\n", RTE_FMT_TAIL(__VA_ARGS__, )))
  22
  23 /* Set any driver error as EPERM */
  24 #define GPU_DRV_RET(function) \
  25         ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0))
  26
  27 /* Array of devices */
  28 static struct rte_gpu *gpus;
  29 /* Number of currently valid devices */
  30 static int16_t gpu_max;
  31 /* Number of currently valid devices */
  32 static int16_t gpu_count;
  33
  34 /* Shared memory between processes. */
  35 static const char *GPU_MEMZONE = "rte_gpu_shared";
  36 static struct {
  37         __extension__ struct rte_gpu_mpshared gpus[0];
  38 } *gpu_shared_mem;
  39
  40 /* Event callback object */
  41 struct rte_gpu_callback {
  42         TAILQ_ENTRY(rte_gpu_callback) next;
  43         rte_gpu_callback_t *function;
  44         void *user_data;
  45         enum rte_gpu_event event;
  46 };
  47 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER;
  48 static void gpu_free_callbacks(struct rte_gpu *dev);
  49
  50 int
  51 rte_gpu_init(size_t dev_max)
  52 {
  53         if (dev_max == 0 || dev_max > INT16_MAX) {
  54                 GPU_LOG(ERR, "invalid array size");
  55                 rte_errno = EINVAL;
  56                 return -rte_errno;
  57         }
  58
  59         /* No lock, it must be called before or during first probing. */
  60         if (gpus != NULL) {
  61                 GPU_LOG(ERR, "already initialized");
  62                 rte_errno = EBUSY;
  63                 return -rte_errno;
  64         }
  65
  66         gpus = calloc(dev_max, sizeof(struct rte_gpu));
  67         if (gpus == NULL) {
  68                 GPU_LOG(ERR, "cannot initialize library");
  69                 rte_errno = ENOMEM;
  70                 return -rte_errno;
  71         }
  72
  73         gpu_max = dev_max;
  74         return 0;
  75 }
  76
  77 uint16_t
  78 rte_gpu_count_avail(void)
  79 {
  80         return gpu_count;
  81 }
  82
  83 bool
  84 rte_gpu_is_valid(int16_t dev_id)
  85 {
  86         if (dev_id >= 0 && dev_id < gpu_max &&
  87                 gpus[dev_id].process_state == RTE_GPU_STATE_INITIALIZED)
  88                 return true;
  89         return false;
  90 }
  91
  92 static bool
  93 gpu_match_parent(int16_t dev_id, int16_t parent)
  94 {
  95         if (parent == RTE_GPU_ID_ANY)
  96                 return true;
  97         return gpus[dev_id].mpshared->info.parent == parent;
  98 }
  99
 100 int16_t
 101 rte_gpu_find_next(int16_t dev_id, int16_t parent)
 102 {
 103         if (dev_id < 0)
 104                 dev_id = 0;
 105         while (dev_id < gpu_max &&
 106                         (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED ||
 107                         !gpu_match_parent(dev_id, parent)))
 108                 dev_id++;
 109
 110         if (dev_id >= gpu_max)
 111                 return RTE_GPU_ID_NONE;
 112         return dev_id;
 113 }
 114
 115 static int16_t
 116 gpu_find_free_id(void)
 117 {
 118         int16_t dev_id;
 119
 120         for (dev_id = 0; dev_id < gpu_max; dev_id++) {
 121                 if (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED)
 122                         return dev_id;
 123         }
 124         return RTE_GPU_ID_NONE;
 125 }
 126
 127 static struct rte_gpu *
 128 gpu_get_by_id(int16_t dev_id)
 129 {
 130         if (!rte_gpu_is_valid(dev_id))
 131                 return NULL;
 132         return &gpus[dev_id];
 133 }
 134
 135 struct rte_gpu *
 136 rte_gpu_get_by_name(const char *name)
 137 {
 138         int16_t dev_id;
 139         struct rte_gpu *dev;
 140
 141         if (name == NULL) {
 142                 rte_errno = EINVAL;
 143                 return NULL;
 144         }
 145
 146         RTE_GPU_FOREACH(dev_id) {
 147                 dev = &gpus[dev_id];
 148                 if (strncmp(name, dev->mpshared->name, RTE_DEV_NAME_MAX_LEN) == 0)
 149                         return dev;
 150         }
 151         return NULL;
 152 }
 153
 154 static int
 155 gpu_shared_mem_init(void)
 156 {
 157         const struct rte_memzone *memzone;
 158
 159         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 160                 memzone = rte_memzone_reserve(GPU_MEMZONE,
 161                                 sizeof(*gpu_shared_mem) +
 162                                 sizeof(*gpu_shared_mem->gpus) * gpu_max,
 163                                 SOCKET_ID_ANY, 0);
 164         } else {
 165                 memzone = rte_memzone_lookup(GPU_MEMZONE);
 166         }
 167         if (memzone == NULL) {
 168                 GPU_LOG(ERR, "cannot initialize shared memory");
 169                 rte_errno = ENOMEM;
 170                 return -rte_errno;
 171         }
 172
 173         gpu_shared_mem = memzone->addr;
 174         return 0;
 175 }
 176
 177 struct rte_gpu *
 178 rte_gpu_allocate(const char *name)
 179 {
 180         int16_t dev_id;
 181         struct rte_gpu *dev;
 182
 183         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 184                 GPU_LOG(ERR, "only primary process can allocate device");
 185                 rte_errno = EPERM;
 186                 return NULL;
 187         }
 188         if (name == NULL) {
 189                 GPU_LOG(ERR, "allocate device without a name");
 190                 rte_errno = EINVAL;
 191                 return NULL;
 192         }
 193
 194         /* implicit initialization of library before adding first device */
 195         if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0)
 196                 return NULL;
 197
 198         /* initialize shared memory before adding first device */
 199         if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0)
 200                 return NULL;
 201
 202         if (rte_gpu_get_by_name(name) != NULL) {
 203                 GPU_LOG(ERR, "device with name %s already exists", name);
 204                 rte_errno = EEXIST;
 205                 return NULL;
 206         }
 207         dev_id = gpu_find_free_id();
 208         if (dev_id == RTE_GPU_ID_NONE) {
 209                 GPU_LOG(ERR, "reached maximum number of devices");
 210                 rte_errno = ENOENT;
 211                 return NULL;
 212         }
 213
 214         dev = &gpus[dev_id];
 215         memset(dev, 0, sizeof(*dev));
 216
 217         dev->mpshared = &gpu_shared_mem->gpus[dev_id];
 218         memset(dev->mpshared, 0, sizeof(*dev->mpshared));
 219
 220         if (rte_strscpy(dev->mpshared->name, name, RTE_DEV_NAME_MAX_LEN) < 0) {
 221                 GPU_LOG(ERR, "device name too long: %s", name);
 222                 rte_errno = ENAMETOOLONG;
 223                 return NULL;
 224         }
 225         dev->mpshared->info.name = dev->mpshared->name;
 226         dev->mpshared->info.dev_id = dev_id;
 227         dev->mpshared->info.numa_node = -1;
 228         dev->mpshared->info.parent = RTE_GPU_ID_NONE;
 229         TAILQ_INIT(&dev->callbacks);
 230         __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
 231
 232         gpu_count++;
 233         GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
 234                         name, dev_id, gpu_count);
 235         return dev;
 236 }
 237
 238 struct rte_gpu *
 239 rte_gpu_attach(const char *name)
 240 {
 241         int16_t dev_id;
 242         struct rte_gpu *dev;
 243         struct rte_gpu_mpshared *shared_dev;
 244
 245         if (rte_eal_process_type() != RTE_PROC_SECONDARY) {
 246                 GPU_LOG(ERR, "only secondary process can attach device");
 247                 rte_errno = EPERM;
 248                 return NULL;
 249         }
 250         if (name == NULL) {
 251                 GPU_LOG(ERR, "attach device without a name");
 252                 rte_errno = EINVAL;
 253                 return NULL;
 254         }
 255
 256         /* implicit initialization of library before adding first device */
 257         if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0)
 258                 return NULL;
 259
 260         /* initialize shared memory before adding first device */
 261         if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0)
 262                 return NULL;
 263
 264         for (dev_id = 0; dev_id < gpu_max; dev_id++) {
 265                 shared_dev = &gpu_shared_mem->gpus[dev_id];
 266                 if (strncmp(name, shared_dev->name, RTE_DEV_NAME_MAX_LEN) == 0)
 267                         break;
 268         }
 269         if (dev_id >= gpu_max) {
 270                 GPU_LOG(ERR, "device with name %s not found", name);
 271                 rte_errno = ENOENT;
 272                 return NULL;
 273         }
 274         dev = &gpus[dev_id];
 275         memset(dev, 0, sizeof(*dev));
 276
 277         TAILQ_INIT(&dev->callbacks);
 278         dev->mpshared = shared_dev;
 279         __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
 280
 281         gpu_count++;
 282         GPU_LOG(DEBUG, "attached device %s (id %d) of total %d",
 283                         name, dev_id, gpu_count);
 284         return dev;
 285 }
 286
 287 int16_t
 288 rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context)
 289 {
 290         struct rte_gpu *dev;
 291
 292         if (!rte_gpu_is_valid(parent)) {
 293                 GPU_LOG(ERR, "add child to invalid parent ID %d", parent);
 294                 rte_errno = ENODEV;
 295                 return -rte_errno;
 296         }
 297
 298         dev = rte_gpu_allocate(name);
 299         if (dev == NULL)
 300                 return -rte_errno;
 301
 302         dev->mpshared->info.parent = parent;
 303         dev->mpshared->info.context = child_context;
 304
 305         rte_gpu_complete_new(dev);
 306         return dev->mpshared->info.dev_id;
 307 }
 308
 309 void
 310 rte_gpu_complete_new(struct rte_gpu *dev)
 311 {
 312         if (dev == NULL)
 313                 return;
 314
 315         dev->process_state = RTE_GPU_STATE_INITIALIZED;
 316         rte_gpu_notify(dev, RTE_GPU_EVENT_NEW);
 317 }
 318
 319 int
 320 rte_gpu_release(struct rte_gpu *dev)
 321 {
 322         int16_t dev_id, child;
 323
 324         if (dev == NULL) {
 325                 rte_errno = ENODEV;
 326                 return -rte_errno;
 327         }
 328         dev_id = dev->mpshared->info.dev_id;
 329         RTE_GPU_FOREACH_CHILD(child, dev_id) {
 330                 GPU_LOG(ERR, "cannot release device %d with child %d",
 331                                 dev_id, child);
 332                 rte_errno = EBUSY;
 333                 return -rte_errno;
 334         }
 335
 336         GPU_LOG(DEBUG, "free device %s (id %d)",
 337                         dev->mpshared->info.name, dev->mpshared->info.dev_id);
 338         rte_gpu_notify(dev, RTE_GPU_EVENT_DEL);
 339
 340         gpu_free_callbacks(dev);
 341         dev->process_state = RTE_GPU_STATE_UNUSED;
 342         __atomic_fetch_sub(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
 343         gpu_count--;
 344
 345         return 0;
 346 }
 347
 348 int
 349 rte_gpu_close(int16_t dev_id)
 350 {
 351         int firsterr, binerr;
 352         int *lasterr = &firsterr;
 353         struct rte_gpu *dev;
 354
 355         dev = gpu_get_by_id(dev_id);
 356         if (dev == NULL) {
 357                 GPU_LOG(ERR, "close invalid device ID %d", dev_id);
 358                 rte_errno = ENODEV;
 359                 return -rte_errno;
 360         }
 361
 362         if (dev->ops.dev_close != NULL) {
 363                 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev));
 364                 if (*lasterr != 0)
 365                         lasterr = &binerr;
 366         }
 367
 368         *lasterr = rte_gpu_release(dev);
 369
 370         rte_errno = -firsterr;
 371         return firsterr;
 372 }
 373
 374 int
 375 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event,
 376                 rte_gpu_callback_t *function, void *user_data)
 377 {
 378         int16_t next_dev, last_dev;
 379         struct rte_gpu_callback_list *callbacks;
 380         struct rte_gpu_callback *callback;
 381
 382         if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
 383                 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id);
 384                 rte_errno = ENODEV;
 385                 return -rte_errno;
 386         }
 387         if (function == NULL) {
 388                 GPU_LOG(ERR, "cannot register callback without function");
 389                 rte_errno = EINVAL;
 390                 return -rte_errno;
 391         }
 392
 393         if (dev_id == RTE_GPU_ID_ANY) {
 394                 next_dev = 0;
 395                 last_dev = gpu_max - 1;
 396         } else {
 397                 next_dev = last_dev = dev_id;
 398         }
 399
 400         rte_rwlock_write_lock(&gpu_callback_lock);
 401         do {
 402                 callbacks = &gpus[next_dev].callbacks;
 403
 404                 /* check if not already registered */
 405                 TAILQ_FOREACH(callback, callbacks, next) {
 406                         if (callback->event == event &&
 407                                         callback->function == function &&
 408                                         callback->user_data == user_data) {
 409                                 GPU_LOG(INFO, "callback already registered");
 410                                 return 0;
 411                         }
 412                 }
 413
 414                 callback = malloc(sizeof(*callback));
 415                 if (callback == NULL) {
 416                         GPU_LOG(ERR, "cannot allocate callback");
 417                         return -ENOMEM;
 418                 }
 419                 callback->function = function;
 420                 callback->user_data = user_data;
 421                 callback->event = event;
 422                 TAILQ_INSERT_TAIL(callbacks, callback, next);
 423
 424         } while (++next_dev <= last_dev);
 425         rte_rwlock_write_unlock(&gpu_callback_lock);
 426
 427         return 0;
 428 }
 429
 430 int
 431 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event,
 432                 rte_gpu_callback_t *function, void *user_data)
 433 {
 434         int16_t next_dev, last_dev;
 435         struct rte_gpu_callback_list *callbacks;
 436         struct rte_gpu_callback *callback, *nextcb;
 437
 438         if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
 439                 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id);
 440                 rte_errno = ENODEV;
 441                 return -rte_errno;
 442         }
 443         if (function == NULL) {
 444                 GPU_LOG(ERR, "cannot unregister callback without function");
 445                 rte_errno = EINVAL;
 446                 return -rte_errno;
 447         }
 448
 449         if (dev_id == RTE_GPU_ID_ANY) {
 450                 next_dev = 0;
 451                 last_dev = gpu_max - 1;
 452         } else {
 453                 next_dev = last_dev = dev_id;
 454         }
 455
 456         rte_rwlock_write_lock(&gpu_callback_lock);
 457         do {
 458                 callbacks = &gpus[next_dev].callbacks;
 459                 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
 460                         if (callback->event != event ||
 461                                         callback->function != function ||
 462                                         (callback->user_data != user_data &&
 463                                         user_data != (void *)-1))
 464                                 continue;
 465                         TAILQ_REMOVE(callbacks, callback, next);
 466                         free(callback);
 467                 }
 468         } while (++next_dev <= last_dev);
 469         rte_rwlock_write_unlock(&gpu_callback_lock);
 470
 471         return 0;
 472 }
 473
 474 static void
 475 gpu_free_callbacks(struct rte_gpu *dev)
 476 {
 477         struct rte_gpu_callback_list *callbacks;
 478         struct rte_gpu_callback *callback, *nextcb;
 479
 480         callbacks = &dev->callbacks;
 481         rte_rwlock_write_lock(&gpu_callback_lock);
 482         RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
 483                 TAILQ_REMOVE(callbacks, callback, next);
 484                 free(callback);
 485         }
 486         rte_rwlock_write_unlock(&gpu_callback_lock);
 487 }
 488
 489 void
 490 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event)
 491 {
 492         int16_t dev_id;
 493         struct rte_gpu_callback *callback;
 494
 495         dev_id = dev->mpshared->info.dev_id;
 496         rte_rwlock_read_lock(&gpu_callback_lock);
 497         TAILQ_FOREACH(callback, &dev->callbacks, next) {
 498                 if (callback->event != event || callback->function == NULL)
 499                         continue;
 500                 callback->function(dev_id, event, callback->user_data);
 501         }
 502         rte_rwlock_read_unlock(&gpu_callback_lock);
 503 }
 504
 505 int
 506 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info)
 507 {
 508         struct rte_gpu *dev;
 509
 510         dev = gpu_get_by_id(dev_id);
 511         if (dev == NULL) {
 512                 GPU_LOG(ERR, "query invalid device ID %d", dev_id);
 513                 rte_errno = ENODEV;
 514                 return -rte_errno;
 515         }
 516         if (info == NULL) {
 517                 GPU_LOG(ERR, "query without storage");
 518                 rte_errno = EINVAL;
 519                 return -rte_errno;
 520         }
 521
 522         if (dev->ops.dev_info_get == NULL) {
 523                 *info = dev->mpshared->info;
 524                 return 0;
 525         }
 526         return GPU_DRV_RET(dev->ops.dev_info_get(dev, info));
 527 }
 528
 529 void *
 530 rte_gpu_mem_alloc(int16_t dev_id, size_t size, unsigned int align)
 531 {
 532         struct rte_gpu *dev;
 533         void *ptr;
 534         int ret;
 535
 536         dev = gpu_get_by_id(dev_id);
 537         if (dev == NULL) {
 538                 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id);
 539                 rte_errno = ENODEV;
 540                 return NULL;
 541         }
 542
 543         if (dev->ops.mem_alloc == NULL) {
 544                 GPU_LOG(ERR, "mem allocation not supported");
 545                 rte_errno = ENOTSUP;
 546                 return NULL;
 547         }
 548
 549         if (size == 0) /* dry-run */
 550                 return NULL;
 551
 552         if (align && !rte_is_power_of_2(align)) {
 553                 GPU_LOG(ERR, "requested alignment is not a power of two %u", align);
 554                 rte_errno = EINVAL;
 555                 return NULL;
 556         }
 557
 558         ret = dev->ops.mem_alloc(dev, size, align, &ptr);
 559
 560         switch (ret) {
 561         case 0:
 562                 return ptr;
 563         case -ENOMEM:
 564         case -E2BIG:
 565                 rte_errno = -ret;
 566                 return NULL;
 567         default:
 568                 rte_errno = -EPERM;
 569                 return NULL;
 570         }
 571 }
 572
 573 int
 574 rte_gpu_mem_free(int16_t dev_id, void *ptr)
 575 {
 576         struct rte_gpu *dev;
 577
 578         dev = gpu_get_by_id(dev_id);
 579         if (dev == NULL) {
 580                 GPU_LOG(ERR, "free mem for invalid device ID %d", dev_id);
 581                 rte_errno = ENODEV;
 582                 return -rte_errno;
 583         }
 584
 585         if (dev->ops.mem_free == NULL) {
 586                 rte_errno = ENOTSUP;
 587                 return -rte_errno;
 588         }
 589
 590         if (ptr == NULL) /* dry-run */
 591                 return 0;
 592
 593         return GPU_DRV_RET(dev->ops.mem_free(dev, ptr));
 594 }
 595
 596 int
 597 rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr)
 598 {
 599         struct rte_gpu *dev;
 600
 601         dev = gpu_get_by_id(dev_id);
 602         if (dev == NULL) {
 603                 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id);
 604                 rte_errno = ENODEV;
 605                 return -rte_errno;
 606         }
 607
 608         if (dev->ops.mem_register == NULL) {
 609                 GPU_LOG(ERR, "mem registration not supported");
 610                 rte_errno = ENOTSUP;
 611                 return -rte_errno;
 612         }
 613
 614         if (ptr == NULL || size == 0) /* dry-run  */
 615                 return 0;
 616
 617         return GPU_DRV_RET(dev->ops.mem_register(dev, size, ptr));
 618 }
 619
 620 int
 621 rte_gpu_mem_unregister(int16_t dev_id, void *ptr)
 622 {
 623         struct rte_gpu *dev;
 624
 625         dev = gpu_get_by_id(dev_id);
 626         if (dev == NULL) {
 627                 GPU_LOG(ERR, "unregister mem for invalid device ID %d", dev_id);
 628                 rte_errno = ENODEV;
 629                 return -rte_errno;
 630         }
 631
 632         if (dev->ops.mem_unregister == NULL) {
 633                 rte_errno = ENOTSUP;
 634                 return -rte_errno;
 635         }
 636
 637         if (ptr == NULL) /* dry-run */
 638                 return 0;
 639
 640         return GPU_DRV_RET(dev->ops.mem_unregister(dev, ptr));
 641 }
 642
 643 void *
 644 rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr)
 645 {
 646         struct rte_gpu *dev;
 647         void *ptr_out;
 648         int ret;
 649
 650         dev = gpu_get_by_id(dev_id);
 651         if (dev == NULL) {
 652                 GPU_LOG(ERR, "mem CPU map for invalid device ID %d", dev_id);
 653                 rte_errno = ENODEV;
 654                 return NULL;
 655         }
 656
 657         if (dev->ops.mem_cpu_map == NULL) {
 658                 GPU_LOG(ERR, "mem CPU map not supported");
 659                 rte_errno = ENOTSUP;
 660                 return NULL;
 661         }
 662
 663         if (ptr == NULL || size == 0) /* dry-run  */
 664                 return NULL;
 665
 666         ret = GPU_DRV_RET(dev->ops.mem_cpu_map(dev, size, ptr, &ptr_out));
 667
 668         switch (ret) {
 669         case 0:
 670                 return ptr_out;
 671         case -ENOMEM:
 672         case -E2BIG:
 673                 rte_errno = -ret;
 674                 return NULL;
 675         default:
 676                 rte_errno = -EPERM;
 677                 return NULL;
 678         }
 679 }
 680
 681 int
 682 rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr)
 683 {
 684         struct rte_gpu *dev;
 685
 686         dev = gpu_get_by_id(dev_id);
 687         if (dev == NULL) {
 688                 GPU_LOG(ERR, "cpu_unmap mem for invalid device ID %d", dev_id);
 689                 rte_errno = ENODEV;
 690                 return -rte_errno;
 691         }
 692
 693         if (dev->ops.mem_cpu_unmap == NULL) {
 694                 rte_errno = ENOTSUP;
 695                 return -rte_errno;
 696         }
 697
 698         if (ptr == NULL) /* dry-run */
 699                 return 0;
 700
 701         return GPU_DRV_RET(dev->ops.mem_cpu_unmap(dev, ptr));
 702 }
 703
 704 int
 705 rte_gpu_wmb(int16_t dev_id)
 706 {
 707         struct rte_gpu *dev;
 708
 709         dev = gpu_get_by_id(dev_id);
 710         if (dev == NULL) {
 711                 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id);
 712                 rte_errno = ENODEV;
 713                 return -rte_errno;
 714         }
 715
 716         if (dev->ops.wmb == NULL) {
 717                 rte_errno = ENOTSUP;
 718                 return -rte_errno;
 719         }
 720         return GPU_DRV_RET(dev->ops.wmb(dev));
 721 }
 722
 723 int
 724 rte_gpu_comm_create_flag(uint16_t dev_id, struct rte_gpu_comm_flag *devflag,
 725                 enum rte_gpu_comm_flag_type mtype)
 726 {
 727         size_t flag_size;
 728         int ret;
 729
 730         if (devflag == NULL) {
 731                 rte_errno = EINVAL;
 732                 return -rte_errno;
 733         }
 734         if (mtype != RTE_GPU_COMM_FLAG_CPU) {
 735                 rte_errno = EINVAL;
 736                 return -rte_errno;
 737         }
 738
 739         flag_size = sizeof(uint32_t);
 740
 741         devflag->ptr = rte_zmalloc(NULL, flag_size, 0);
 742         if (devflag->ptr == NULL) {
 743                 rte_errno = ENOMEM;
 744                 return -rte_errno;
 745         }
 746
 747         ret = rte_gpu_mem_register(dev_id, flag_size, devflag->ptr);
 748         if (ret < 0) {
 749                 rte_errno = ENOMEM;
 750                 return -rte_errno;
 751         }
 752
 753         devflag->mtype = mtype;
 754         devflag->dev_id = dev_id;
 755
 756         return 0;
 757 }
 758
 759 int
 760 rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag *devflag)
 761 {
 762         int ret;
 763
 764         if (devflag == NULL) {
 765                 rte_errno = EINVAL;
 766                 return -rte_errno;
 767         }
 768
 769         ret = rte_gpu_mem_unregister(devflag->dev_id, devflag->ptr);
 770         if (ret < 0) {
 771                 rte_errno = EINVAL;
 772                 return -1;
 773         }
 774
 775         rte_free(devflag->ptr);
 776
 777         return 0;
 778 }
 779
 780 int
 781 rte_gpu_comm_set_flag(struct rte_gpu_comm_flag *devflag, uint32_t val)
 782 {
 783         if (devflag == NULL) {
 784                 rte_errno = EINVAL;
 785                 return -rte_errno;
 786         }
 787
 788         if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) {
 789                 rte_errno = EINVAL;
 790                 return -rte_errno;
 791         }
 792
 793         RTE_GPU_VOLATILE(*devflag->ptr) = val;
 794
 795         return 0;
 796 }
 797
 798 int
 799 rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag *devflag, uint32_t *val)
 800 {
 801         if (devflag == NULL) {
 802                 rte_errno = EINVAL;
 803                 return -rte_errno;
 804         }
 805         if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) {
 806                 rte_errno = EINVAL;
 807                 return -rte_errno;
 808         }
 809
 810         *val = RTE_GPU_VOLATILE(*devflag->ptr);
 811
 812         return 0;
 813 }
 814
 815 struct rte_gpu_comm_list *
 816 rte_gpu_comm_create_list(uint16_t dev_id,
 817                 uint32_t num_comm_items)
 818 {
 819         struct rte_gpu_comm_list *comm_list;
 820         uint32_t idx_l;
 821         int ret;
 822         struct rte_gpu *dev;
 823         struct rte_gpu_info info;
 824
 825         if (num_comm_items == 0) {
 826                 rte_errno = EINVAL;
 827                 return NULL;
 828         }
 829
 830         dev = gpu_get_by_id(dev_id);
 831         if (dev == NULL) {
 832                 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id);
 833                 rte_errno = ENODEV;
 834                 return NULL;
 835         }
 836
 837         ret = rte_gpu_info_get(dev_id, &info);
 838         if (ret < 0) {
 839                 rte_errno = ENODEV;
 840                 return NULL;
 841         }
 842
 843         comm_list = rte_zmalloc(NULL,
 844                         sizeof(struct rte_gpu_comm_list) * num_comm_items, 0);
 845         if (comm_list == NULL) {
 846                 rte_errno = ENOMEM;
 847                 return NULL;
 848         }
 849
 850         ret = rte_gpu_mem_register(dev_id,
 851                         sizeof(struct rte_gpu_comm_list) * num_comm_items, comm_list);
 852         if (ret < 0) {
 853                 rte_errno = ENOMEM;
 854                 return NULL;
 855         }
 856
 857         /*
 858          * Use GPU memory CPU map feature if enabled in the driver
 859          * to allocate the status flags of the list.
 860          * Allocating this flag in GPU memory will reduce
 861          * the latency when GPU workload is polling this flag.
 862          */
 863         comm_list[0].status_d = rte_gpu_mem_alloc(dev_id,
 864                         sizeof(enum rte_gpu_comm_list_status) * num_comm_items,
 865                         info.page_size);
 866         if (ret < 0) {
 867                 rte_errno = ENOMEM;
 868                 return NULL;
 869         }
 870
 871         comm_list[0].status_h = rte_gpu_mem_cpu_map(dev_id,
 872                         sizeof(enum rte_gpu_comm_list_status) * num_comm_items,
 873                         comm_list[0].status_d);
 874         if (comm_list[0].status_h == NULL) {
 875                 /*
 876                  * If CPU mapping is not supported by driver
 877                  * use regular CPU registered memory.
 878                  */
 879                 comm_list[0].status_h = rte_zmalloc(NULL,
 880                                 sizeof(enum rte_gpu_comm_list_status) * num_comm_items, 0);
 881                 if (comm_list[0].status_h == NULL) {
 882                         rte_errno = ENOMEM;
 883                         return NULL;
 884                 }
 885
 886                 ret = rte_gpu_mem_register(dev_id,
 887                                 sizeof(enum rte_gpu_comm_list_status) * num_comm_items,
 888                                 comm_list[0].status_h);
 889                 if (ret < 0) {
 890                         rte_errno = ENOMEM;
 891                         return NULL;
 892                 }
 893
 894                 comm_list[0].status_d = comm_list[0].status_h;
 895         }
 896
 897         for (idx_l = 0; idx_l < num_comm_items; idx_l++) {
 898                 comm_list[idx_l].pkt_list = rte_zmalloc(NULL,
 899                                 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 0);
 900                 if (comm_list[idx_l].pkt_list == NULL) {
 901                         rte_errno = ENOMEM;
 902                         return NULL;
 903                 }
 904
 905                 ret = rte_gpu_mem_register(dev_id,
 906                                 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX,
 907                                 comm_list[idx_l].pkt_list);
 908                 if (ret < 0) {
 909                         rte_errno = ENOMEM;
 910                         return NULL;
 911                 }
 912
 913                 comm_list[idx_l].num_pkts = 0;
 914                 comm_list[idx_l].dev_id = dev_id;
 915
 916                 comm_list[idx_l].mbufs = rte_zmalloc(NULL,
 917                                 sizeof(struct rte_mbuf *) * RTE_GPU_COMM_LIST_PKTS_MAX, 0);
 918                 if (comm_list[idx_l].mbufs == NULL) {
 919                         rte_errno = ENOMEM;
 920                         return NULL;
 921                 }
 922
 923                 if (idx_l > 0) {
 924                         comm_list[idx_l].status_h = &(comm_list[0].status_h[idx_l]);
 925                         comm_list[idx_l].status_d = &(comm_list[0].status_d[idx_l]);
 926
 927                         ret = rte_gpu_comm_set_status(&comm_list[idx_l], RTE_GPU_COMM_LIST_FREE);
 928                         if (ret < 0) {
 929                                 rte_errno = ENOMEM;
 930                                 return NULL;
 931                         }
 932                 }
 933         }
 934
 935         return comm_list;
 936 }
 937
 938 int
 939 rte_gpu_comm_destroy_list(struct rte_gpu_comm_list *comm_list,
 940                 uint32_t num_comm_items)
 941 {
 942         uint32_t idx_l;
 943         int ret;
 944         uint16_t dev_id;
 945
 946         if (comm_list == NULL) {
 947                 rte_errno = EINVAL;
 948                 return -rte_errno;
 949         }
 950
 951         dev_id = comm_list[0].dev_id;
 952
 953         for (idx_l = 0; idx_l < num_comm_items; idx_l++) {
 954                 ret = rte_gpu_mem_unregister(dev_id, comm_list[idx_l].pkt_list);
 955                 if (ret < 0) {
 956                         rte_errno = EINVAL;
 957                         return -1;
 958                 }
 959
 960                 rte_free(comm_list[idx_l].pkt_list);
 961                 rte_free(comm_list[idx_l].mbufs);
 962         }
 963
 964         ret = rte_gpu_mem_unregister(dev_id, comm_list);
 965         if (ret < 0) {
 966                 rte_errno = EINVAL;
 967                 return -1;
 968         }
 969
 970         ret = rte_gpu_mem_cpu_unmap(dev_id, comm_list[0].status_d);
 971         if (ret == 0) {
 972                 rte_gpu_mem_free(dev_id, comm_list[0].status_d);
 973         } else {
 974                 rte_gpu_mem_unregister(dev_id, comm_list[0].status_h);
 975                 rte_free(comm_list[0].status_h);
 976         }
 977
 978         rte_free(comm_list);
 979
 980         return 0;
 981 }
 982
 983 int
 984 rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list *comm_list_item,
 985                 struct rte_mbuf **mbufs, uint32_t num_mbufs)
 986 {
 987         uint32_t idx;
 988         int ret;
 989
 990         if (comm_list_item == NULL || comm_list_item->pkt_list == NULL ||
 991                         mbufs == NULL || num_mbufs > RTE_GPU_COMM_LIST_PKTS_MAX) {
 992                 rte_errno = EINVAL;
 993                 return -rte_errno;
 994         }
 995
 996         for (idx = 0; idx < num_mbufs; idx++) {
 997                 /* support only unchained mbufs */
 998                 if (unlikely((mbufs[idx]->nb_segs > 1) ||
 999                                 (mbufs[idx]->next != NULL) ||
1000                                 (mbufs[idx]->data_len != mbufs[idx]->pkt_len))) {
1001                         rte_errno = ENOTSUP;
1002                         return -rte_errno;
1003                 }
1004                 comm_list_item->pkt_list[idx].addr =
1005                                 rte_pktmbuf_mtod_offset(mbufs[idx], uintptr_t, 0);
1006                 comm_list_item->pkt_list[idx].size = mbufs[idx]->pkt_len;
1007                 comm_list_item->mbufs[idx] = mbufs[idx];
1008         }
1009
1010         RTE_GPU_VOLATILE(comm_list_item->num_pkts) = num_mbufs;
1011         rte_gpu_wmb(comm_list_item->dev_id);
1012         ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_READY);
1013         if (ret < 0) {
1014                 rte_errno = EINVAL;
1015                 return -rte_errno;
1016         }
1017
1018         return 0;
1019 }
1020
1021 int
1022 rte_gpu_comm_set_status(struct rte_gpu_comm_list *comm_list_item,
1023                 enum rte_gpu_comm_list_status status)
1024 {
1025         if (comm_list_item == NULL) {
1026                 rte_errno = EINVAL;
1027                 return -rte_errno;
1028         }
1029
1030         RTE_GPU_VOLATILE(comm_list_item->status_h[0]) = status;
1031
1032         return 0;
1033 }
1034
1035 int
1036 rte_gpu_comm_get_status(struct rte_gpu_comm_list *comm_list_item,
1037                 enum rte_gpu_comm_list_status *status)
1038 {
1039         if (comm_list_item == NULL || status == NULL) {
1040                 rte_errno = EINVAL;
1041                 return -rte_errno;
1042         }
1043
1044         *status = RTE_GPU_VOLATILE(comm_list_item->status_h[0]);
1045
1046         return 0;
1047 }
1048
1049 int
1050 rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list *comm_list_item)
1051 {
1052         uint32_t idx = 0;
1053         enum rte_gpu_comm_list_status status;
1054         int ret;
1055
1056         if (comm_list_item == NULL) {
1057                 rte_errno = EINVAL;
1058                 return -rte_errno;
1059         }
1060
1061         ret = rte_gpu_comm_get_status(comm_list_item, &status);
1062         if (ret < 0) {
1063                 rte_errno = EINVAL;
1064                 return -rte_errno;
1065         }
1066
1067         if (status == RTE_GPU_COMM_LIST_READY) {
1068                 GPU_LOG(ERR, "packet list is still in progress");
1069                 rte_errno = EINVAL;
1070                 return -rte_errno;
1071         }
1072
1073         for (idx = 0; idx < RTE_GPU_COMM_LIST_PKTS_MAX; idx++) {
1074                 if (comm_list_item->pkt_list[idx].addr == 0)
1075                         break;
1076
1077                 comm_list_item->pkt_list[idx].addr = 0;
1078                 comm_list_item->pkt_list[idx].size = 0;
1079                 comm_list_item->mbufs[idx] = NULL;
1080         }
1081
1082         ret = rte_gpu_comm_set_status(comm_list_item, RTE_GPU_COMM_LIST_FREE);
1083         if (ret < 0) {
1084                 rte_errno = EINVAL;
1085                 return -rte_errno;
1086         }
1087         RTE_GPU_VOLATILE(comm_list_item->num_pkts) = 0;
1088         rte_mb();
1089
1090         return 0;
1091 }