lib/gpudev/gpudev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_eal.h>
   6 #include <rte_tailq.h>
   7 #include <rte_string_fns.h>
   8 #include <rte_memzone.h>
   9 #include <rte_malloc.h>
  10 #include <rte_errno.h>
  11 #include <rte_log.h>
  12
  13 #include "rte_gpudev.h"
  14 #include "gpudev_driver.h"
  15
  16 /* Logging */
  17 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE);
  18 #define GPU_LOG(level, ...) \
  19         rte_log(RTE_LOG_ ## level, gpu_logtype, RTE_FMT("gpu: " \
  20                 RTE_FMT_HEAD(__VA_ARGS__, ) "\n", RTE_FMT_TAIL(__VA_ARGS__, )))
  21
  22 /* Set any driver error as EPERM */
  23 #define GPU_DRV_RET(function) \
  24         ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0))
  25
  26 /* Array of devices */
  27 static struct rte_gpu *gpus;
  28 /* Number of currently valid devices */
  29 static int16_t gpu_max;
  30 /* Number of currently valid devices */
  31 static int16_t gpu_count;
  32
  33 /* Shared memory between processes. */
  34 static const char *GPU_MEMZONE = "rte_gpu_shared";
  35 static struct {
  36         __extension__ struct rte_gpu_mpshared gpus[0];
  37 } *gpu_shared_mem;
  38
  39 /* Event callback object */
  40 struct rte_gpu_callback {
  41         TAILQ_ENTRY(rte_gpu_callback) next;
  42         rte_gpu_callback_t *function;
  43         void *user_data;
  44         enum rte_gpu_event event;
  45 };
  46 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER;
  47 static void gpu_free_callbacks(struct rte_gpu *dev);
  48
  49 int
  50 rte_gpu_init(size_t dev_max)
  51 {
  52         if (dev_max == 0 || dev_max > INT16_MAX) {
  53                 GPU_LOG(ERR, "invalid array size");
  54                 rte_errno = EINVAL;
  55                 return -rte_errno;
  56         }
  57
  58         /* No lock, it must be called before or during first probing. */
  59         if (gpus != NULL) {
  60                 GPU_LOG(ERR, "already initialized");
  61                 rte_errno = EBUSY;
  62                 return -rte_errno;
  63         }
  64
  65         gpus = calloc(dev_max, sizeof(struct rte_gpu));
  66         if (gpus == NULL) {
  67                 GPU_LOG(ERR, "cannot initialize library");
  68                 rte_errno = ENOMEM;
  69                 return -rte_errno;
  70         }
  71
  72         gpu_max = dev_max;
  73         return 0;
  74 }
  75
  76 uint16_t
  77 rte_gpu_count_avail(void)
  78 {
  79         return gpu_count;
  80 }
  81
  82 bool
  83 rte_gpu_is_valid(int16_t dev_id)
  84 {
  85         if (dev_id >= 0 && dev_id < gpu_max &&
  86                 gpus[dev_id].process_state == RTE_GPU_STATE_INITIALIZED)
  87                 return true;
  88         return false;
  89 }
  90
  91 static bool
  92 gpu_match_parent(int16_t dev_id, int16_t parent)
  93 {
  94         if (parent == RTE_GPU_ID_ANY)
  95                 return true;
  96         return gpus[dev_id].mpshared->info.parent == parent;
  97 }
  98
  99 int16_t
 100 rte_gpu_find_next(int16_t dev_id, int16_t parent)
 101 {
 102         if (dev_id < 0)
 103                 dev_id = 0;
 104         while (dev_id < gpu_max &&
 105                         (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED ||
 106                         !gpu_match_parent(dev_id, parent)))
 107                 dev_id++;
 108
 109         if (dev_id >= gpu_max)
 110                 return RTE_GPU_ID_NONE;
 111         return dev_id;
 112 }
 113
 114 static int16_t
 115 gpu_find_free_id(void)
 116 {
 117         int16_t dev_id;
 118
 119         for (dev_id = 0; dev_id < gpu_max; dev_id++) {
 120                 if (gpus[dev_id].process_state == RTE_GPU_STATE_UNUSED)
 121                         return dev_id;
 122         }
 123         return RTE_GPU_ID_NONE;
 124 }
 125
 126 static struct rte_gpu *
 127 gpu_get_by_id(int16_t dev_id)
 128 {
 129         if (!rte_gpu_is_valid(dev_id))
 130                 return NULL;
 131         return &gpus[dev_id];
 132 }
 133
 134 struct rte_gpu *
 135 rte_gpu_get_by_name(const char *name)
 136 {
 137         int16_t dev_id;
 138         struct rte_gpu *dev;
 139
 140         if (name == NULL) {
 141                 rte_errno = EINVAL;
 142                 return NULL;
 143         }
 144
 145         RTE_GPU_FOREACH(dev_id) {
 146                 dev = &gpus[dev_id];
 147                 if (strncmp(name, dev->mpshared->name, RTE_DEV_NAME_MAX_LEN) == 0)
 148                         return dev;
 149         }
 150         return NULL;
 151 }
 152
 153 static int
 154 gpu_shared_mem_init(void)
 155 {
 156         const struct rte_memzone *memzone;
 157
 158         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 159                 memzone = rte_memzone_reserve(GPU_MEMZONE,
 160                                 sizeof(*gpu_shared_mem) +
 161                                 sizeof(*gpu_shared_mem->gpus) * gpu_max,
 162                                 SOCKET_ID_ANY, 0);
 163         } else {
 164                 memzone = rte_memzone_lookup(GPU_MEMZONE);
 165         }
 166         if (memzone == NULL) {
 167                 GPU_LOG(ERR, "cannot initialize shared memory");
 168                 rte_errno = ENOMEM;
 169                 return -rte_errno;
 170         }
 171
 172         gpu_shared_mem = memzone->addr;
 173         return 0;
 174 }
 175
 176 struct rte_gpu *
 177 rte_gpu_allocate(const char *name)
 178 {
 179         int16_t dev_id;
 180         struct rte_gpu *dev;
 181
 182         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 183                 GPU_LOG(ERR, "only primary process can allocate device");
 184                 rte_errno = EPERM;
 185                 return NULL;
 186         }
 187         if (name == NULL) {
 188                 GPU_LOG(ERR, "allocate device without a name");
 189                 rte_errno = EINVAL;
 190                 return NULL;
 191         }
 192
 193         /* implicit initialization of library before adding first device */
 194         if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0)
 195                 return NULL;
 196
 197         /* initialize shared memory before adding first device */
 198         if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0)
 199                 return NULL;
 200
 201         if (rte_gpu_get_by_name(name) != NULL) {
 202                 GPU_LOG(ERR, "device with name %s already exists", name);
 203                 rte_errno = EEXIST;
 204                 return NULL;
 205         }
 206         dev_id = gpu_find_free_id();
 207         if (dev_id == RTE_GPU_ID_NONE) {
 208                 GPU_LOG(ERR, "reached maximum number of devices");
 209                 rte_errno = ENOENT;
 210                 return NULL;
 211         }
 212
 213         dev = &gpus[dev_id];
 214         memset(dev, 0, sizeof(*dev));
 215
 216         dev->mpshared = &gpu_shared_mem->gpus[dev_id];
 217         memset(dev->mpshared, 0, sizeof(*dev->mpshared));
 218
 219         if (rte_strscpy(dev->mpshared->name, name, RTE_DEV_NAME_MAX_LEN) < 0) {
 220                 GPU_LOG(ERR, "device name too long: %s", name);
 221                 rte_errno = ENAMETOOLONG;
 222                 return NULL;
 223         }
 224         dev->mpshared->info.name = dev->mpshared->name;
 225         dev->mpshared->info.dev_id = dev_id;
 226         dev->mpshared->info.numa_node = -1;
 227         dev->mpshared->info.parent = RTE_GPU_ID_NONE;
 228         TAILQ_INIT(&dev->callbacks);
 229         __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
 230
 231         gpu_count++;
 232         GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
 233                         name, dev_id, gpu_count);
 234         return dev;
 235 }
 236
 237 struct rte_gpu *
 238 rte_gpu_attach(const char *name)
 239 {
 240         int16_t dev_id;
 241         struct rte_gpu *dev;
 242         struct rte_gpu_mpshared *shared_dev;
 243
 244         if (rte_eal_process_type() != RTE_PROC_SECONDARY) {
 245                 GPU_LOG(ERR, "only secondary process can attach device");
 246                 rte_errno = EPERM;
 247                 return NULL;
 248         }
 249         if (name == NULL) {
 250                 GPU_LOG(ERR, "attach device without a name");
 251                 rte_errno = EINVAL;
 252                 return NULL;
 253         }
 254
 255         /* implicit initialization of library before adding first device */
 256         if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0)
 257                 return NULL;
 258
 259         /* initialize shared memory before adding first device */
 260         if (gpu_shared_mem == NULL && gpu_shared_mem_init() < 0)
 261                 return NULL;
 262
 263         for (dev_id = 0; dev_id < gpu_max; dev_id++) {
 264                 shared_dev = &gpu_shared_mem->gpus[dev_id];
 265                 if (strncmp(name, shared_dev->name, RTE_DEV_NAME_MAX_LEN) == 0)
 266                         break;
 267         }
 268         if (dev_id >= gpu_max) {
 269                 GPU_LOG(ERR, "device with name %s not found", name);
 270                 rte_errno = ENOENT;
 271                 return NULL;
 272         }
 273         dev = &gpus[dev_id];
 274         memset(dev, 0, sizeof(*dev));
 275
 276         TAILQ_INIT(&dev->callbacks);
 277         dev->mpshared = shared_dev;
 278         __atomic_fetch_add(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
 279
 280         gpu_count++;
 281         GPU_LOG(DEBUG, "attached device %s (id %d) of total %d",
 282                         name, dev_id, gpu_count);
 283         return dev;
 284 }
 285
 286 int16_t
 287 rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context)
 288 {
 289         struct rte_gpu *dev;
 290
 291         if (!rte_gpu_is_valid(parent)) {
 292                 GPU_LOG(ERR, "add child to invalid parent ID %d", parent);
 293                 rte_errno = ENODEV;
 294                 return -rte_errno;
 295         }
 296
 297         dev = rte_gpu_allocate(name);
 298         if (dev == NULL)
 299                 return -rte_errno;
 300
 301         dev->mpshared->info.parent = parent;
 302         dev->mpshared->info.context = child_context;
 303
 304         rte_gpu_complete_new(dev);
 305         return dev->mpshared->info.dev_id;
 306 }
 307
 308 void
 309 rte_gpu_complete_new(struct rte_gpu *dev)
 310 {
 311         if (dev == NULL)
 312                 return;
 313
 314         dev->process_state = RTE_GPU_STATE_INITIALIZED;
 315         rte_gpu_notify(dev, RTE_GPU_EVENT_NEW);
 316 }
 317
 318 int
 319 rte_gpu_release(struct rte_gpu *dev)
 320 {
 321         int16_t dev_id, child;
 322
 323         if (dev == NULL) {
 324                 rte_errno = ENODEV;
 325                 return -rte_errno;
 326         }
 327         dev_id = dev->mpshared->info.dev_id;
 328         RTE_GPU_FOREACH_CHILD(child, dev_id) {
 329                 GPU_LOG(ERR, "cannot release device %d with child %d",
 330                                 dev_id, child);
 331                 rte_errno = EBUSY;
 332                 return -rte_errno;
 333         }
 334
 335         GPU_LOG(DEBUG, "free device %s (id %d)",
 336                         dev->mpshared->info.name, dev->mpshared->info.dev_id);
 337         rte_gpu_notify(dev, RTE_GPU_EVENT_DEL);
 338
 339         gpu_free_callbacks(dev);
 340         dev->process_state = RTE_GPU_STATE_UNUSED;
 341         __atomic_fetch_sub(&dev->mpshared->process_refcnt, 1, __ATOMIC_RELAXED);
 342         gpu_count--;
 343
 344         return 0;
 345 }
 346
 347 int
 348 rte_gpu_close(int16_t dev_id)
 349 {
 350         int firsterr, binerr;
 351         int *lasterr = &firsterr;
 352         struct rte_gpu *dev;
 353
 354         dev = gpu_get_by_id(dev_id);
 355         if (dev == NULL) {
 356                 GPU_LOG(ERR, "close invalid device ID %d", dev_id);
 357                 rte_errno = ENODEV;
 358                 return -rte_errno;
 359         }
 360
 361         if (dev->ops.dev_close != NULL) {
 362                 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev));
 363                 if (*lasterr != 0)
 364                         lasterr = &binerr;
 365         }
 366
 367         *lasterr = rte_gpu_release(dev);
 368
 369         rte_errno = -firsterr;
 370         return firsterr;
 371 }
 372
 373 int
 374 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event,
 375                 rte_gpu_callback_t *function, void *user_data)
 376 {
 377         int16_t next_dev, last_dev;
 378         struct rte_gpu_callback_list *callbacks;
 379         struct rte_gpu_callback *callback;
 380
 381         if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
 382                 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id);
 383                 rte_errno = ENODEV;
 384                 return -rte_errno;
 385         }
 386         if (function == NULL) {
 387                 GPU_LOG(ERR, "cannot register callback without function");
 388                 rte_errno = EINVAL;
 389                 return -rte_errno;
 390         }
 391
 392         if (dev_id == RTE_GPU_ID_ANY) {
 393                 next_dev = 0;
 394                 last_dev = gpu_max - 1;
 395         } else {
 396                 next_dev = last_dev = dev_id;
 397         }
 398
 399         rte_rwlock_write_lock(&gpu_callback_lock);
 400         do {
 401                 callbacks = &gpus[next_dev].callbacks;
 402
 403                 /* check if not already registered */
 404                 TAILQ_FOREACH(callback, callbacks, next) {
 405                         if (callback->event == event &&
 406                                         callback->function == function &&
 407                                         callback->user_data == user_data) {
 408                                 GPU_LOG(INFO, "callback already registered");
 409                                 return 0;
 410                         }
 411                 }
 412
 413                 callback = malloc(sizeof(*callback));
 414                 if (callback == NULL) {
 415                         GPU_LOG(ERR, "cannot allocate callback");
 416                         return -ENOMEM;
 417                 }
 418                 callback->function = function;
 419                 callback->user_data = user_data;
 420                 callback->event = event;
 421                 TAILQ_INSERT_TAIL(callbacks, callback, next);
 422
 423         } while (++next_dev <= last_dev);
 424         rte_rwlock_write_unlock(&gpu_callback_lock);
 425
 426         return 0;
 427 }
 428
 429 int
 430 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event,
 431                 rte_gpu_callback_t *function, void *user_data)
 432 {
 433         int16_t next_dev, last_dev;
 434         struct rte_gpu_callback_list *callbacks;
 435         struct rte_gpu_callback *callback, *nextcb;
 436
 437         if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
 438                 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id);
 439                 rte_errno = ENODEV;
 440                 return -rte_errno;
 441         }
 442         if (function == NULL) {
 443                 GPU_LOG(ERR, "cannot unregister callback without function");
 444                 rte_errno = EINVAL;
 445                 return -rte_errno;
 446         }
 447
 448         if (dev_id == RTE_GPU_ID_ANY) {
 449                 next_dev = 0;
 450                 last_dev = gpu_max - 1;
 451         } else {
 452                 next_dev = last_dev = dev_id;
 453         }
 454
 455         rte_rwlock_write_lock(&gpu_callback_lock);
 456         do {
 457                 callbacks = &gpus[next_dev].callbacks;
 458                 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
 459                         if (callback->event != event ||
 460                                         callback->function != function ||
 461                                         (callback->user_data != user_data &&
 462                                         user_data != (void *)-1))
 463                                 continue;
 464                         TAILQ_REMOVE(callbacks, callback, next);
 465                         free(callback);
 466                 }
 467         } while (++next_dev <= last_dev);
 468         rte_rwlock_write_unlock(&gpu_callback_lock);
 469
 470         return 0;
 471 }
 472
 473 static void
 474 gpu_free_callbacks(struct rte_gpu *dev)
 475 {
 476         struct rte_gpu_callback_list *callbacks;
 477         struct rte_gpu_callback *callback, *nextcb;
 478
 479         callbacks = &dev->callbacks;
 480         rte_rwlock_write_lock(&gpu_callback_lock);
 481         RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
 482                 TAILQ_REMOVE(callbacks, callback, next);
 483                 free(callback);
 484         }
 485         rte_rwlock_write_unlock(&gpu_callback_lock);
 486 }
 487
 488 void
 489 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event)
 490 {
 491         int16_t dev_id;
 492         struct rte_gpu_callback *callback;
 493
 494         dev_id = dev->mpshared->info.dev_id;
 495         rte_rwlock_read_lock(&gpu_callback_lock);
 496         TAILQ_FOREACH(callback, &dev->callbacks, next) {
 497                 if (callback->event != event || callback->function == NULL)
 498                         continue;
 499                 callback->function(dev_id, event, callback->user_data);
 500         }
 501         rte_rwlock_read_unlock(&gpu_callback_lock);
 502 }
 503
 504 int
 505 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info)
 506 {
 507         struct rte_gpu *dev;
 508
 509         dev = gpu_get_by_id(dev_id);
 510         if (dev == NULL) {
 511                 GPU_LOG(ERR, "query invalid device ID %d", dev_id);
 512                 rte_errno = ENODEV;
 513                 return -rte_errno;
 514         }
 515         if (info == NULL) {
 516                 GPU_LOG(ERR, "query without storage");
 517                 rte_errno = EINVAL;
 518                 return -rte_errno;
 519         }
 520
 521         if (dev->ops.dev_info_get == NULL) {
 522                 *info = dev->mpshared->info;
 523                 return 0;
 524         }
 525         return GPU_DRV_RET(dev->ops.dev_info_get(dev, info));
 526 }
 527
 528 void *
 529 rte_gpu_mem_alloc(int16_t dev_id, size_t size, unsigned int align)
 530 {
 531         struct rte_gpu *dev;
 532         void *ptr;
 533         int ret;
 534
 535         dev = gpu_get_by_id(dev_id);
 536         if (dev == NULL) {
 537                 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id);
 538                 rte_errno = ENODEV;
 539                 return NULL;
 540         }
 541
 542         if (dev->ops.mem_alloc == NULL) {
 543                 GPU_LOG(ERR, "mem allocation not supported");
 544                 rte_errno = ENOTSUP;
 545                 return NULL;
 546         }
 547
 548         if (size == 0) /* dry-run */
 549                 return NULL;
 550
 551         if (align && !rte_is_power_of_2(align)) {
 552                 GPU_LOG(ERR, "requested alignment is not a power of two %u", align);
 553                 rte_errno = EINVAL;
 554                 return NULL;
 555         }
 556
 557         ret = dev->ops.mem_alloc(dev, size, align, &ptr);
 558
 559         switch (ret) {
 560         case 0:
 561                 return ptr;
 562         case -ENOMEM:
 563         case -E2BIG:
 564                 rte_errno = -ret;
 565                 return NULL;
 566         default:
 567                 rte_errno = -EPERM;
 568                 return NULL;
 569         }
 570 }
 571
 572 int
 573 rte_gpu_mem_free(int16_t dev_id, void *ptr)
 574 {
 575         struct rte_gpu *dev;
 576
 577         dev = gpu_get_by_id(dev_id);
 578         if (dev == NULL) {
 579                 GPU_LOG(ERR, "free mem for invalid device ID %d", dev_id);
 580                 rte_errno = ENODEV;
 581                 return -rte_errno;
 582         }
 583
 584         if (dev->ops.mem_free == NULL) {
 585                 rte_errno = ENOTSUP;
 586                 return -rte_errno;
 587         }
 588
 589         if (ptr == NULL) /* dry-run */
 590                 return 0;
 591
 592         return GPU_DRV_RET(dev->ops.mem_free(dev, ptr));
 593 }
 594
 595 int
 596 rte_gpu_mem_register(int16_t dev_id, size_t size, void *ptr)
 597 {
 598         struct rte_gpu *dev;
 599
 600         dev = gpu_get_by_id(dev_id);
 601         if (dev == NULL) {
 602                 GPU_LOG(ERR, "alloc mem for invalid device ID %d", dev_id);
 603                 rte_errno = ENODEV;
 604                 return -rte_errno;
 605         }
 606
 607         if (dev->ops.mem_register == NULL) {
 608                 GPU_LOG(ERR, "mem registration not supported");
 609                 rte_errno = ENOTSUP;
 610                 return -rte_errno;
 611         }
 612
 613         if (ptr == NULL || size == 0) /* dry-run  */
 614                 return 0;
 615
 616         return GPU_DRV_RET(dev->ops.mem_register(dev, size, ptr));
 617 }
 618
 619 int
 620 rte_gpu_mem_unregister(int16_t dev_id, void *ptr)
 621 {
 622         struct rte_gpu *dev;
 623
 624         dev = gpu_get_by_id(dev_id);
 625         if (dev == NULL) {
 626                 GPU_LOG(ERR, "unregister mem for invalid device ID %d", dev_id);
 627                 rte_errno = ENODEV;
 628                 return -rte_errno;
 629         }
 630
 631         if (dev->ops.mem_unregister == NULL) {
 632                 rte_errno = ENOTSUP;
 633                 return -rte_errno;
 634         }
 635
 636         if (ptr == NULL) /* dry-run */
 637                 return 0;
 638
 639         return GPU_DRV_RET(dev->ops.mem_unregister(dev, ptr));
 640 }
 641
 642 void *
 643 rte_gpu_mem_cpu_map(int16_t dev_id, size_t size, void *ptr)
 644 {
 645         struct rte_gpu *dev;
 646         void *ptr_out;
 647         int ret;
 648
 649         dev = gpu_get_by_id(dev_id);
 650         if (dev == NULL) {
 651                 GPU_LOG(ERR, "mem CPU map for invalid device ID %d", dev_id);
 652                 rte_errno = ENODEV;
 653                 return NULL;
 654         }
 655
 656         if (dev->ops.mem_cpu_map == NULL) {
 657                 GPU_LOG(ERR, "mem CPU map not supported");
 658                 rte_errno = ENOTSUP;
 659                 return NULL;
 660         }
 661
 662         if (ptr == NULL || size == 0) /* dry-run  */
 663                 return NULL;
 664
 665         ret = GPU_DRV_RET(dev->ops.mem_cpu_map(dev, size, ptr, &ptr_out));
 666
 667         switch (ret) {
 668         case 0:
 669                 return ptr_out;
 670         case -ENOMEM:
 671         case -E2BIG:
 672                 rte_errno = -ret;
 673                 return NULL;
 674         default:
 675                 rte_errno = -EPERM;
 676                 return NULL;
 677         }
 678 }
 679
 680 int
 681 rte_gpu_mem_cpu_unmap(int16_t dev_id, void *ptr)
 682 {
 683         struct rte_gpu *dev;
 684
 685         dev = gpu_get_by_id(dev_id);
 686         if (dev == NULL) {
 687                 GPU_LOG(ERR, "cpu_unmap mem for invalid device ID %d", dev_id);
 688                 rte_errno = ENODEV;
 689                 return -rte_errno;
 690         }
 691
 692         if (dev->ops.mem_cpu_unmap == NULL) {
 693                 rte_errno = ENOTSUP;
 694                 return -rte_errno;
 695         }
 696
 697         if (ptr == NULL) /* dry-run */
 698                 return 0;
 699
 700         return GPU_DRV_RET(dev->ops.mem_cpu_unmap(dev, ptr));
 701 }
 702
 703 int
 704 rte_gpu_wmb(int16_t dev_id)
 705 {
 706         struct rte_gpu *dev;
 707
 708         dev = gpu_get_by_id(dev_id);
 709         if (dev == NULL) {
 710                 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id);
 711                 rte_errno = ENODEV;
 712                 return -rte_errno;
 713         }
 714
 715         if (dev->ops.wmb == NULL) {
 716                 rte_errno = ENOTSUP;
 717                 return -rte_errno;
 718         }
 719         return GPU_DRV_RET(dev->ops.wmb(dev));
 720 }
 721
 722 int
 723 rte_gpu_comm_create_flag(uint16_t dev_id, struct rte_gpu_comm_flag *devflag,
 724                 enum rte_gpu_comm_flag_type mtype)
 725 {
 726         size_t flag_size;
 727         int ret;
 728
 729         if (devflag == NULL) {
 730                 rte_errno = EINVAL;
 731                 return -rte_errno;
 732         }
 733         if (mtype != RTE_GPU_COMM_FLAG_CPU) {
 734                 rte_errno = EINVAL;
 735                 return -rte_errno;
 736         }
 737
 738         flag_size = sizeof(uint32_t);
 739
 740         devflag->ptr = rte_zmalloc(NULL, flag_size, 0);
 741         if (devflag->ptr == NULL) {
 742                 rte_errno = ENOMEM;
 743                 return -rte_errno;
 744         }
 745
 746         ret = rte_gpu_mem_register(dev_id, flag_size, devflag->ptr);
 747         if (ret < 0) {
 748                 rte_errno = ENOMEM;
 749                 return -rte_errno;
 750         }
 751
 752         devflag->mtype = mtype;
 753         devflag->dev_id = dev_id;
 754
 755         return 0;
 756 }
 757
 758 int
 759 rte_gpu_comm_destroy_flag(struct rte_gpu_comm_flag *devflag)
 760 {
 761         int ret;
 762
 763         if (devflag == NULL) {
 764                 rte_errno = EINVAL;
 765                 return -rte_errno;
 766         }
 767
 768         ret = rte_gpu_mem_unregister(devflag->dev_id, devflag->ptr);
 769         if (ret < 0) {
 770                 rte_errno = EINVAL;
 771                 return -1;
 772         }
 773
 774         rte_free(devflag->ptr);
 775
 776         return 0;
 777 }
 778
 779 int
 780 rte_gpu_comm_set_flag(struct rte_gpu_comm_flag *devflag, uint32_t val)
 781 {
 782         if (devflag == NULL) {
 783                 rte_errno = EINVAL;
 784                 return -rte_errno;
 785         }
 786
 787         if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) {
 788                 rte_errno = EINVAL;
 789                 return -rte_errno;
 790         }
 791
 792         RTE_GPU_VOLATILE(*devflag->ptr) = val;
 793
 794         return 0;
 795 }
 796
 797 int
 798 rte_gpu_comm_get_flag_value(struct rte_gpu_comm_flag *devflag, uint32_t *val)
 799 {
 800         if (devflag == NULL) {
 801                 rte_errno = EINVAL;
 802                 return -rte_errno;
 803         }
 804         if (devflag->mtype != RTE_GPU_COMM_FLAG_CPU) {
 805                 rte_errno = EINVAL;
 806                 return -rte_errno;
 807         }
 808
 809         *val = RTE_GPU_VOLATILE(*devflag->ptr);
 810
 811         return 0;
 812 }
 813
 814 struct rte_gpu_comm_list *
 815 rte_gpu_comm_create_list(uint16_t dev_id,
 816                 uint32_t num_comm_items)
 817 {
 818         struct rte_gpu_comm_list *comm_list;
 819         uint32_t idx_l;
 820         int ret;
 821         struct rte_gpu *dev;
 822
 823         if (num_comm_items == 0) {
 824                 rte_errno = EINVAL;
 825                 return NULL;
 826         }
 827
 828         dev = gpu_get_by_id(dev_id);
 829         if (dev == NULL) {
 830                 GPU_LOG(ERR, "memory barrier for invalid device ID %d", dev_id);
 831                 rte_errno = ENODEV;
 832                 return NULL;
 833         }
 834
 835         comm_list = rte_zmalloc(NULL,
 836                         sizeof(struct rte_gpu_comm_list) * num_comm_items, 0);
 837         if (comm_list == NULL) {
 838                 rte_errno = ENOMEM;
 839                 return NULL;
 840         }
 841
 842         ret = rte_gpu_mem_register(dev_id,
 843                         sizeof(struct rte_gpu_comm_list) * num_comm_items, comm_list);
 844         if (ret < 0) {
 845                 rte_errno = ENOMEM;
 846                 return NULL;
 847         }
 848
 849         for (idx_l = 0; idx_l < num_comm_items; idx_l++) {
 850                 comm_list[idx_l].pkt_list = rte_zmalloc(NULL,
 851                                 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX, 0);
 852                 if (comm_list[idx_l].pkt_list == NULL) {
 853                         rte_errno = ENOMEM;
 854                         return NULL;
 855                 }
 856
 857                 ret = rte_gpu_mem_register(dev_id,
 858                                 sizeof(struct rte_gpu_comm_pkt) * RTE_GPU_COMM_LIST_PKTS_MAX,
 859                                 comm_list[idx_l].pkt_list);
 860                 if (ret < 0) {
 861                         rte_errno = ENOMEM;
 862                         return NULL;
 863                 }
 864
 865                 RTE_GPU_VOLATILE(comm_list[idx_l].status) = RTE_GPU_COMM_LIST_FREE;
 866                 comm_list[idx_l].num_pkts = 0;
 867                 comm_list[idx_l].dev_id = dev_id;
 868
 869                 comm_list[idx_l].mbufs = rte_zmalloc(NULL,
 870                                 sizeof(struct rte_mbuf *) * RTE_GPU_COMM_LIST_PKTS_MAX, 0);
 871                 if (comm_list[idx_l].mbufs == NULL) {
 872                         rte_errno = ENOMEM;
 873                         return NULL;
 874                 }
 875         }
 876
 877         return comm_list;
 878 }
 879
 880 int
 881 rte_gpu_comm_destroy_list(struct rte_gpu_comm_list *comm_list,
 882                 uint32_t num_comm_items)
 883 {
 884         uint32_t idx_l;
 885         int ret;
 886         uint16_t dev_id;
 887
 888         if (comm_list == NULL) {
 889                 rte_errno = EINVAL;
 890                 return -rte_errno;
 891         }
 892
 893         dev_id = comm_list[0].dev_id;
 894
 895         for (idx_l = 0; idx_l < num_comm_items; idx_l++) {
 896                 ret = rte_gpu_mem_unregister(dev_id, comm_list[idx_l].pkt_list);
 897                 if (ret < 0) {
 898                         rte_errno = EINVAL;
 899                         return -1;
 900                 }
 901
 902                 rte_free(comm_list[idx_l].pkt_list);
 903                 rte_free(comm_list[idx_l].mbufs);
 904         }
 905
 906         ret = rte_gpu_mem_unregister(dev_id, comm_list);
 907         if (ret < 0) {
 908                 rte_errno = EINVAL;
 909                 return -1;
 910         }
 911
 912         rte_free(comm_list);
 913
 914         return 0;
 915 }
 916
 917 int
 918 rte_gpu_comm_populate_list_pkts(struct rte_gpu_comm_list *comm_list_item,
 919                 struct rte_mbuf **mbufs, uint32_t num_mbufs)
 920 {
 921         uint32_t idx;
 922
 923         if (comm_list_item == NULL || comm_list_item->pkt_list == NULL ||
 924                         mbufs == NULL || num_mbufs > RTE_GPU_COMM_LIST_PKTS_MAX) {
 925                 rte_errno = EINVAL;
 926                 return -rte_errno;
 927         }
 928
 929         for (idx = 0; idx < num_mbufs; idx++) {
 930                 /* support only unchained mbufs */
 931                 if (unlikely((mbufs[idx]->nb_segs > 1) ||
 932                                 (mbufs[idx]->next != NULL) ||
 933                                 (mbufs[idx]->data_len != mbufs[idx]->pkt_len))) {
 934                         rte_errno = ENOTSUP;
 935                         return -rte_errno;
 936                 }
 937                 comm_list_item->pkt_list[idx].addr =
 938                                 rte_pktmbuf_mtod_offset(mbufs[idx], uintptr_t, 0);
 939                 comm_list_item->pkt_list[idx].size = mbufs[idx]->pkt_len;
 940                 comm_list_item->mbufs[idx] = mbufs[idx];
 941         }
 942
 943         RTE_GPU_VOLATILE(comm_list_item->num_pkts) = num_mbufs;
 944         rte_gpu_wmb(comm_list_item->dev_id);
 945         RTE_GPU_VOLATILE(comm_list_item->status) = RTE_GPU_COMM_LIST_READY;
 946
 947         return 0;
 948 }
 949
 950 int
 951 rte_gpu_comm_cleanup_list(struct rte_gpu_comm_list *comm_list_item)
 952 {
 953         uint32_t idx = 0;
 954
 955         if (comm_list_item == NULL) {
 956                 rte_errno = EINVAL;
 957                 return -rte_errno;
 958         }
 959
 960         if (RTE_GPU_VOLATILE(comm_list_item->status) ==
 961                         RTE_GPU_COMM_LIST_READY) {
 962                 GPU_LOG(ERR, "packet list is still in progress");
 963                 rte_errno = EINVAL;
 964                 return -rte_errno;
 965         }
 966
 967         for (idx = 0; idx < RTE_GPU_COMM_LIST_PKTS_MAX; idx++) {
 968                 if (comm_list_item->pkt_list[idx].addr == 0)
 969                         break;
 970
 971                 comm_list_item->pkt_list[idx].addr = 0;
 972                 comm_list_item->pkt_list[idx].size = 0;
 973                 comm_list_item->mbufs[idx] = NULL;
 974         }
 975
 976         RTE_GPU_VOLATILE(comm_list_item->status) = RTE_GPU_COMM_LIST_FREE;
 977         RTE_GPU_VOLATILE(comm_list_item->num_pkts) = 0;
 978         rte_mb();
 979
 980         return 0;
 981 }