lib/librte_eal/common/rte_service.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <unistd.h>
   7 #include <inttypes.h>
   8 #include <limits.h>
   9 #include <string.h>
  10
  11 #include <rte_compat.h>
  12 #include <rte_service.h>
  13 #include <rte_service_component.h>
  14
  15 #include <rte_eal.h>
  16 #include <rte_lcore.h>
  17 #include <rte_common.h>
  18 #include <rte_debug.h>
  19 #include <rte_cycles.h>
  20 #include <rte_atomic.h>
  21 #include <rte_memory.h>
  22 #include <rte_malloc.h>
  23 #include <rte_spinlock.h>
  24
  25 #include "eal_private.h"
  26
  27 #define RTE_SERVICE_NUM_MAX 64
  28
  29 #define SERVICE_F_REGISTERED    (1 << 0)
  30 #define SERVICE_F_STATS_ENABLED (1 << 1)
  31 #define SERVICE_F_START_CHECK   (1 << 2)
  32
  33 /* runstates for services and lcores, denoting if they are active or not */
  34 #define RUNSTATE_STOPPED 0
  35 #define RUNSTATE_RUNNING 1
  36
  37 /* internal representation of a service */
  38 struct rte_service_spec_impl {
  39         /* public part of the struct */
  40         struct rte_service_spec spec;
  41
  42         /* spin lock that when set indicates a service core is currently
  43          * running this service callback. When not set, a core may take the
  44          * lock and then run the service callback.
  45          */
  46         rte_spinlock_t execute_lock;
  47
  48         /* API set/get-able variables */
  49         int8_t app_runstate;
  50         int8_t comp_runstate;
  51         uint8_t internal_flags;
  52
  53         /* per service statistics */
  54         /* Indicates how many cores the service is mapped to run on.
  55          * It does not indicate the number of cores the service is running
  56          * on currently.
  57          */
  58         uint32_t num_mapped_cores;
  59         uint64_t calls;
  60         uint64_t cycles_spent;
  61 } __rte_cache_aligned;
  62
  63 /* the internal values of a service core */
  64 struct core_state {
  65         /* map of services IDs are run on this core */
  66         uint64_t service_mask;
  67         uint8_t runstate; /* running or stopped */
  68         uint8_t thread_active; /* indicates when thread is in service_run() */
  69         uint8_t is_service_core; /* set if core is currently a service core */
  70         uint8_t service_active_on_lcore[RTE_SERVICE_NUM_MAX];
  71         uint64_t loops;
  72         uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
  73 } __rte_cache_aligned;
  74
  75 static uint32_t rte_service_count;
  76 static struct rte_service_spec_impl *rte_services;
  77 static struct core_state *lcore_states;
  78 static uint32_t rte_service_library_initialized;
  79
  80 int32_t
  81 rte_service_init(void)
  82 {
  83         if (rte_service_library_initialized) {
  84                 RTE_LOG(NOTICE, EAL,
  85                         "service library init() called, init flag %d\n",
  86                         rte_service_library_initialized);
  87                 return -EALREADY;
  88         }
  89
  90         rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX,
  91                         sizeof(struct rte_service_spec_impl),
  92                         RTE_CACHE_LINE_SIZE);
  93         if (!rte_services) {
  94                 RTE_LOG(ERR, EAL, "error allocating rte services array\n");
  95                 goto fail_mem;
  96         }
  97
  98         lcore_states = rte_calloc("rte_service_core_states", RTE_MAX_LCORE,
  99                         sizeof(struct core_state), RTE_CACHE_LINE_SIZE);
 100         if (!lcore_states) {
 101                 RTE_LOG(ERR, EAL, "error allocating core states array\n");
 102                 goto fail_mem;
 103         }
 104
 105         int i;
 106         int count = 0;
 107         struct rte_config *cfg = rte_eal_get_configuration();
 108         for (i = 0; i < RTE_MAX_LCORE; i++) {
 109                 if (lcore_config[i].core_role == ROLE_SERVICE) {
 110                         if ((unsigned int)i == cfg->main_lcore)
 111                                 continue;
 112                         rte_service_lcore_add(i);
 113                         count++;
 114                 }
 115         }
 116
 117         rte_service_library_initialized = 1;
 118         return 0;
 119 fail_mem:
 120         rte_free(rte_services);
 121         rte_free(lcore_states);
 122         return -ENOMEM;
 123 }
 124
 125 void
 126 rte_service_finalize(void)
 127 {
 128         if (!rte_service_library_initialized)
 129                 return;
 130
 131         rte_service_lcore_reset_all();
 132         rte_eal_mp_wait_lcore();
 133
 134         rte_free(rte_services);
 135         rte_free(lcore_states);
 136
 137         rte_service_library_initialized = 0;
 138 }
 139
 140 /* returns 1 if service is registered and has not been unregistered
 141  * Returns 0 if service never registered, or has been unregistered
 142  */
 143 static inline int
 144 service_valid(uint32_t id)
 145 {
 146         return !!(rte_services[id].internal_flags & SERVICE_F_REGISTERED);
 147 }
 148
 149 static struct rte_service_spec_impl *
 150 service_get(uint32_t id)
 151 {
 152         return &rte_services[id];
 153 }
 154
 155 /* validate ID and retrieve service pointer, or return error value */
 156 #define SERVICE_VALID_GET_OR_ERR_RET(id, service, retval) do {          \
 157         if (id >= RTE_SERVICE_NUM_MAX || !service_valid(id))            \
 158                 return retval;                                          \
 159         service = &rte_services[id];                                    \
 160 } while (0)
 161
 162 /* returns 1 if statistics should be collected for service
 163  * Returns 0 if statistics should not be collected for service
 164  */
 165 static inline int
 166 service_stats_enabled(struct rte_service_spec_impl *impl)
 167 {
 168         return !!(impl->internal_flags & SERVICE_F_STATS_ENABLED);
 169 }
 170
 171 static inline int
 172 service_mt_safe(struct rte_service_spec_impl *s)
 173 {
 174         return !!(s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE);
 175 }
 176
 177 int32_t
 178 rte_service_set_stats_enable(uint32_t id, int32_t enabled)
 179 {
 180         struct rte_service_spec_impl *s;
 181         SERVICE_VALID_GET_OR_ERR_RET(id, s, 0);
 182
 183         if (enabled)
 184                 s->internal_flags |= SERVICE_F_STATS_ENABLED;
 185         else
 186                 s->internal_flags &= ~(SERVICE_F_STATS_ENABLED);
 187
 188         return 0;
 189 }
 190
 191 int32_t
 192 rte_service_set_runstate_mapped_check(uint32_t id, int32_t enabled)
 193 {
 194         struct rte_service_spec_impl *s;
 195         SERVICE_VALID_GET_OR_ERR_RET(id, s, 0);
 196
 197         if (enabled)
 198                 s->internal_flags |= SERVICE_F_START_CHECK;
 199         else
 200                 s->internal_flags &= ~(SERVICE_F_START_CHECK);
 201
 202         return 0;
 203 }
 204
 205 uint32_t
 206 rte_service_get_count(void)
 207 {
 208         return rte_service_count;
 209 }
 210
 211 int32_t
 212 rte_service_get_by_name(const char *name, uint32_t *service_id)
 213 {
 214         if (!service_id)
 215                 return -EINVAL;
 216
 217         int i;
 218         for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
 219                 if (service_valid(i) &&
 220                                 strcmp(name, rte_services[i].spec.name) == 0) {
 221                         *service_id = i;
 222                         return 0;
 223                 }
 224         }
 225
 226         return -ENODEV;
 227 }
 228
 229 const char *
 230 rte_service_get_name(uint32_t id)
 231 {
 232         struct rte_service_spec_impl *s;
 233         SERVICE_VALID_GET_OR_ERR_RET(id, s, 0);
 234         return s->spec.name;
 235 }
 236
 237 int32_t
 238 rte_service_probe_capability(uint32_t id, uint32_t capability)
 239 {
 240         struct rte_service_spec_impl *s;
 241         SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
 242         return !!(s->spec.capabilities & capability);
 243 }
 244
 245 int32_t
 246 rte_service_component_register(const struct rte_service_spec *spec,
 247                                uint32_t *id_ptr)
 248 {
 249         uint32_t i;
 250         int32_t free_slot = -1;
 251
 252         if (spec->callback == NULL || strlen(spec->name) == 0)
 253                 return -EINVAL;
 254
 255         for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
 256                 if (!service_valid(i)) {
 257                         free_slot = i;
 258                         break;
 259                 }
 260         }
 261
 262         if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))
 263                 return -ENOSPC;
 264
 265         struct rte_service_spec_impl *s = &rte_services[free_slot];
 266         s->spec = *spec;
 267         s->internal_flags |= SERVICE_F_REGISTERED | SERVICE_F_START_CHECK;
 268
 269         rte_service_count++;
 270
 271         if (id_ptr)
 272                 *id_ptr = free_slot;
 273
 274         return 0;
 275 }
 276
 277 int32_t
 278 rte_service_component_unregister(uint32_t id)
 279 {
 280         uint32_t i;
 281         struct rte_service_spec_impl *s;
 282         SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
 283
 284         rte_service_count--;
 285
 286         s->internal_flags &= ~(SERVICE_F_REGISTERED);
 287
 288         /* clear the run-bit in all cores */
 289         for (i = 0; i < RTE_MAX_LCORE; i++)
 290                 lcore_states[i].service_mask &= ~(UINT64_C(1) << id);
 291
 292         memset(&rte_services[id], 0, sizeof(struct rte_service_spec_impl));
 293
 294         return 0;
 295 }
 296
 297 int32_t
 298 rte_service_component_runstate_set(uint32_t id, uint32_t runstate)
 299 {
 300         struct rte_service_spec_impl *s;
 301         SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
 302
 303         /* comp_runstate act as the guard variable. Use store-release
 304          * memory order. This synchronizes with load-acquire in
 305          * service_run and service_runstate_get function.
 306          */
 307         if (runstate)
 308                 __atomic_store_n(&s->comp_runstate, RUNSTATE_RUNNING,
 309                         __ATOMIC_RELEASE);
 310         else
 311                 __atomic_store_n(&s->comp_runstate, RUNSTATE_STOPPED,
 312                         __ATOMIC_RELEASE);
 313
 314         return 0;
 315 }
 316
 317 int32_t
 318 rte_service_runstate_set(uint32_t id, uint32_t runstate)
 319 {
 320         struct rte_service_spec_impl *s;
 321         SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
 322
 323         /* app_runstate act as the guard variable. Use store-release
 324          * memory order. This synchronizes with load-acquire in
 325          * service_run runstate_get function.
 326          */
 327         if (runstate)
 328                 __atomic_store_n(&s->app_runstate, RUNSTATE_RUNNING,
 329                         __ATOMIC_RELEASE);
 330         else
 331                 __atomic_store_n(&s->app_runstate, RUNSTATE_STOPPED,
 332                         __ATOMIC_RELEASE);
 333
 334         return 0;
 335 }
 336
 337 int32_t
 338 rte_service_runstate_get(uint32_t id)
 339 {
 340         struct rte_service_spec_impl *s;
 341         SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
 342
 343         /* comp_runstate and app_runstate act as the guard variables.
 344          * Use load-acquire memory order. This synchronizes with
 345          * store-release in service state set functions.
 346          */
 347         if (__atomic_load_n(&s->comp_runstate, __ATOMIC_ACQUIRE) ==
 348                         RUNSTATE_RUNNING &&
 349             __atomic_load_n(&s->app_runstate, __ATOMIC_ACQUIRE) ==
 350                         RUNSTATE_RUNNING) {
 351                 int check_disabled = !(s->internal_flags &
 352                         SERVICE_F_START_CHECK);
 353                 int lcore_mapped = (__atomic_load_n(&s->num_mapped_cores,
 354                         __ATOMIC_RELAXED) > 0);
 355
 356                 return (check_disabled | lcore_mapped);
 357         } else
 358                 return 0;
 359
 360 }
 361
 362 static inline void
 363 service_runner_do_callback(struct rte_service_spec_impl *s,
 364                            struct core_state *cs, uint32_t service_idx)
 365 {
 366         void *userdata = s->spec.callback_userdata;
 367
 368         if (service_stats_enabled(s)) {
 369                 uint64_t start = rte_rdtsc();
 370                 s->spec.callback(userdata);
 371                 uint64_t end = rte_rdtsc();
 372                 s->cycles_spent += end - start;
 373                 cs->calls_per_service[service_idx]++;
 374                 s->calls++;
 375         } else
 376                 s->spec.callback(userdata);
 377 }
 378
 379
 380 /* Expects the service 's' is valid. */
 381 static int32_t
 382 service_run(uint32_t i, struct core_state *cs, uint64_t service_mask,
 383             struct rte_service_spec_impl *s, uint32_t serialize_mt_unsafe)
 384 {
 385         if (!s)
 386                 return -EINVAL;
 387
 388         /* comp_runstate and app_runstate act as the guard variables.
 389          * Use load-acquire memory order. This synchronizes with
 390          * store-release in service state set functions.
 391          */
 392         if (__atomic_load_n(&s->comp_runstate, __ATOMIC_ACQUIRE) !=
 393                         RUNSTATE_RUNNING ||
 394             __atomic_load_n(&s->app_runstate, __ATOMIC_ACQUIRE) !=
 395                         RUNSTATE_RUNNING ||
 396             !(service_mask & (UINT64_C(1) << i))) {
 397                 cs->service_active_on_lcore[i] = 0;
 398                 return -ENOEXEC;
 399         }
 400
 401         cs->service_active_on_lcore[i] = 1;
 402
 403         if ((service_mt_safe(s) == 0) && (serialize_mt_unsafe == 1)) {
 404                 if (!rte_spinlock_trylock(&s->execute_lock))
 405                         return -EBUSY;
 406
 407                 service_runner_do_callback(s, cs, i);
 408                 rte_spinlock_unlock(&s->execute_lock);
 409         } else
 410                 service_runner_do_callback(s, cs, i);
 411
 412         return 0;
 413 }
 414
 415 int32_t
 416 rte_service_may_be_active(uint32_t id)
 417 {
 418         uint32_t ids[RTE_MAX_LCORE] = {0};
 419         int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
 420         int i;
 421
 422         if (id >= RTE_SERVICE_NUM_MAX || !service_valid(id))
 423                 return -EINVAL;
 424
 425         for (i = 0; i < lcore_count; i++) {
 426                 if (lcore_states[ids[i]].service_active_on_lcore[id])
 427                         return 1;
 428         }
 429
 430         return 0;
 431 }
 432
 433 int32_t
 434 rte_service_run_iter_on_app_lcore(uint32_t id, uint32_t serialize_mt_unsafe)
 435 {
 436         struct core_state *cs = &lcore_states[rte_lcore_id()];
 437         struct rte_service_spec_impl *s;
 438
 439         SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
 440
 441         /* Increment num_mapped_cores to reflect that this core is
 442          * now mapped capable of running the service.
 443          */
 444         __atomic_add_fetch(&s->num_mapped_cores, 1, __ATOMIC_RELAXED);
 445
 446         int ret = service_run(id, cs, UINT64_MAX, s, serialize_mt_unsafe);
 447
 448         __atomic_sub_fetch(&s->num_mapped_cores, 1, __ATOMIC_RELAXED);
 449
 450         return ret;
 451 }
 452
 453 static int32_t
 454 service_runner_func(void *arg)
 455 {
 456         RTE_SET_USED(arg);
 457         uint32_t i;
 458         const int lcore = rte_lcore_id();
 459         struct core_state *cs = &lcore_states[lcore];
 460
 461         __atomic_store_n(&cs->thread_active, 1, __ATOMIC_SEQ_CST);
 462
 463         /* runstate act as the guard variable. Use load-acquire
 464          * memory order here to synchronize with store-release
 465          * in runstate update functions.
 466          */
 467         while (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) ==
 468                         RUNSTATE_RUNNING) {
 469                 const uint64_t service_mask = cs->service_mask;
 470
 471                 for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
 472                         if (!service_valid(i))
 473                                 continue;
 474                         /* return value ignored as no change to code flow */
 475                         service_run(i, cs, service_mask, service_get(i), 1);
 476                 }
 477
 478                 cs->loops++;
 479         }
 480
 481         /* Use SEQ CST memory ordering to avoid any re-ordering around
 482          * this store, ensuring that once this store is visible, the service
 483          * lcore thread really is done in service cores code.
 484          */
 485         __atomic_store_n(&cs->thread_active, 0, __ATOMIC_SEQ_CST);
 486         return 0;
 487 }
 488
 489 int32_t
 490 rte_service_lcore_may_be_active(uint32_t lcore)
 491 {
 492         if (lcore >= RTE_MAX_LCORE || !lcore_states[lcore].is_service_core)
 493                 return -EINVAL;
 494
 495         /* Load thread_active using ACQUIRE to avoid instructions dependent on
 496          * the result being re-ordered before this load completes.
 497          */
 498         return __atomic_load_n(&lcore_states[lcore].thread_active,
 499                                __ATOMIC_ACQUIRE);
 500 }
 501
 502 int32_t
 503 rte_service_lcore_count(void)
 504 {
 505         int32_t count = 0;
 506         uint32_t i;
 507         for (i = 0; i < RTE_MAX_LCORE; i++)
 508                 count += lcore_states[i].is_service_core;
 509         return count;
 510 }
 511
 512 int32_t
 513 rte_service_lcore_list(uint32_t array[], uint32_t n)
 514 {
 515         uint32_t count = rte_service_lcore_count();
 516         if (count > n)
 517                 return -ENOMEM;
 518
 519         if (!array)
 520                 return -EINVAL;
 521
 522         uint32_t i;
 523         uint32_t idx = 0;
 524         for (i = 0; i < RTE_MAX_LCORE; i++) {
 525                 struct core_state *cs = &lcore_states[i];
 526                 if (cs->is_service_core) {
 527                         array[idx] = i;
 528                         idx++;
 529                 }
 530         }
 531
 532         return count;
 533 }
 534
 535 int32_t
 536 rte_service_lcore_count_services(uint32_t lcore)
 537 {
 538         if (lcore >= RTE_MAX_LCORE)
 539                 return -EINVAL;
 540
 541         struct core_state *cs = &lcore_states[lcore];
 542         if (!cs->is_service_core)
 543                 return -ENOTSUP;
 544
 545         return __builtin_popcountll(cs->service_mask);
 546 }
 547
 548 int32_t
 549 rte_service_start_with_defaults(void)
 550 {
 551         /* create a default mapping from cores to services, then start the
 552          * services to make them transparent to unaware applications.
 553          */
 554         uint32_t i;
 555         int ret;
 556         uint32_t count = rte_service_get_count();
 557
 558         int32_t lcore_iter = 0;
 559         uint32_t ids[RTE_MAX_LCORE] = {0};
 560         int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
 561
 562         if (lcore_count == 0)
 563                 return -ENOTSUP;
 564
 565         for (i = 0; (int)i < lcore_count; i++)
 566                 rte_service_lcore_start(ids[i]);
 567
 568         for (i = 0; i < count; i++) {
 569                 /* do 1:1 core mapping here, with each service getting
 570                  * assigned a single core by default. Adding multiple services
 571                  * should multiplex to a single core, or 1:1 if there are the
 572                  * same amount of services as service-cores
 573                  */
 574                 ret = rte_service_map_lcore_set(i, ids[lcore_iter], 1);
 575                 if (ret)
 576                         return -ENODEV;
 577
 578                 lcore_iter++;
 579                 if (lcore_iter >= lcore_count)
 580                         lcore_iter = 0;
 581
 582                 ret = rte_service_runstate_set(i, 1);
 583                 if (ret)
 584                         return -ENOEXEC;
 585         }
 586
 587         return 0;
 588 }
 589
 590 static int32_t
 591 service_update(uint32_t sid, uint32_t lcore, uint32_t *set, uint32_t *enabled)
 592 {
 593         /* validate ID, or return error value */
 594         if (sid >= RTE_SERVICE_NUM_MAX || !service_valid(sid) ||
 595             lcore >= RTE_MAX_LCORE || !lcore_states[lcore].is_service_core)
 596                 return -EINVAL;
 597
 598         uint64_t sid_mask = UINT64_C(1) << sid;
 599         if (set) {
 600                 uint64_t lcore_mapped = lcore_states[lcore].service_mask &
 601                         sid_mask;
 602
 603                 if (*set && !lcore_mapped) {
 604                         lcore_states[lcore].service_mask |= sid_mask;
 605                         __atomic_add_fetch(&rte_services[sid].num_mapped_cores,
 606                                 1, __ATOMIC_RELAXED);
 607                 }
 608                 if (!*set && lcore_mapped) {
 609                         lcore_states[lcore].service_mask &= ~(sid_mask);
 610                         __atomic_sub_fetch(&rte_services[sid].num_mapped_cores,
 611                                 1, __ATOMIC_RELAXED);
 612                 }
 613         }
 614
 615         if (enabled)
 616                 *enabled = !!(lcore_states[lcore].service_mask & (sid_mask));
 617
 618         return 0;
 619 }
 620
 621 int32_t
 622 rte_service_map_lcore_set(uint32_t id, uint32_t lcore, uint32_t enabled)
 623 {
 624         uint32_t on = enabled > 0;
 625         return service_update(id, lcore, &on, 0);
 626 }
 627
 628 int32_t
 629 rte_service_map_lcore_get(uint32_t id, uint32_t lcore)
 630 {
 631         uint32_t enabled;
 632         int ret = service_update(id, lcore, 0, &enabled);
 633         if (ret == 0)
 634                 return enabled;
 635         return ret;
 636 }
 637
 638 static void
 639 set_lcore_state(uint32_t lcore, int32_t state)
 640 {
 641         /* mark core state in hugepage backed config */
 642         struct rte_config *cfg = rte_eal_get_configuration();
 643         cfg->lcore_role[lcore] = state;
 644
 645         /* mark state in process local lcore_config */
 646         lcore_config[lcore].core_role = state;
 647
 648         /* update per-lcore optimized state tracking */
 649         lcore_states[lcore].is_service_core = (state == ROLE_SERVICE);
 650 }
 651
 652 int32_t
 653 rte_service_lcore_reset_all(void)
 654 {
 655         /* loop over cores, reset all to mask 0 */
 656         uint32_t i;
 657         for (i = 0; i < RTE_MAX_LCORE; i++) {
 658                 if (lcore_states[i].is_service_core) {
 659                         lcore_states[i].service_mask = 0;
 660                         set_lcore_state(i, ROLE_RTE);
 661                         /* runstate act as guard variable Use
 662                          * store-release memory order here to synchronize
 663                          * with load-acquire in runstate read functions.
 664                          */
 665                         __atomic_store_n(&lcore_states[i].runstate,
 666                                 RUNSTATE_STOPPED, __ATOMIC_RELEASE);
 667                 }
 668         }
 669         for (i = 0; i < RTE_SERVICE_NUM_MAX; i++)
 670                 __atomic_store_n(&rte_services[i].num_mapped_cores, 0,
 671                         __ATOMIC_RELAXED);
 672
 673         return 0;
 674 }
 675
 676 int32_t
 677 rte_service_lcore_add(uint32_t lcore)
 678 {
 679         if (lcore >= RTE_MAX_LCORE)
 680                 return -EINVAL;
 681         if (lcore_states[lcore].is_service_core)
 682                 return -EALREADY;
 683
 684         set_lcore_state(lcore, ROLE_SERVICE);
 685
 686         /* ensure that after adding a core the mask and state are defaults */
 687         lcore_states[lcore].service_mask = 0;
 688         /* Use store-release memory order here to synchronize with
 689          * load-acquire in runstate read functions.
 690          */
 691         __atomic_store_n(&lcore_states[lcore].runstate, RUNSTATE_STOPPED,
 692                 __ATOMIC_RELEASE);
 693
 694         return rte_eal_wait_lcore(lcore);
 695 }
 696
 697 int32_t
 698 rte_service_lcore_del(uint32_t lcore)
 699 {
 700         if (lcore >= RTE_MAX_LCORE)
 701                 return -EINVAL;
 702
 703         struct core_state *cs = &lcore_states[lcore];
 704         if (!cs->is_service_core)
 705                 return -EINVAL;
 706
 707         /* runstate act as the guard variable. Use load-acquire
 708          * memory order here to synchronize with store-release
 709          * in runstate update functions.
 710          */
 711         if (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) !=
 712                         RUNSTATE_STOPPED)
 713                 return -EBUSY;
 714
 715         set_lcore_state(lcore, ROLE_RTE);
 716
 717         rte_smp_wmb();
 718         return 0;
 719 }
 720
 721 int32_t
 722 rte_service_lcore_start(uint32_t lcore)
 723 {
 724         if (lcore >= RTE_MAX_LCORE)
 725                 return -EINVAL;
 726
 727         struct core_state *cs = &lcore_states[lcore];
 728         if (!cs->is_service_core)
 729                 return -EINVAL;
 730
 731         /* runstate act as the guard variable. Use load-acquire
 732          * memory order here to synchronize with store-release
 733          * in runstate update functions.
 734          */
 735         if (__atomic_load_n(&cs->runstate, __ATOMIC_ACQUIRE) ==
 736                         RUNSTATE_RUNNING)
 737                 return -EALREADY;
 738
 739         /* set core to run state first, and then launch otherwise it will
 740          * return immediately as runstate keeps it in the service poll loop
 741          */
 742         /* Use load-acquire memory order here to synchronize with
 743          * store-release in runstate update functions.
 744          */
 745         __atomic_store_n(&cs->runstate, RUNSTATE_RUNNING, __ATOMIC_RELEASE);
 746
 747         int ret = rte_eal_remote_launch(service_runner_func, 0, lcore);
 748         /* returns -EBUSY if the core is already launched, 0 on success */
 749         return ret;
 750 }
 751
 752 int32_t
 753 rte_service_lcore_stop(uint32_t lcore)
 754 {
 755         if (lcore >= RTE_MAX_LCORE)
 756                 return -EINVAL;
 757
 758         /* runstate act as the guard variable. Use load-acquire
 759          * memory order here to synchronize with store-release
 760          * in runstate update functions.
 761          */
 762         if (__atomic_load_n(&lcore_states[lcore].runstate, __ATOMIC_ACQUIRE) ==
 763                         RUNSTATE_STOPPED)
 764                 return -EALREADY;
 765
 766         uint32_t i;
 767         uint64_t service_mask = lcore_states[lcore].service_mask;
 768         for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
 769                 int32_t enabled = service_mask & (UINT64_C(1) << i);
 770                 int32_t service_running = rte_service_runstate_get(i);
 771                 int32_t only_core = (1 ==
 772                         __atomic_load_n(&rte_services[i].num_mapped_cores,
 773                                 __ATOMIC_RELAXED));
 774
 775                 /* if the core is mapped, and the service is running, and this
 776                  * is the only core that is mapped, the service would cease to
 777                  * run if this core stopped, so fail instead.
 778                  */
 779                 if (enabled && service_running && only_core)
 780                         return -EBUSY;
 781         }
 782
 783         /* Use store-release memory order here to synchronize with
 784          * load-acquire in runstate read functions.
 785          */
 786         __atomic_store_n(&lcore_states[lcore].runstate, RUNSTATE_STOPPED,
 787                 __ATOMIC_RELEASE);
 788
 789         return 0;
 790 }
 791
 792 int32_t
 793 rte_service_attr_get(uint32_t id, uint32_t attr_id, uint64_t *attr_value)
 794 {
 795         struct rte_service_spec_impl *s;
 796         SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
 797
 798         if (!attr_value)
 799                 return -EINVAL;
 800
 801         switch (attr_id) {
 802         case RTE_SERVICE_ATTR_CYCLES:
 803                 *attr_value = s->cycles_spent;
 804                 return 0;
 805         case RTE_SERVICE_ATTR_CALL_COUNT:
 806                 *attr_value = s->calls;
 807                 return 0;
 808         default:
 809                 return -EINVAL;
 810         }
 811 }
 812
 813 int32_t
 814 rte_service_lcore_attr_get(uint32_t lcore, uint32_t attr_id,
 815                            uint64_t *attr_value)
 816 {
 817         struct core_state *cs;
 818
 819         if (lcore >= RTE_MAX_LCORE || !attr_value)
 820                 return -EINVAL;
 821
 822         cs = &lcore_states[lcore];
 823         if (!cs->is_service_core)
 824                 return -ENOTSUP;
 825
 826         switch (attr_id) {
 827         case RTE_SERVICE_LCORE_ATTR_LOOPS:
 828                 *attr_value = cs->loops;
 829                 return 0;
 830         default:
 831                 return -EINVAL;
 832         }
 833 }
 834
 835 int32_t
 836 rte_service_attr_reset_all(uint32_t id)
 837 {
 838         struct rte_service_spec_impl *s;
 839         SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
 840
 841         s->cycles_spent = 0;
 842         s->calls = 0;
 843         return 0;
 844 }
 845
 846 int32_t
 847 rte_service_lcore_attr_reset_all(uint32_t lcore)
 848 {
 849         struct core_state *cs;
 850
 851         if (lcore >= RTE_MAX_LCORE)
 852                 return -EINVAL;
 853
 854         cs = &lcore_states[lcore];
 855         if (!cs->is_service_core)
 856                 return -ENOTSUP;
 857
 858         cs->loops = 0;
 859
 860         return 0;
 861 }
 862
 863 static void
 864 service_dump_one(FILE *f, struct rte_service_spec_impl *s)
 865 {
 866         /* avoid divide by zero */
 867         int calls = 1;
 868
 869         if (s->calls != 0)
 870                 calls = s->calls;
 871         fprintf(f, "  %s: stats %d\tcalls %"PRIu64"\tcycles %"
 872                         PRIu64"\tavg: %"PRIu64"\n",
 873                         s->spec.name, service_stats_enabled(s), s->calls,
 874                         s->cycles_spent, s->cycles_spent / calls);
 875 }
 876
 877 static void
 878 service_dump_calls_per_lcore(FILE *f, uint32_t lcore)
 879 {
 880         uint32_t i;
 881         struct core_state *cs = &lcore_states[lcore];
 882
 883         fprintf(f, "%02d\t", lcore);
 884         for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
 885                 if (!service_valid(i))
 886                         continue;
 887                 fprintf(f, "%"PRIu64"\t", cs->calls_per_service[i]);
 888         }
 889         fprintf(f, "\n");
 890 }
 891
 892 int32_t
 893 rte_service_dump(FILE *f, uint32_t id)
 894 {
 895         uint32_t i;
 896         int print_one = (id != UINT32_MAX);
 897
 898         /* print only the specified service */
 899         if (print_one) {
 900                 struct rte_service_spec_impl *s;
 901                 SERVICE_VALID_GET_OR_ERR_RET(id, s, -EINVAL);
 902                 fprintf(f, "Service %s Summary\n", s->spec.name);
 903                 service_dump_one(f, s);
 904                 return 0;
 905         }
 906
 907         /* print all services, as UINT32_MAX was passed as id */
 908         fprintf(f, "Services Summary\n");
 909         for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
 910                 if (!service_valid(i))
 911                         continue;
 912                 service_dump_one(f, &rte_services[i]);
 913         }
 914
 915         fprintf(f, "Service Cores Summary\n");
 916         for (i = 0; i < RTE_MAX_LCORE; i++) {
 917                 if (lcore_config[i].core_role != ROLE_SERVICE)
 918                         continue;
 919
 920                 service_dump_calls_per_lcore(f, i);
 921         }
 922
 923         return 0;
 924 }