lib/power/rte_power_empty_poll.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_lcore.h>
   8 #include <rte_cycles.h>
   9 #include <rte_atomic.h>
  10 #include <rte_malloc.h>
  11 #include <inttypes.h>
  12
  13 #include "rte_power.h"
  14 #include "rte_power_empty_poll.h"
  15
  16 #define INTERVALS_PER_SECOND 100     /* (10ms) */
  17 #define SECONDS_TO_TRAIN_FOR 2
  18 #define DEFAULT_MED_TO_HIGH_PERCENT_THRESHOLD 70
  19 #define DEFAULT_HIGH_TO_MED_PERCENT_THRESHOLD 30
  20 #define DEFAULT_CYCLES_PER_PACKET 800
  21
  22 static struct ep_params *ep_params;
  23 static uint32_t med_to_high_threshold = DEFAULT_MED_TO_HIGH_PERCENT_THRESHOLD;
  24 static uint32_t high_to_med_threshold = DEFAULT_HIGH_TO_MED_PERCENT_THRESHOLD;
  25
  26 static uint32_t avail_freqs[RTE_MAX_LCORE][NUM_FREQS];
  27
  28 static uint32_t total_avail_freqs[RTE_MAX_LCORE];
  29
  30 static uint32_t freq_index[NUM_FREQ];
  31
  32 static uint32_t
  33 get_freq_index(enum freq_val index)
  34 {
  35         return freq_index[index];
  36 }
  37
  38
  39 static int
  40 set_power_freq(int lcore_id, enum freq_val freq, bool specific_freq)
  41 {
  42         int err = 0;
  43         uint32_t power_freq_index;
  44         if (!specific_freq)
  45                 power_freq_index = get_freq_index(freq);
  46         else
  47                 power_freq_index = freq;
  48
  49         err = rte_power_set_freq(lcore_id, power_freq_index);
  50
  51         return err;
  52 }
  53
  54
  55 static __rte_always_inline void
  56 exit_training_state(struct priority_worker *poll_stats)
  57 {
  58         RTE_SET_USED(poll_stats);
  59 }
  60
  61 static __rte_always_inline void
  62 enter_training_state(struct priority_worker *poll_stats)
  63 {
  64         poll_stats->iter_counter = 0;
  65         poll_stats->cur_freq = LOW;
  66         poll_stats->queue_state = TRAINING;
  67 }
  68
  69 static __rte_always_inline void
  70 enter_normal_state(struct priority_worker *poll_stats)
  71 {
  72         /* Clear the averages arrays and strs */
  73         memset(poll_stats->edpi_av, 0, sizeof(poll_stats->edpi_av));
  74         poll_stats->ec = 0;
  75
  76         poll_stats->cur_freq = MED;
  77         poll_stats->iter_counter = 0;
  78         poll_stats->threshold_ctr = 0;
  79         poll_stats->queue_state = MED_NORMAL;
  80         RTE_LOG(INFO, POWER, "Set the power freq to MED\n");
  81         set_power_freq(poll_stats->lcore_id, MED, false);
  82
  83         poll_stats->thresh[MED].threshold_percent = med_to_high_threshold;
  84         poll_stats->thresh[HGH].threshold_percent = high_to_med_threshold;
  85 }
  86
  87 static __rte_always_inline void
  88 enter_busy_state(struct priority_worker *poll_stats)
  89 {
  90         memset(poll_stats->edpi_av, 0, sizeof(poll_stats->edpi_av));
  91         poll_stats->ec = 0;
  92
  93         poll_stats->cur_freq = HGH;
  94         poll_stats->iter_counter = 0;
  95         poll_stats->threshold_ctr = 0;
  96         poll_stats->queue_state = HGH_BUSY;
  97         set_power_freq(poll_stats->lcore_id, HGH, false);
  98 }
  99
 100 static __rte_always_inline void
 101 enter_purge_state(struct priority_worker *poll_stats)
 102 {
 103         poll_stats->iter_counter = 0;
 104         poll_stats->queue_state = LOW_PURGE;
 105 }
 106
 107 static __rte_always_inline void
 108 set_state(struct priority_worker *poll_stats,
 109                 enum queue_state new_state)
 110 {
 111         enum queue_state old_state = poll_stats->queue_state;
 112         if (old_state != new_state) {
 113
 114                 /* Call any old state exit functions */
 115                 if (old_state == TRAINING)
 116                         exit_training_state(poll_stats);
 117
 118                 /* Call any new state entry functions */
 119                 if (new_state == TRAINING)
 120                         enter_training_state(poll_stats);
 121                 if (new_state == MED_NORMAL)
 122                         enter_normal_state(poll_stats);
 123                 if (new_state == HGH_BUSY)
 124                         enter_busy_state(poll_stats);
 125                 if (new_state == LOW_PURGE)
 126                         enter_purge_state(poll_stats);
 127         }
 128 }
 129
 130 static __rte_always_inline void
 131 set_policy(struct priority_worker *poll_stats,
 132                 struct ep_policy *policy)
 133 {
 134         set_state(poll_stats, policy->state);
 135
 136         if (policy->state == TRAINING)
 137                 return;
 138
 139         poll_stats->thresh[MED_NORMAL].base_edpi = policy->med_base_edpi;
 140         poll_stats->thresh[HGH_BUSY].base_edpi = policy->hgh_base_edpi;
 141
 142         poll_stats->thresh[MED_NORMAL].trained = true;
 143         poll_stats->thresh[HGH_BUSY].trained = true;
 144
 145 }
 146
 147 static void
 148 update_training_stats(struct priority_worker *poll_stats,
 149                 uint32_t freq,
 150                 bool specific_freq,
 151                 uint32_t max_train_iter)
 152 {
 153         RTE_SET_USED(specific_freq);
 154
 155         uint64_t p0_empty_deq;
 156
 157         if (poll_stats->cur_freq == freq &&
 158                         poll_stats->thresh[freq].trained == false) {
 159                 if (poll_stats->thresh[freq].cur_train_iter == 0) {
 160
 161                         set_power_freq(poll_stats->lcore_id,
 162                                         freq, specific_freq);
 163
 164                         poll_stats->empty_dequeues_prev =
 165                                 poll_stats->empty_dequeues;
 166
 167                         poll_stats->thresh[freq].cur_train_iter++;
 168
 169                         return;
 170                 } else if (poll_stats->thresh[freq].cur_train_iter
 171                                 <= max_train_iter) {
 172
 173                         p0_empty_deq = poll_stats->empty_dequeues -
 174                                 poll_stats->empty_dequeues_prev;
 175
 176                         poll_stats->empty_dequeues_prev =
 177                                 poll_stats->empty_dequeues;
 178
 179                         poll_stats->thresh[freq].base_edpi += p0_empty_deq;
 180                         poll_stats->thresh[freq].cur_train_iter++;
 181
 182                 } else {
 183                         if (poll_stats->thresh[freq].trained == false) {
 184                                 poll_stats->thresh[freq].base_edpi =
 185                                         poll_stats->thresh[freq].base_edpi /
 186                                         max_train_iter;
 187
 188                                 /* Add on a factor of 0.05%
 189                                  * this should remove any
 190                                  * false negatives when the system is 0% busy
 191                                  */
 192                                 poll_stats->thresh[freq].base_edpi +=
 193                                 poll_stats->thresh[freq].base_edpi / 2000;
 194
 195                                 poll_stats->thresh[freq].trained = true;
 196                                 poll_stats->cur_freq++;
 197
 198                         }
 199                 }
 200         }
 201 }
 202
 203 static __rte_always_inline uint32_t
 204 update_stats(struct priority_worker *poll_stats)
 205 {
 206         uint64_t tot_edpi = 0;
 207         uint32_t j, percent;
 208
 209         struct priority_worker *s = poll_stats;
 210
 211         uint64_t cur_edpi = s->empty_dequeues - s->empty_dequeues_prev;
 212
 213         s->empty_dequeues_prev = s->empty_dequeues;
 214
 215         if (s->thresh[s->cur_freq].base_edpi < cur_edpi) {
 216
 217                 /* edpi mean empty poll counter difference per interval */
 218                 RTE_LOG(DEBUG, POWER, "cur_edpi is too large "
 219                                 "cur edpi %"PRId64" "
 220                                 "base edpi %"PRId64"\n",
 221                                 cur_edpi,
 222                                 s->thresh[s->cur_freq].base_edpi);
 223                 /* Value to make us fail need debug log*/
 224                 return 1000UL;
 225         }
 226
 227         s->edpi_av[s->ec++ % BINS_AV] = cur_edpi;
 228
 229         for (j = 0; j < BINS_AV; j++) {
 230                 tot_edpi += s->edpi_av[j];
 231         }
 232
 233         tot_edpi = tot_edpi / BINS_AV;
 234
 235         percent = 100 - (uint32_t)(((float)tot_edpi /
 236                         (float)s->thresh[s->cur_freq].base_edpi) * 100);
 237
 238         return (uint32_t)percent;
 239 }
 240
 241
 242 static __rte_always_inline void
 243 update_stats_normal(struct priority_worker *poll_stats)
 244 {
 245         uint32_t percent;
 246
 247         if (poll_stats->thresh[poll_stats->cur_freq].base_edpi == 0) {
 248
 249                 enum freq_val cur_freq = poll_stats->cur_freq;
 250
 251                 /* edpi mean empty poll counter difference per interval */
 252                 RTE_LOG(DEBUG, POWER, "cure freq is %d, edpi is %"PRIu64"\n",
 253                                 cur_freq,
 254                                 poll_stats->thresh[cur_freq].base_edpi);
 255                 return;
 256         }
 257
 258         percent = update_stats(poll_stats);
 259
 260         if (percent > 100) {
 261                 /* edpi mean empty poll counter difference per interval */
 262                 RTE_LOG(DEBUG, POWER, "Edpi is bigger than threshold\n");
 263                 return;
 264         }
 265
 266         if (poll_stats->cur_freq == LOW)
 267                 RTE_LOG(INFO, POWER, "Purge Mode is not currently supported\n");
 268         else if (poll_stats->cur_freq == MED) {
 269
 270                 if (percent >
 271                         poll_stats->thresh[MED].threshold_percent) {
 272
 273                         if (poll_stats->threshold_ctr < INTERVALS_PER_SECOND)
 274                                 poll_stats->threshold_ctr++;
 275                         else {
 276                                 set_state(poll_stats, HGH_BUSY);
 277                                 RTE_LOG(INFO, POWER, "MOVE to HGH\n");
 278                         }
 279
 280                 } else {
 281                         /* reset */
 282                         poll_stats->threshold_ctr = 0;
 283                 }
 284
 285         } else if (poll_stats->cur_freq == HGH) {
 286
 287                 if (percent <
 288                                 poll_stats->thresh[HGH].threshold_percent) {
 289
 290                         if (poll_stats->threshold_ctr < INTERVALS_PER_SECOND)
 291                                 poll_stats->threshold_ctr++;
 292                         else {
 293                                 set_state(poll_stats, MED_NORMAL);
 294                                 RTE_LOG(INFO, POWER, "MOVE to MED\n");
 295                         }
 296                 } else {
 297                         /* reset */
 298                         poll_stats->threshold_ctr = 0;
 299                 }
 300
 301         }
 302 }
 303
 304 static int
 305 empty_poll_training(struct priority_worker *poll_stats,
 306                 uint32_t max_train_iter)
 307 {
 308
 309         if (poll_stats->iter_counter < INTERVALS_PER_SECOND) {
 310                 poll_stats->iter_counter++;
 311                 return 0;
 312         }
 313
 314
 315         update_training_stats(poll_stats,
 316                         LOW,
 317                         false,
 318                         max_train_iter);
 319
 320         update_training_stats(poll_stats,
 321                         MED,
 322                         false,
 323                         max_train_iter);
 324
 325         update_training_stats(poll_stats,
 326                         HGH,
 327                         false,
 328                         max_train_iter);
 329
 330
 331         if (poll_stats->thresh[LOW].trained == true
 332                         && poll_stats->thresh[MED].trained == true
 333                         && poll_stats->thresh[HGH].trained == true) {
 334
 335                 set_state(poll_stats, MED_NORMAL);
 336
 337                 RTE_LOG(INFO, POWER, "LOW threshold is %"PRIu64"\n",
 338                                 poll_stats->thresh[LOW].base_edpi);
 339
 340                 RTE_LOG(INFO, POWER, "MED threshold is %"PRIu64"\n",
 341                                 poll_stats->thresh[MED].base_edpi);
 342
 343
 344                 RTE_LOG(INFO, POWER, "HIGH threshold is %"PRIu64"\n",
 345                                 poll_stats->thresh[HGH].base_edpi);
 346
 347                 RTE_LOG(INFO, POWER, "Training is Complete for %d\n",
 348                                 poll_stats->lcore_id);
 349         }
 350
 351         return 0;
 352 }
 353
 354 void
 355 rte_empty_poll_detection(struct rte_timer *tim, void *arg)
 356 {
 357
 358         uint32_t i;
 359
 360         struct priority_worker *poll_stats;
 361
 362         RTE_SET_USED(tim);
 363
 364         RTE_SET_USED(arg);
 365
 366         for (i = 0; i < NUM_NODES; i++) {
 367
 368                 poll_stats = &(ep_params->wrk_data.wrk_stats[i]);
 369
 370                 if (rte_lcore_is_enabled(poll_stats->lcore_id) == 0)
 371                         continue;
 372
 373                 switch (poll_stats->queue_state) {
 374                 case(TRAINING):
 375                         empty_poll_training(poll_stats,
 376                                         ep_params->max_train_iter);
 377                         break;
 378
 379                 case(HGH_BUSY):
 380                 case(MED_NORMAL):
 381                         update_stats_normal(poll_stats);
 382                         break;
 383
 384                 case(LOW_PURGE):
 385                         break;
 386                 default:
 387                         break;
 388
 389                 }
 390
 391         }
 392
 393 }
 394
 395 int
 396 rte_power_empty_poll_stat_init(struct ep_params **eptr, uint8_t *freq_tlb,
 397                 struct ep_policy *policy)
 398 {
 399         uint32_t i;
 400         /* Allocate the ep_params structure */
 401         ep_params = rte_zmalloc_socket(NULL,
 402                         sizeof(struct ep_params),
 403                         0,
 404                         rte_socket_id());
 405
 406         if (!ep_params)
 407                 return -1;
 408
 409         if (freq_tlb == NULL) {
 410                 freq_index[LOW] = 14;
 411                 freq_index[MED] = 9;
 412                 freq_index[HGH] = 1;
 413         } else {
 414                 freq_index[LOW] = freq_tlb[LOW];
 415                 freq_index[MED] = freq_tlb[MED];
 416                 freq_index[HGH] = freq_tlb[HGH];
 417         }
 418
 419         RTE_LOG(INFO, POWER, "Initialize the Empty Poll\n");
 420
 421         /* Train for pre-defined period */
 422         ep_params->max_train_iter = INTERVALS_PER_SECOND * SECONDS_TO_TRAIN_FOR;
 423
 424         struct stats_data *w = &ep_params->wrk_data;
 425
 426         *eptr = ep_params;
 427
 428         /* initialize all wrk_stats state */
 429         for (i = 0; i < NUM_NODES; i++) {
 430
 431                 if (rte_lcore_is_enabled(i) == 0)
 432                         continue;
 433                 /*init the freqs table */
 434                 total_avail_freqs[i] = rte_power_freqs(i,
 435                                 avail_freqs[i],
 436                                 NUM_FREQS);
 437
 438                 RTE_LOG(INFO, POWER, "total avail freq is %d , lcoreid %d\n",
 439                                 total_avail_freqs[i],
 440                                 i);
 441
 442                 if (get_freq_index(LOW) > total_avail_freqs[i])
 443                         return -1;
 444
 445                 if (rte_get_main_lcore() != i) {
 446                         w->wrk_stats[i].lcore_id = i;
 447                         set_policy(&w->wrk_stats[i], policy);
 448                 }
 449         }
 450
 451         return 0;
 452 }
 453
 454 void
 455 rte_power_empty_poll_stat_free(void)
 456 {
 457
 458         RTE_LOG(INFO, POWER, "Close the Empty Poll\n");
 459
 460         if (ep_params != NULL)
 461                 rte_free(ep_params);
 462 }
 463
 464 int
 465 rte_power_empty_poll_stat_update(unsigned int lcore_id)
 466 {
 467         struct priority_worker *poll_stats;
 468
 469         if (lcore_id >= NUM_NODES)
 470                 return -1;
 471
 472         poll_stats = &(ep_params->wrk_data.wrk_stats[lcore_id]);
 473
 474         if (poll_stats->lcore_id == 0)
 475                 poll_stats->lcore_id = lcore_id;
 476
 477         poll_stats->empty_dequeues++;
 478
 479         return 0;
 480 }
 481
 482 int
 483 rte_power_poll_stat_update(unsigned int lcore_id, uint8_t nb_pkt)
 484 {
 485
 486         struct priority_worker *poll_stats;
 487
 488         if (lcore_id >= NUM_NODES)
 489                 return -1;
 490
 491         poll_stats = &(ep_params->wrk_data.wrk_stats[lcore_id]);
 492
 493         if (poll_stats->lcore_id == 0)
 494                 poll_stats->lcore_id = lcore_id;
 495
 496         poll_stats->num_dequeue_pkts += nb_pkt;
 497
 498         return 0;
 499 }
 500
 501
 502 uint64_t
 503 rte_power_empty_poll_stat_fetch(unsigned int lcore_id)
 504 {
 505         struct priority_worker *poll_stats;
 506
 507         if (lcore_id >= NUM_NODES)
 508                 return -1;
 509
 510         poll_stats = &(ep_params->wrk_data.wrk_stats[lcore_id]);
 511
 512         if (poll_stats->lcore_id == 0)
 513                 poll_stats->lcore_id = lcore_id;
 514
 515         return poll_stats->empty_dequeues;
 516 }
 517
 518 uint64_t
 519 rte_power_poll_stat_fetch(unsigned int lcore_id)
 520 {
 521         struct priority_worker *poll_stats;
 522
 523         if (lcore_id >= NUM_NODES)
 524                 return -1;
 525
 526         poll_stats = &(ep_params->wrk_data.wrk_stats[lcore_id]);
 527
 528         if (poll_stats->lcore_id == 0)
 529                 poll_stats->lcore_id = lcore_id;
 530
 531         return poll_stats->num_dequeue_pkts;
 532 }