lib/sched/rte_sched.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <string.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_log.h>
  10 #include <rte_memory.h>
  11 #include <rte_malloc.h>
  12 #include <rte_cycles.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_branch_prediction.h>
  15 #include <rte_mbuf.h>
  16 #include <rte_bitmap.h>
  17 #include <rte_reciprocal.h>
  18
  19 #include "rte_sched.h"
  20 #include "rte_sched_common.h"
  21 #include "rte_approx.h"
  22
  23 #ifdef __INTEL_COMPILER
  24 #pragma warning(disable:2259) /* conversion may lose significant bits */
  25 #endif
  26
  27 #ifdef RTE_SCHED_VECTOR
  28 #include <rte_vect.h>
  29
  30 #ifdef RTE_ARCH_X86
  31 #define SCHED_VECTOR_SSE4
  32 #elif defined(__ARM_NEON)
  33 #define SCHED_VECTOR_NEON
  34 #endif
  35
  36 #endif
  37
  38 #define RTE_SCHED_TB_RATE_CONFIG_ERR          (1e-7)
  39 #define RTE_SCHED_WRR_SHIFT                   3
  40 #define RTE_SCHED_MAX_QUEUES_PER_TC           RTE_SCHED_BE_QUEUES_PER_PIPE
  41 #define RTE_SCHED_GRINDER_PCACHE_SIZE         (64 / RTE_SCHED_QUEUES_PER_PIPE)
  42 #define RTE_SCHED_PIPE_INVALID                UINT32_MAX
  43 #define RTE_SCHED_BMP_POS_INVALID             UINT32_MAX
  44
  45 /* Scaling for cycles_per_byte calculation
  46  * Chosen so that minimum rate is 480 bit/sec
  47  */
  48 #define RTE_SCHED_TIME_SHIFT                  8
  49
  50 struct rte_sched_pipe_profile {
  51         /* Token bucket (TB) */
  52         uint64_t tb_period;
  53         uint64_t tb_credits_per_period;
  54         uint64_t tb_size;
  55
  56         /* Pipe traffic classes */
  57         uint64_t tc_period;
  58         uint64_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
  59         uint8_t tc_ov_weight;
  60
  61         /* Pipe best-effort traffic class queues */
  62         uint8_t  wrr_cost[RTE_SCHED_BE_QUEUES_PER_PIPE];
  63 };
  64
  65 struct rte_sched_pipe {
  66         /* Token bucket (TB) */
  67         uint64_t tb_time; /* time of last update */
  68         uint64_t tb_credits;
  69
  70         /* Pipe profile and flags */
  71         uint32_t profile;
  72
  73         /* Traffic classes (TCs) */
  74         uint64_t tc_time; /* time of next update */
  75         uint64_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
  76
  77         /* Weighted Round Robin (WRR) */
  78         uint8_t wrr_tokens[RTE_SCHED_BE_QUEUES_PER_PIPE];
  79
  80         /* TC oversubscription */
  81         uint64_t tc_ov_credits;
  82         uint8_t tc_ov_period_id;
  83 } __rte_cache_aligned;
  84
  85 struct rte_sched_queue {
  86         uint16_t qw;
  87         uint16_t qr;
  88 };
  89
  90 struct rte_sched_queue_extra {
  91         struct rte_sched_queue_stats stats;
  92 #ifdef RTE_SCHED_CMAN
  93         RTE_STD_C11
  94         union {
  95                 struct rte_red red;
  96                 struct rte_pie pie;
  97         };
  98 #endif
  99 };
 100
 101 enum grinder_state {
 102         e_GRINDER_PREFETCH_PIPE = 0,
 103         e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS,
 104         e_GRINDER_PREFETCH_MBUF,
 105         e_GRINDER_READ_MBUF
 106 };
 107
 108 struct rte_sched_subport_profile {
 109         /* Token bucket (TB) */
 110         uint64_t tb_period;
 111         uint64_t tb_credits_per_period;
 112         uint64_t tb_size;
 113
 114         uint64_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 115         uint64_t tc_period;
 116 };
 117
 118 struct rte_sched_grinder {
 119         /* Pipe cache */
 120         uint16_t pcache_qmask[RTE_SCHED_GRINDER_PCACHE_SIZE];
 121         uint32_t pcache_qindex[RTE_SCHED_GRINDER_PCACHE_SIZE];
 122         uint32_t pcache_w;
 123         uint32_t pcache_r;
 124
 125         /* Current pipe */
 126         enum grinder_state state;
 127         uint32_t productive;
 128         uint32_t pindex;
 129         struct rte_sched_subport *subport;
 130         struct rte_sched_subport_profile *subport_params;
 131         struct rte_sched_pipe *pipe;
 132         struct rte_sched_pipe_profile *pipe_params;
 133
 134         /* TC cache */
 135         uint8_t tccache_qmask[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 136         uint32_t tccache_qindex[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 137         uint32_t tccache_w;
 138         uint32_t tccache_r;
 139
 140         /* Current TC */
 141         uint32_t tc_index;
 142         struct rte_sched_queue *queue[RTE_SCHED_MAX_QUEUES_PER_TC];
 143         struct rte_mbuf **qbase[RTE_SCHED_MAX_QUEUES_PER_TC];
 144         uint32_t qindex[RTE_SCHED_MAX_QUEUES_PER_TC];
 145         uint16_t qsize;
 146         uint32_t qmask;
 147         uint32_t qpos;
 148         struct rte_mbuf *pkt;
 149
 150         /* WRR */
 151         uint16_t wrr_tokens[RTE_SCHED_BE_QUEUES_PER_PIPE];
 152         uint16_t wrr_mask[RTE_SCHED_BE_QUEUES_PER_PIPE];
 153         uint8_t wrr_cost[RTE_SCHED_BE_QUEUES_PER_PIPE];
 154 };
 155
 156 struct rte_sched_subport {
 157         /* Token bucket (TB) */
 158         uint64_t tb_time; /* time of last update */
 159         uint64_t tb_credits;
 160
 161         /* Traffic classes (TCs) */
 162         uint64_t tc_time; /* time of next update */
 163         uint64_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 164
 165         /* TC oversubscription */
 166         uint64_t tc_ov_wm;
 167         uint64_t tc_ov_wm_min;
 168         uint64_t tc_ov_wm_max;
 169         uint8_t tc_ov_period_id;
 170         uint8_t tc_ov;
 171         uint32_t tc_ov_n;
 172         double tc_ov_rate;
 173
 174         /* Statistics */
 175         struct rte_sched_subport_stats stats __rte_cache_aligned;
 176
 177         /* subport profile */
 178         uint32_t profile;
 179         /* Subport pipes */
 180         uint32_t n_pipes_per_subport_enabled;
 181         uint32_t n_pipe_profiles;
 182         uint32_t n_max_pipe_profiles;
 183
 184         /* Pipe best-effort TC rate */
 185         uint64_t pipe_tc_be_rate_max;
 186
 187         /* Pipe queues size */
 188         uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 189
 190 #ifdef RTE_SCHED_CMAN
 191         bool cman_enabled;
 192         enum rte_sched_cman_mode cman;
 193
 194         RTE_STD_C11
 195         union {
 196                 struct rte_red_config red_config[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][RTE_COLORS];
 197                 struct rte_pie_config pie_config[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 198         };
 199 #endif
 200
 201         /* Scheduling loop detection */
 202         uint32_t pipe_loop;
 203         uint32_t pipe_exhaustion;
 204
 205         /* Bitmap */
 206         struct rte_bitmap *bmp;
 207         uint32_t grinder_base_bmp_pos[RTE_SCHED_PORT_N_GRINDERS] __rte_aligned_16;
 208
 209         /* Grinders */
 210         struct rte_sched_grinder grinder[RTE_SCHED_PORT_N_GRINDERS];
 211         uint32_t busy_grinders;
 212
 213         /* Queue base calculation */
 214         uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE];
 215         uint32_t qsize_sum;
 216
 217         struct rte_sched_pipe *pipe;
 218         struct rte_sched_queue *queue;
 219         struct rte_sched_queue_extra *queue_extra;
 220         struct rte_sched_pipe_profile *pipe_profiles;
 221         uint8_t *bmp_array;
 222         struct rte_mbuf **queue_array;
 223         uint8_t memory[0] __rte_cache_aligned;
 224 } __rte_cache_aligned;
 225
 226 struct rte_sched_port {
 227         /* User parameters */
 228         uint32_t n_subports_per_port;
 229         uint32_t n_pipes_per_subport;
 230         uint32_t n_pipes_per_subport_log2;
 231         uint16_t pipe_queue[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 232         uint8_t pipe_tc[RTE_SCHED_QUEUES_PER_PIPE];
 233         uint8_t tc_queue[RTE_SCHED_QUEUES_PER_PIPE];
 234         uint32_t n_subport_profiles;
 235         uint32_t n_max_subport_profiles;
 236         uint64_t rate;
 237         uint32_t mtu;
 238         uint32_t frame_overhead;
 239         int socket;
 240
 241         /* Timing */
 242         uint64_t time_cpu_cycles;     /* Current CPU time measured in CPU cyles */
 243         uint64_t time_cpu_bytes;      /* Current CPU time measured in bytes */
 244         uint64_t time;                /* Current NIC TX time measured in bytes */
 245         struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */
 246         uint64_t cycles_per_byte;
 247
 248         /* Grinders */
 249         struct rte_mbuf **pkts_out;
 250         uint32_t n_pkts_out;
 251         uint32_t subport_id;
 252
 253         /* Large data structures */
 254         struct rte_sched_subport_profile *subport_profiles;
 255         struct rte_sched_subport *subports[0] __rte_cache_aligned;
 256 } __rte_cache_aligned;
 257
 258 enum rte_sched_subport_array {
 259         e_RTE_SCHED_SUBPORT_ARRAY_PIPE = 0,
 260         e_RTE_SCHED_SUBPORT_ARRAY_QUEUE,
 261         e_RTE_SCHED_SUBPORT_ARRAY_QUEUE_EXTRA,
 262         e_RTE_SCHED_SUBPORT_ARRAY_PIPE_PROFILES,
 263         e_RTE_SCHED_SUBPORT_ARRAY_BMP_ARRAY,
 264         e_RTE_SCHED_SUBPORT_ARRAY_QUEUE_ARRAY,
 265         e_RTE_SCHED_SUBPORT_ARRAY_TOTAL,
 266 };
 267
 268 static inline uint32_t
 269 rte_sched_subport_pipe_queues(struct rte_sched_subport *subport)
 270 {
 271         return RTE_SCHED_QUEUES_PER_PIPE * subport->n_pipes_per_subport_enabled;
 272 }
 273
 274 static inline struct rte_mbuf **
 275 rte_sched_subport_pipe_qbase(struct rte_sched_subport *subport, uint32_t qindex)
 276 {
 277         uint32_t pindex = qindex >> 4;
 278         uint32_t qpos = qindex & (RTE_SCHED_QUEUES_PER_PIPE - 1);
 279
 280         return (subport->queue_array + pindex *
 281                 subport->qsize_sum + subport->qsize_add[qpos]);
 282 }
 283
 284 static inline uint16_t
 285 rte_sched_subport_pipe_qsize(struct rte_sched_port *port,
 286 struct rte_sched_subport *subport, uint32_t qindex)
 287 {
 288         uint32_t tc = port->pipe_tc[qindex & (RTE_SCHED_QUEUES_PER_PIPE - 1)];
 289
 290         return subport->qsize[tc];
 291 }
 292
 293 static inline uint32_t
 294 rte_sched_port_queues_per_port(struct rte_sched_port *port)
 295 {
 296         uint32_t n_queues = 0, i;
 297
 298         for (i = 0; i < port->n_subports_per_port; i++)
 299                 n_queues += rte_sched_subport_pipe_queues(port->subports[i]);
 300
 301         return n_queues;
 302 }
 303
 304 static inline uint16_t
 305 rte_sched_port_pipe_queue(struct rte_sched_port *port, uint32_t traffic_class)
 306 {
 307         uint16_t pipe_queue = port->pipe_queue[traffic_class];
 308
 309         return pipe_queue;
 310 }
 311
 312 static inline uint8_t
 313 rte_sched_port_pipe_tc(struct rte_sched_port *port, uint32_t qindex)
 314 {
 315         uint8_t pipe_tc = port->pipe_tc[qindex & (RTE_SCHED_QUEUES_PER_PIPE - 1)];
 316
 317         return pipe_tc;
 318 }
 319
 320 static inline uint8_t
 321 rte_sched_port_tc_queue(struct rte_sched_port *port, uint32_t qindex)
 322 {
 323         uint8_t tc_queue = port->tc_queue[qindex & (RTE_SCHED_QUEUES_PER_PIPE - 1)];
 324
 325         return tc_queue;
 326 }
 327
 328 static int
 329 pipe_profile_check(struct rte_sched_pipe_params *params,
 330         uint64_t rate, uint16_t *qsize)
 331 {
 332         uint32_t i;
 333
 334         /* Pipe parameters */
 335         if (params == NULL) {
 336                 RTE_LOG(ERR, SCHED,
 337                         "%s: Incorrect value for parameter params\n", __func__);
 338                 return -EINVAL;
 339         }
 340
 341         /* TB rate: non-zero, not greater than port rate */
 342         if (params->tb_rate == 0 ||
 343                 params->tb_rate > rate) {
 344                 RTE_LOG(ERR, SCHED,
 345                         "%s: Incorrect value for tb rate\n", __func__);
 346                 return -EINVAL;
 347         }
 348
 349         /* TB size: non-zero */
 350         if (params->tb_size == 0) {
 351                 RTE_LOG(ERR, SCHED,
 352                         "%s: Incorrect value for tb size\n", __func__);
 353                 return -EINVAL;
 354         }
 355
 356         /* TC rate: non-zero if qsize non-zero, less than pipe rate */
 357         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 358                 if ((qsize[i] == 0 && params->tc_rate[i] != 0) ||
 359                         (qsize[i] != 0 && (params->tc_rate[i] == 0 ||
 360                         params->tc_rate[i] > params->tb_rate))) {
 361                         RTE_LOG(ERR, SCHED,
 362                                 "%s: Incorrect value for qsize or tc_rate\n", __func__);
 363                         return -EINVAL;
 364                 }
 365         }
 366
 367         if (params->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE] == 0 ||
 368                 qsize[RTE_SCHED_TRAFFIC_CLASS_BE] == 0) {
 369                 RTE_LOG(ERR, SCHED,
 370                         "%s: Incorrect value for be traffic class rate\n", __func__);
 371                 return -EINVAL;
 372         }
 373
 374         /* TC period: non-zero */
 375         if (params->tc_period == 0) {
 376                 RTE_LOG(ERR, SCHED,
 377                         "%s: Incorrect value for tc period\n", __func__);
 378                 return -EINVAL;
 379         }
 380
 381         /*  Best effort tc oversubscription weight: non-zero */
 382         if (params->tc_ov_weight == 0) {
 383                 RTE_LOG(ERR, SCHED,
 384                         "%s: Incorrect value for tc ov weight\n", __func__);
 385                 return -EINVAL;
 386         }
 387
 388         /* Queue WRR weights: non-zero */
 389         for (i = 0; i < RTE_SCHED_BE_QUEUES_PER_PIPE; i++) {
 390                 if (params->wrr_weights[i] == 0) {
 391                         RTE_LOG(ERR, SCHED,
 392                                 "%s: Incorrect value for wrr weight\n", __func__);
 393                         return -EINVAL;
 394                 }
 395         }
 396
 397         return 0;
 398 }
 399
 400 static int
 401 subport_profile_check(struct rte_sched_subport_profile_params *params,
 402         uint64_t rate)
 403 {
 404         uint32_t i;
 405
 406         /* Check user parameters */
 407         if (params == NULL) {
 408                 RTE_LOG(ERR, SCHED, "%s: "
 409                 "Incorrect value for parameter params\n", __func__);
 410                 return -EINVAL;
 411         }
 412
 413         if (params->tb_rate == 0 || params->tb_rate > rate) {
 414                 RTE_LOG(ERR, SCHED, "%s: "
 415                 "Incorrect value for tb rate\n", __func__);
 416                 return -EINVAL;
 417         }
 418
 419         if (params->tb_size == 0) {
 420                 RTE_LOG(ERR, SCHED, "%s: "
 421                 "Incorrect value for tb size\n", __func__);
 422                 return -EINVAL;
 423         }
 424
 425         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 426                 uint64_t tc_rate = params->tc_rate[i];
 427
 428                 if (tc_rate == 0 || (tc_rate > params->tb_rate)) {
 429                         RTE_LOG(ERR, SCHED, "%s: "
 430                         "Incorrect value for tc rate\n", __func__);
 431                         return -EINVAL;
 432                 }
 433         }
 434
 435         if (params->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE] == 0) {
 436                 RTE_LOG(ERR, SCHED, "%s: "
 437                 "Incorrect tc rate(best effort)\n", __func__);
 438                 return -EINVAL;
 439         }
 440
 441         if (params->tc_period == 0) {
 442                 RTE_LOG(ERR, SCHED, "%s: "
 443                 "Incorrect value for tc period\n", __func__);
 444                 return -EINVAL;
 445         }
 446
 447         return 0;
 448 }
 449
 450 static int
 451 rte_sched_port_check_params(struct rte_sched_port_params *params)
 452 {
 453         uint32_t i;
 454
 455         if (params == NULL) {
 456                 RTE_LOG(ERR, SCHED,
 457                         "%s: Incorrect value for parameter params\n", __func__);
 458                 return -EINVAL;
 459         }
 460
 461         /* socket */
 462         if (params->socket < 0) {
 463                 RTE_LOG(ERR, SCHED,
 464                         "%s: Incorrect value for socket id\n", __func__);
 465                 return -EINVAL;
 466         }
 467
 468         /* rate */
 469         if (params->rate == 0) {
 470                 RTE_LOG(ERR, SCHED,
 471                         "%s: Incorrect value for rate\n", __func__);
 472                 return -EINVAL;
 473         }
 474
 475         /* mtu */
 476         if (params->mtu == 0) {
 477                 RTE_LOG(ERR, SCHED,
 478                         "%s: Incorrect value for mtu\n", __func__);
 479                 return -EINVAL;
 480         }
 481
 482         /* n_subports_per_port: non-zero, limited to 16 bits, power of 2 */
 483         if (params->n_subports_per_port == 0 ||
 484             params->n_subports_per_port > 1u << 16 ||
 485             !rte_is_power_of_2(params->n_subports_per_port)) {
 486                 RTE_LOG(ERR, SCHED,
 487                         "%s: Incorrect value for number of subports\n", __func__);
 488                 return -EINVAL;
 489         }
 490
 491         if (params->subport_profiles == NULL ||
 492                 params->n_subport_profiles == 0 ||
 493                 params->n_max_subport_profiles == 0 ||
 494                 params->n_subport_profiles > params->n_max_subport_profiles) {
 495                 RTE_LOG(ERR, SCHED,
 496                 "%s: Incorrect value for subport profiles\n", __func__);
 497                 return -EINVAL;
 498         }
 499
 500         for (i = 0; i < params->n_subport_profiles; i++) {
 501                 struct rte_sched_subport_profile_params *p =
 502                                                 params->subport_profiles + i;
 503                 int status;
 504
 505                 status = subport_profile_check(p, params->rate);
 506                 if (status != 0) {
 507                         RTE_LOG(ERR, SCHED,
 508                         "%s: subport profile check failed(%d)\n",
 509                         __func__, status);
 510                         return -EINVAL;
 511                 }
 512         }
 513
 514         /* n_pipes_per_subport: non-zero, power of 2 */
 515         if (params->n_pipes_per_subport == 0 ||
 516             !rte_is_power_of_2(params->n_pipes_per_subport)) {
 517                 RTE_LOG(ERR, SCHED,
 518                         "%s: Incorrect value for maximum pipes number\n", __func__);
 519                 return -EINVAL;
 520         }
 521
 522         return 0;
 523 }
 524
 525 static uint32_t
 526 rte_sched_subport_get_array_base(struct rte_sched_subport_params *params,
 527         enum rte_sched_subport_array array)
 528 {
 529         uint32_t n_pipes_per_subport = params->n_pipes_per_subport_enabled;
 530         uint32_t n_subport_pipe_queues =
 531                 RTE_SCHED_QUEUES_PER_PIPE * n_pipes_per_subport;
 532
 533         uint32_t size_pipe = n_pipes_per_subport * sizeof(struct rte_sched_pipe);
 534         uint32_t size_queue =
 535                 n_subport_pipe_queues * sizeof(struct rte_sched_queue);
 536         uint32_t size_queue_extra
 537                 = n_subport_pipe_queues * sizeof(struct rte_sched_queue_extra);
 538         uint32_t size_pipe_profiles = params->n_max_pipe_profiles *
 539                 sizeof(struct rte_sched_pipe_profile);
 540         uint32_t size_bmp_array =
 541                 rte_bitmap_get_memory_footprint(n_subport_pipe_queues);
 542         uint32_t size_per_pipe_queue_array, size_queue_array;
 543
 544         uint32_t base, i;
 545
 546         size_per_pipe_queue_array = 0;
 547         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 548                 if (i < RTE_SCHED_TRAFFIC_CLASS_BE)
 549                         size_per_pipe_queue_array +=
 550                                 params->qsize[i] * sizeof(struct rte_mbuf *);
 551                 else
 552                         size_per_pipe_queue_array += RTE_SCHED_MAX_QUEUES_PER_TC *
 553                                 params->qsize[i] * sizeof(struct rte_mbuf *);
 554         }
 555         size_queue_array = n_pipes_per_subport * size_per_pipe_queue_array;
 556
 557         base = 0;
 558
 559         if (array == e_RTE_SCHED_SUBPORT_ARRAY_PIPE)
 560                 return base;
 561         base += RTE_CACHE_LINE_ROUNDUP(size_pipe);
 562
 563         if (array == e_RTE_SCHED_SUBPORT_ARRAY_QUEUE)
 564                 return base;
 565         base += RTE_CACHE_LINE_ROUNDUP(size_queue);
 566
 567         if (array == e_RTE_SCHED_SUBPORT_ARRAY_QUEUE_EXTRA)
 568                 return base;
 569         base += RTE_CACHE_LINE_ROUNDUP(size_queue_extra);
 570
 571         if (array == e_RTE_SCHED_SUBPORT_ARRAY_PIPE_PROFILES)
 572                 return base;
 573         base += RTE_CACHE_LINE_ROUNDUP(size_pipe_profiles);
 574
 575         if (array == e_RTE_SCHED_SUBPORT_ARRAY_BMP_ARRAY)
 576                 return base;
 577         base += RTE_CACHE_LINE_ROUNDUP(size_bmp_array);
 578
 579         if (array == e_RTE_SCHED_SUBPORT_ARRAY_QUEUE_ARRAY)
 580                 return base;
 581         base += RTE_CACHE_LINE_ROUNDUP(size_queue_array);
 582
 583         return base;
 584 }
 585
 586 static void
 587 rte_sched_subport_config_qsize(struct rte_sched_subport *subport)
 588 {
 589         uint32_t i;
 590
 591         subport->qsize_add[0] = 0;
 592
 593         /* Strict priority traffic class */
 594         for (i = 1; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 595                 subport->qsize_add[i] = subport->qsize_add[i-1] + subport->qsize[i-1];
 596
 597         /* Best-effort traffic class */
 598         subport->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 1] =
 599                 subport->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE] +
 600                 subport->qsize[RTE_SCHED_TRAFFIC_CLASS_BE];
 601         subport->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 2] =
 602                 subport->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 1] +
 603                 subport->qsize[RTE_SCHED_TRAFFIC_CLASS_BE];
 604         subport->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 3] =
 605                 subport->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 2] +
 606                 subport->qsize[RTE_SCHED_TRAFFIC_CLASS_BE];
 607
 608         subport->qsize_sum = subport->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 3] +
 609                 subport->qsize[RTE_SCHED_TRAFFIC_CLASS_BE];
 610 }
 611
 612 static void
 613 rte_sched_port_log_pipe_profile(struct rte_sched_subport *subport, uint32_t i)
 614 {
 615         struct rte_sched_pipe_profile *p = subport->pipe_profiles + i;
 616
 617         RTE_LOG(DEBUG, SCHED, "Low level config for pipe profile %u:\n"
 618                 "       Token bucket: period = %"PRIu64", credits per period = %"PRIu64", size = %"PRIu64"\n"
 619                 "       Traffic classes: period = %"PRIu64",\n"
 620                 "       credits per period = [%"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64
 621                 ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64
 622                 ", %"PRIu64", %"PRIu64", %"PRIu64"]\n"
 623                 "       Best-effort traffic class oversubscription: weight = %hhu\n"
 624                 "       WRR cost: [%hhu, %hhu, %hhu, %hhu]\n",
 625                 i,
 626
 627                 /* Token bucket */
 628                 p->tb_period,
 629                 p->tb_credits_per_period,
 630                 p->tb_size,
 631
 632                 /* Traffic classes */
 633                 p->tc_period,
 634                 p->tc_credits_per_period[0],
 635                 p->tc_credits_per_period[1],
 636                 p->tc_credits_per_period[2],
 637                 p->tc_credits_per_period[3],
 638                 p->tc_credits_per_period[4],
 639                 p->tc_credits_per_period[5],
 640                 p->tc_credits_per_period[6],
 641                 p->tc_credits_per_period[7],
 642                 p->tc_credits_per_period[8],
 643                 p->tc_credits_per_period[9],
 644                 p->tc_credits_per_period[10],
 645                 p->tc_credits_per_period[11],
 646                 p->tc_credits_per_period[12],
 647
 648                 /* Best-effort traffic class oversubscription */
 649                 p->tc_ov_weight,
 650
 651                 /* WRR */
 652                 p->wrr_cost[0], p->wrr_cost[1], p->wrr_cost[2], p->wrr_cost[3]);
 653 }
 654
 655 static void
 656 rte_sched_port_log_subport_profile(struct rte_sched_port *port, uint32_t i)
 657 {
 658         struct rte_sched_subport_profile *p = port->subport_profiles + i;
 659
 660         RTE_LOG(DEBUG, SCHED, "Low level config for subport profile %u:\n"
 661         "Token bucket: period = %"PRIu64", credits per period = %"PRIu64","
 662         "size = %"PRIu64"\n"
 663         "Traffic classes: period = %"PRIu64",\n"
 664         "credits per period = [%"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64
 665         " %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64
 666         " %"PRIu64", %"PRIu64", %"PRIu64"]\n",
 667         i,
 668
 669         /* Token bucket */
 670         p->tb_period,
 671         p->tb_credits_per_period,
 672         p->tb_size,
 673
 674         /* Traffic classes */
 675         p->tc_period,
 676         p->tc_credits_per_period[0],
 677         p->tc_credits_per_period[1],
 678         p->tc_credits_per_period[2],
 679         p->tc_credits_per_period[3],
 680         p->tc_credits_per_period[4],
 681         p->tc_credits_per_period[5],
 682         p->tc_credits_per_period[6],
 683         p->tc_credits_per_period[7],
 684         p->tc_credits_per_period[8],
 685         p->tc_credits_per_period[9],
 686         p->tc_credits_per_period[10],
 687         p->tc_credits_per_period[11],
 688         p->tc_credits_per_period[12]);
 689 }
 690
 691 static inline uint64_t
 692 rte_sched_time_ms_to_bytes(uint64_t time_ms, uint64_t rate)
 693 {
 694         uint64_t time = time_ms;
 695
 696         time = (time * rate) / 1000;
 697
 698         return time;
 699 }
 700
 701 static void
 702 rte_sched_pipe_profile_convert(struct rte_sched_subport *subport,
 703         struct rte_sched_pipe_params *src,
 704         struct rte_sched_pipe_profile *dst,
 705         uint64_t rate)
 706 {
 707         uint32_t wrr_cost[RTE_SCHED_BE_QUEUES_PER_PIPE];
 708         uint32_t lcd1, lcd2, lcd;
 709         uint32_t i;
 710
 711         /* Token Bucket */
 712         if (src->tb_rate == rate) {
 713                 dst->tb_credits_per_period = 1;
 714                 dst->tb_period = 1;
 715         } else {
 716                 double tb_rate = (double) src->tb_rate
 717                                 / (double) rate;
 718                 double d = RTE_SCHED_TB_RATE_CONFIG_ERR;
 719
 720                 rte_approx_64(tb_rate, d, &dst->tb_credits_per_period,
 721                         &dst->tb_period);
 722         }
 723
 724         dst->tb_size = src->tb_size;
 725
 726         /* Traffic Classes */
 727         dst->tc_period = rte_sched_time_ms_to_bytes(src->tc_period,
 728                                                 rate);
 729
 730         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 731                 if (subport->qsize[i])
 732                         dst->tc_credits_per_period[i]
 733                                 = rte_sched_time_ms_to_bytes(src->tc_period,
 734                                         src->tc_rate[i]);
 735
 736         dst->tc_ov_weight = src->tc_ov_weight;
 737
 738         /* WRR queues */
 739         wrr_cost[0] = src->wrr_weights[0];
 740         wrr_cost[1] = src->wrr_weights[1];
 741         wrr_cost[2] = src->wrr_weights[2];
 742         wrr_cost[3] = src->wrr_weights[3];
 743
 744         lcd1 = rte_get_lcd(wrr_cost[0], wrr_cost[1]);
 745         lcd2 = rte_get_lcd(wrr_cost[2], wrr_cost[3]);
 746         lcd = rte_get_lcd(lcd1, lcd2);
 747
 748         wrr_cost[0] = lcd / wrr_cost[0];
 749         wrr_cost[1] = lcd / wrr_cost[1];
 750         wrr_cost[2] = lcd / wrr_cost[2];
 751         wrr_cost[3] = lcd / wrr_cost[3];
 752
 753         dst->wrr_cost[0] = (uint8_t) wrr_cost[0];
 754         dst->wrr_cost[1] = (uint8_t) wrr_cost[1];
 755         dst->wrr_cost[2] = (uint8_t) wrr_cost[2];
 756         dst->wrr_cost[3] = (uint8_t) wrr_cost[3];
 757 }
 758
 759 static void
 760 rte_sched_subport_profile_convert(struct rte_sched_subport_profile_params *src,
 761         struct rte_sched_subport_profile *dst,
 762         uint64_t rate)
 763 {
 764         uint32_t i;
 765
 766         /* Token Bucket */
 767         if (src->tb_rate == rate) {
 768                 dst->tb_credits_per_period = 1;
 769                 dst->tb_period = 1;
 770         } else {
 771                 double tb_rate = (double) src->tb_rate
 772                                 / (double) rate;
 773                 double d = RTE_SCHED_TB_RATE_CONFIG_ERR;
 774
 775                 rte_approx_64(tb_rate, d, &dst->tb_credits_per_period,
 776                         &dst->tb_period);
 777         }
 778
 779         dst->tb_size = src->tb_size;
 780
 781         /* Traffic Classes */
 782         dst->tc_period = rte_sched_time_ms_to_bytes(src->tc_period, rate);
 783
 784         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 785                 dst->tc_credits_per_period[i]
 786                         = rte_sched_time_ms_to_bytes(src->tc_period,
 787                                 src->tc_rate[i]);
 788 }
 789
 790 static void
 791 rte_sched_subport_config_pipe_profile_table(struct rte_sched_subport *subport,
 792         struct rte_sched_subport_params *params, uint64_t rate)
 793 {
 794         uint32_t i;
 795
 796         for (i = 0; i < subport->n_pipe_profiles; i++) {
 797                 struct rte_sched_pipe_params *src = params->pipe_profiles + i;
 798                 struct rte_sched_pipe_profile *dst = subport->pipe_profiles + i;
 799
 800                 rte_sched_pipe_profile_convert(subport, src, dst, rate);
 801                 rte_sched_port_log_pipe_profile(subport, i);
 802         }
 803
 804         subport->pipe_tc_be_rate_max = 0;
 805         for (i = 0; i < subport->n_pipe_profiles; i++) {
 806                 struct rte_sched_pipe_params *src = params->pipe_profiles + i;
 807                 uint64_t pipe_tc_be_rate = src->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE];
 808
 809                 if (subport->pipe_tc_be_rate_max < pipe_tc_be_rate)
 810                         subport->pipe_tc_be_rate_max = pipe_tc_be_rate;
 811         }
 812 }
 813
 814 static void
 815 rte_sched_port_config_subport_profile_table(struct rte_sched_port *port,
 816         struct rte_sched_port_params *params,
 817         uint64_t rate)
 818 {
 819         uint32_t i;
 820
 821         for (i = 0; i < port->n_subport_profiles; i++) {
 822                 struct rte_sched_subport_profile_params *src
 823                                 = params->subport_profiles + i;
 824                 struct rte_sched_subport_profile *dst
 825                                 = port->subport_profiles + i;
 826
 827                 rte_sched_subport_profile_convert(src, dst, rate);
 828                 rte_sched_port_log_subport_profile(port, i);
 829         }
 830 }
 831
 832 static int
 833 rte_sched_subport_check_params(struct rte_sched_subport_params *params,
 834         uint32_t n_max_pipes_per_subport,
 835         uint64_t rate)
 836 {
 837         uint32_t i;
 838
 839         /* Check user parameters */
 840         if (params == NULL) {
 841                 RTE_LOG(ERR, SCHED,
 842                         "%s: Incorrect value for parameter params\n", __func__);
 843                 return -EINVAL;
 844         }
 845
 846         /* qsize: if non-zero, power of 2,
 847          * no bigger than 32K (due to 16-bit read/write pointers)
 848          */
 849         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 850                 uint16_t qsize = params->qsize[i];
 851
 852                 if (qsize != 0 && !rte_is_power_of_2(qsize)) {
 853                         RTE_LOG(ERR, SCHED,
 854                                 "%s: Incorrect value for qsize\n", __func__);
 855                         return -EINVAL;
 856                 }
 857         }
 858
 859         if (params->qsize[RTE_SCHED_TRAFFIC_CLASS_BE] == 0) {
 860                 RTE_LOG(ERR, SCHED, "%s: Incorrect qsize\n", __func__);
 861                 return -EINVAL;
 862         }
 863
 864         /* n_pipes_per_subport: non-zero, power of 2 */
 865         if (params->n_pipes_per_subport_enabled == 0 ||
 866                 params->n_pipes_per_subport_enabled > n_max_pipes_per_subport ||
 867             !rte_is_power_of_2(params->n_pipes_per_subport_enabled)) {
 868                 RTE_LOG(ERR, SCHED,
 869                         "%s: Incorrect value for pipes number\n", __func__);
 870                 return -EINVAL;
 871         }
 872
 873         /* pipe_profiles and n_pipe_profiles */
 874         if (params->pipe_profiles == NULL ||
 875             params->n_pipe_profiles == 0 ||
 876                 params->n_max_pipe_profiles == 0 ||
 877                 params->n_pipe_profiles > params->n_max_pipe_profiles) {
 878                 RTE_LOG(ERR, SCHED,
 879                         "%s: Incorrect value for pipe profiles\n", __func__);
 880                 return -EINVAL;
 881         }
 882
 883         for (i = 0; i < params->n_pipe_profiles; i++) {
 884                 struct rte_sched_pipe_params *p = params->pipe_profiles + i;
 885                 int status;
 886
 887                 status = pipe_profile_check(p, rate, &params->qsize[0]);
 888                 if (status != 0) {
 889                         RTE_LOG(ERR, SCHED,
 890                                 "%s: Pipe profile check failed(%d)\n", __func__, status);
 891                         return -EINVAL;
 892                 }
 893         }
 894
 895         return 0;
 896 }
 897
 898 uint32_t
 899 rte_sched_port_get_memory_footprint(struct rte_sched_port_params *port_params,
 900         struct rte_sched_subport_params **subport_params)
 901 {
 902         uint32_t size0 = 0, size1 = 0, i;
 903         int status;
 904
 905         status = rte_sched_port_check_params(port_params);
 906         if (status != 0) {
 907                 RTE_LOG(ERR, SCHED,
 908                         "%s: Port scheduler port params check failed (%d)\n",
 909                         __func__, status);
 910
 911                 return 0;
 912         }
 913
 914         for (i = 0; i < port_params->n_subports_per_port; i++) {
 915                 struct rte_sched_subport_params *sp = subport_params[i];
 916
 917                 status = rte_sched_subport_check_params(sp,
 918                                 port_params->n_pipes_per_subport,
 919                                 port_params->rate);
 920                 if (status != 0) {
 921                         RTE_LOG(ERR, SCHED,
 922                                 "%s: Port scheduler subport params check failed (%d)\n",
 923                                 __func__, status);
 924
 925                         return 0;
 926                 }
 927         }
 928
 929         size0 = sizeof(struct rte_sched_port);
 930
 931         for (i = 0; i < port_params->n_subports_per_port; i++) {
 932                 struct rte_sched_subport_params *sp = subport_params[i];
 933
 934                 size1 += rte_sched_subport_get_array_base(sp,
 935                                         e_RTE_SCHED_SUBPORT_ARRAY_TOTAL);
 936         }
 937
 938         return size0 + size1;
 939 }
 940
 941 struct rte_sched_port *
 942 rte_sched_port_config(struct rte_sched_port_params *params)
 943 {
 944         struct rte_sched_port *port = NULL;
 945         uint32_t size0, size1, size2;
 946         uint32_t cycles_per_byte;
 947         uint32_t i, j;
 948         int status;
 949
 950         status = rte_sched_port_check_params(params);
 951         if (status != 0) {
 952                 RTE_LOG(ERR, SCHED,
 953                         "%s: Port scheduler params check failed (%d)\n",
 954                         __func__, status);
 955                 return NULL;
 956         }
 957
 958         size0 = sizeof(struct rte_sched_port);
 959         size1 = params->n_subports_per_port * sizeof(struct rte_sched_subport *);
 960         size2 = params->n_max_subport_profiles *
 961                 sizeof(struct rte_sched_subport_profile);
 962
 963         /* Allocate memory to store the data structures */
 964         port = rte_zmalloc_socket("qos_params", size0 + size1,
 965                                  RTE_CACHE_LINE_SIZE, params->socket);
 966         if (port == NULL) {
 967                 RTE_LOG(ERR, SCHED, "%s: Memory allocation fails\n", __func__);
 968
 969                 return NULL;
 970         }
 971
 972         /* Allocate memory to store the subport profile */
 973         port->subport_profiles  = rte_zmalloc_socket("subport_profile", size2,
 974                                         RTE_CACHE_LINE_SIZE, params->socket);
 975         if (port->subport_profiles == NULL) {
 976                 RTE_LOG(ERR, SCHED, "%s: Memory allocation fails\n", __func__);
 977                 rte_free(port);
 978                 return NULL;
 979         }
 980
 981         /* User parameters */
 982         port->n_subports_per_port = params->n_subports_per_port;
 983         port->n_subport_profiles = params->n_subport_profiles;
 984         port->n_max_subport_profiles = params->n_max_subport_profiles;
 985         port->n_pipes_per_subport = params->n_pipes_per_subport;
 986         port->n_pipes_per_subport_log2 =
 987                         __builtin_ctz(params->n_pipes_per_subport);
 988         port->socket = params->socket;
 989
 990         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 991                 port->pipe_queue[i] = i;
 992
 993         for (i = 0, j = 0; i < RTE_SCHED_QUEUES_PER_PIPE; i++) {
 994                 port->pipe_tc[i] = j;
 995
 996                 if (j < RTE_SCHED_TRAFFIC_CLASS_BE)
 997                         j++;
 998         }
 999
1000         for (i = 0, j = 0; i < RTE_SCHED_QUEUES_PER_PIPE; i++) {
1001                 port->tc_queue[i] = j;
1002
1003                 if (i >= RTE_SCHED_TRAFFIC_CLASS_BE)
1004                         j++;
1005         }
1006         port->rate = params->rate;
1007         port->mtu = params->mtu + params->frame_overhead;
1008         port->frame_overhead = params->frame_overhead;
1009
1010         /* Timing */
1011         port->time_cpu_cycles = rte_get_tsc_cycles();
1012         port->time_cpu_bytes = 0;
1013         port->time = 0;
1014
1015         /* Subport profile table */
1016         rte_sched_port_config_subport_profile_table(port, params, port->rate);
1017
1018         cycles_per_byte = (rte_get_tsc_hz() << RTE_SCHED_TIME_SHIFT)
1019                 / params->rate;
1020         port->inv_cycles_per_byte = rte_reciprocal_value(cycles_per_byte);
1021         port->cycles_per_byte = cycles_per_byte;
1022
1023         /* Grinders */
1024         port->pkts_out = NULL;
1025         port->n_pkts_out = 0;
1026         port->subport_id = 0;
1027
1028         return port;
1029 }
1030
1031 static inline void
1032 rte_sched_subport_free(struct rte_sched_port *port,
1033         struct rte_sched_subport *subport)
1034 {
1035         uint32_t n_subport_pipe_queues;
1036         uint32_t qindex;
1037
1038         if (subport == NULL)
1039                 return;
1040
1041         n_subport_pipe_queues = rte_sched_subport_pipe_queues(subport);
1042
1043         /* Free enqueued mbufs */
1044         for (qindex = 0; qindex < n_subport_pipe_queues; qindex++) {
1045                 struct rte_mbuf **mbufs =
1046                         rte_sched_subport_pipe_qbase(subport, qindex);
1047                 uint16_t qsize = rte_sched_subport_pipe_qsize(port, subport, qindex);
1048                 if (qsize != 0) {
1049                         struct rte_sched_queue *queue = subport->queue + qindex;
1050                         uint16_t qr = queue->qr & (qsize - 1);
1051                         uint16_t qw = queue->qw & (qsize - 1);
1052
1053                         for (; qr != qw; qr = (qr + 1) & (qsize - 1))
1054                                 rte_pktmbuf_free(mbufs[qr]);
1055                 }
1056         }
1057
1058         rte_free(subport);
1059 }
1060
1061 void
1062 rte_sched_port_free(struct rte_sched_port *port)
1063 {
1064         uint32_t i;
1065
1066         /* Check user parameters */
1067         if (port == NULL)
1068                 return;
1069
1070         for (i = 0; i < port->n_subports_per_port; i++)
1071                 rte_sched_subport_free(port, port->subports[i]);
1072
1073         rte_free(port->subport_profiles);
1074         rte_free(port);
1075 }
1076
1077 static void
1078 rte_sched_free_memory(struct rte_sched_port *port, uint32_t n_subports)
1079 {
1080         uint32_t i;
1081
1082         for (i = 0; i < n_subports; i++) {
1083                 struct rte_sched_subport *subport = port->subports[i];
1084
1085                 rte_sched_subport_free(port, subport);
1086         }
1087
1088         rte_free(port->subport_profiles);
1089         rte_free(port);
1090 }
1091
1092 #ifdef RTE_SCHED_CMAN
1093 static int
1094 rte_sched_red_config(struct rte_sched_port *port,
1095         struct rte_sched_subport *s,
1096         struct rte_sched_subport_params *params,
1097         uint32_t n_subports)
1098 {
1099         uint32_t i;
1100
1101         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
1102
1103                 uint32_t j;
1104
1105                 for (j = 0; j < RTE_COLORS; j++) {
1106                         /* if min/max are both zero, then RED is disabled */
1107                         if ((params->cman_params->red_params[i][j].min_th |
1108                                  params->cman_params->red_params[i][j].max_th) == 0) {
1109                                 continue;
1110                         }
1111
1112                         if (rte_red_config_init(&s->red_config[i][j],
1113                                 params->cman_params->red_params[i][j].wq_log2,
1114                                 params->cman_params->red_params[i][j].min_th,
1115                                 params->cman_params->red_params[i][j].max_th,
1116                                 params->cman_params->red_params[i][j].maxp_inv) != 0) {
1117                                 rte_sched_free_memory(port, n_subports);
1118
1119                                 RTE_LOG(NOTICE, SCHED,
1120                                 "%s: RED configuration init fails\n", __func__);
1121                                 return -EINVAL;
1122                         }
1123                 }
1124         }
1125         s->cman = RTE_SCHED_CMAN_RED;
1126         return 0;
1127 }
1128
1129 static int
1130 rte_sched_pie_config(struct rte_sched_port *port,
1131         struct rte_sched_subport *s,
1132         struct rte_sched_subport_params *params,
1133         uint32_t n_subports)
1134 {
1135         uint32_t i;
1136
1137         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
1138                 if (params->cman_params->pie_params[i].tailq_th > params->qsize[i]) {
1139                         RTE_LOG(NOTICE, SCHED,
1140                         "%s: PIE tailq threshold incorrect\n", __func__);
1141                         return -EINVAL;
1142                 }
1143
1144                 if (rte_pie_config_init(&s->pie_config[i],
1145                         params->cman_params->pie_params[i].qdelay_ref,
1146                         params->cman_params->pie_params[i].dp_update_interval,
1147                         params->cman_params->pie_params[i].max_burst,
1148                         params->cman_params->pie_params[i].tailq_th) != 0) {
1149                         rte_sched_free_memory(port, n_subports);
1150
1151                         RTE_LOG(NOTICE, SCHED,
1152                         "%s: PIE configuration init fails\n", __func__);
1153                         return -EINVAL;
1154                         }
1155         }
1156         s->cman = RTE_SCHED_CMAN_PIE;
1157         return 0;
1158 }
1159
1160 static int
1161 rte_sched_cman_config(struct rte_sched_port *port,
1162         struct rte_sched_subport *s,
1163         struct rte_sched_subport_params *params,
1164         uint32_t n_subports)
1165 {
1166         if (params->cman_params->cman_mode == RTE_SCHED_CMAN_RED)
1167                 return rte_sched_red_config(port, s, params, n_subports);
1168
1169         else if (params->cman_params->cman_mode == RTE_SCHED_CMAN_PIE)
1170                 return rte_sched_pie_config(port, s, params, n_subports);
1171
1172         return -EINVAL;
1173 }
1174 #endif
1175
1176 int
1177 rte_sched_subport_config(struct rte_sched_port *port,
1178         uint32_t subport_id,
1179         struct rte_sched_subport_params *params,
1180         uint32_t subport_profile_id)
1181 {
1182         struct rte_sched_subport *s = NULL;
1183         uint32_t n_subports = subport_id;
1184         struct rte_sched_subport_profile *profile;
1185         uint32_t n_subport_pipe_queues, i;
1186         uint32_t size0, size1, bmp_mem_size;
1187         int status;
1188         int ret;
1189
1190         /* Check user parameters */
1191         if (port == NULL) {
1192                 RTE_LOG(ERR, SCHED,
1193                         "%s: Incorrect value for parameter port\n", __func__);
1194                 return 0;
1195         }
1196
1197         if (subport_id >= port->n_subports_per_port) {
1198                 RTE_LOG(ERR, SCHED,
1199                         "%s: Incorrect value for subport id\n", __func__);
1200                 ret = -EINVAL;
1201                 goto out;
1202         }
1203
1204         if (subport_profile_id >= port->n_max_subport_profiles) {
1205                 RTE_LOG(ERR, SCHED, "%s: "
1206                         "Number of subport profile exceeds the max limit\n",
1207                         __func__);
1208                 ret = -EINVAL;
1209                 goto out;
1210         }
1211
1212         /** Memory is allocated only on first invocation of the api for a
1213          * given subport. Subsequent invocation on same subport will just
1214          * update subport bandwidth parameter.
1215          **/
1216         if (port->subports[subport_id] == NULL) {
1217
1218                 status = rte_sched_subport_check_params(params,
1219                         port->n_pipes_per_subport,
1220                         port->rate);
1221                 if (status != 0) {
1222                         RTE_LOG(NOTICE, SCHED,
1223                                 "%s: Port scheduler params check failed (%d)\n",
1224                                 __func__, status);
1225                         ret = -EINVAL;
1226                         goto out;
1227                 }
1228
1229                 /* Determine the amount of memory to allocate */
1230                 size0 = sizeof(struct rte_sched_subport);
1231                 size1 = rte_sched_subport_get_array_base(params,
1232                                         e_RTE_SCHED_SUBPORT_ARRAY_TOTAL);
1233
1234                 /* Allocate memory to store the data structures */
1235                 s = rte_zmalloc_socket("subport_params", size0 + size1,
1236                         RTE_CACHE_LINE_SIZE, port->socket);
1237                 if (s == NULL) {
1238                         RTE_LOG(ERR, SCHED,
1239                                 "%s: Memory allocation fails\n", __func__);
1240                         ret = -ENOMEM;
1241                         goto out;
1242                 }
1243
1244                 n_subports++;
1245
1246                 subport_profile_id = 0;
1247
1248                 /* Port */
1249                 port->subports[subport_id] = s;
1250
1251                 s->tb_time = port->time;
1252
1253                 /* compile time checks */
1254                 RTE_BUILD_BUG_ON(RTE_SCHED_PORT_N_GRINDERS == 0);
1255                 RTE_BUILD_BUG_ON(RTE_SCHED_PORT_N_GRINDERS &
1256                         (RTE_SCHED_PORT_N_GRINDERS - 1));
1257
1258                 /* User parameters */
1259                 s->n_pipes_per_subport_enabled =
1260                                 params->n_pipes_per_subport_enabled;
1261                 memcpy(s->qsize, params->qsize, sizeof(params->qsize));
1262                 s->n_pipe_profiles = params->n_pipe_profiles;
1263                 s->n_max_pipe_profiles = params->n_max_pipe_profiles;
1264
1265 #ifdef RTE_SCHED_CMAN
1266                 if (params->cman_params != NULL) {
1267                         s->cman_enabled = true;
1268                         status = rte_sched_cman_config(port, s, params, n_subports);
1269                         if (status) {
1270                                 RTE_LOG(NOTICE, SCHED,
1271                                         "%s: CMAN configuration fails\n", __func__);
1272                                 return status;
1273                         }
1274                 } else {
1275                         s->cman_enabled = false;
1276                 }
1277 #endif
1278
1279                 /* Scheduling loop detection */
1280                 s->pipe_loop = RTE_SCHED_PIPE_INVALID;
1281                 s->pipe_exhaustion = 0;
1282
1283                 /* Grinders */
1284                 s->busy_grinders = 0;
1285
1286                 /* Queue base calculation */
1287                 rte_sched_subport_config_qsize(s);
1288
1289                 /* Large data structures */
1290                 s->pipe = (struct rte_sched_pipe *)
1291                         (s->memory + rte_sched_subport_get_array_base(params,
1292                         e_RTE_SCHED_SUBPORT_ARRAY_PIPE));
1293                 s->queue = (struct rte_sched_queue *)
1294                         (s->memory + rte_sched_subport_get_array_base(params,
1295                         e_RTE_SCHED_SUBPORT_ARRAY_QUEUE));
1296                 s->queue_extra = (struct rte_sched_queue_extra *)
1297                         (s->memory + rte_sched_subport_get_array_base(params,
1298                         e_RTE_SCHED_SUBPORT_ARRAY_QUEUE_EXTRA));
1299                 s->pipe_profiles = (struct rte_sched_pipe_profile *)
1300                         (s->memory + rte_sched_subport_get_array_base(params,
1301                         e_RTE_SCHED_SUBPORT_ARRAY_PIPE_PROFILES));
1302                 s->bmp_array =  s->memory + rte_sched_subport_get_array_base(
1303                                 params, e_RTE_SCHED_SUBPORT_ARRAY_BMP_ARRAY);
1304                 s->queue_array = (struct rte_mbuf **)
1305                         (s->memory + rte_sched_subport_get_array_base(params,
1306                         e_RTE_SCHED_SUBPORT_ARRAY_QUEUE_ARRAY));
1307
1308                 /* Pipe profile table */
1309                 rte_sched_subport_config_pipe_profile_table(s, params,
1310                                                             port->rate);
1311
1312                 /* Bitmap */
1313                 n_subport_pipe_queues = rte_sched_subport_pipe_queues(s);
1314                 bmp_mem_size = rte_bitmap_get_memory_footprint(
1315                                                 n_subport_pipe_queues);
1316                 s->bmp = rte_bitmap_init(n_subport_pipe_queues, s->bmp_array,
1317                                         bmp_mem_size);
1318                 if (s->bmp == NULL) {
1319                         RTE_LOG(ERR, SCHED,
1320                                 "%s: Subport bitmap init error\n", __func__);
1321                         ret = -EINVAL;
1322                         goto out;
1323                 }
1324
1325                 for (i = 0; i < RTE_SCHED_PORT_N_GRINDERS; i++)
1326                         s->grinder_base_bmp_pos[i] = RTE_SCHED_PIPE_INVALID;
1327
1328 #ifdef RTE_SCHED_SUBPORT_TC_OV
1329                 /* TC oversubscription */
1330                 s->tc_ov_wm_min = port->mtu;
1331                 s->tc_ov_period_id = 0;
1332                 s->tc_ov = 0;
1333                 s->tc_ov_n = 0;
1334                 s->tc_ov_rate = 0;
1335 #endif
1336         }
1337
1338         {
1339         /* update subport parameters from subport profile table*/
1340                 profile = port->subport_profiles + subport_profile_id;
1341
1342                 s = port->subports[subport_id];
1343
1344                 s->tb_credits = profile->tb_size / 2;
1345
1346                 s->tc_time = port->time + profile->tc_period;
1347
1348                 for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
1349                         if (s->qsize[i])
1350                                 s->tc_credits[i] =
1351                                         profile->tc_credits_per_period[i];
1352                         else
1353                                 profile->tc_credits_per_period[i] = 0;
1354
1355 #ifdef RTE_SCHED_SUBPORT_TC_OV
1356                 s->tc_ov_wm_max = rte_sched_time_ms_to_bytes(profile->tc_period,
1357                                                         s->pipe_tc_be_rate_max);
1358                 s->tc_ov_wm = s->tc_ov_wm_max;
1359 #endif
1360                 s->profile = subport_profile_id;
1361
1362         }
1363
1364         rte_sched_port_log_subport_profile(port, subport_profile_id);
1365
1366         return 0;
1367
1368 out:
1369         rte_sched_free_memory(port, n_subports);
1370
1371         return ret;
1372 }
1373
1374 int
1375 rte_sched_pipe_config(struct rte_sched_port *port,
1376         uint32_t subport_id,
1377         uint32_t pipe_id,
1378         int32_t pipe_profile)
1379 {
1380         struct rte_sched_subport *s;
1381         struct rte_sched_subport_profile *sp;
1382         struct rte_sched_pipe *p;
1383         struct rte_sched_pipe_profile *params;
1384         uint32_t n_subports = subport_id + 1;
1385         uint32_t deactivate, profile, i;
1386         int ret;
1387
1388         /* Check user parameters */
1389         profile = (uint32_t) pipe_profile;
1390         deactivate = (pipe_profile < 0);
1391
1392         if (port == NULL) {
1393                 RTE_LOG(ERR, SCHED,
1394                         "%s: Incorrect value for parameter port\n", __func__);
1395                 return -EINVAL;
1396         }
1397
1398         if (subport_id >= port->n_subports_per_port) {
1399                 RTE_LOG(ERR, SCHED,
1400                         "%s: Incorrect value for parameter subport id\n", __func__);
1401                 ret = -EINVAL;
1402                 goto out;
1403         }
1404
1405         s = port->subports[subport_id];
1406         if (pipe_id >= s->n_pipes_per_subport_enabled) {
1407                 RTE_LOG(ERR, SCHED,
1408                         "%s: Incorrect value for parameter pipe id\n", __func__);
1409                 ret = -EINVAL;
1410                 goto out;
1411         }
1412
1413         if (!deactivate && profile >= s->n_pipe_profiles) {
1414                 RTE_LOG(ERR, SCHED,
1415                         "%s: Incorrect value for parameter pipe profile\n", __func__);
1416                 ret = -EINVAL;
1417                 goto out;
1418         }
1419
1420         sp = port->subport_profiles + s->profile;
1421         /* Handle the case when pipe already has a valid configuration */
1422         p = s->pipe + pipe_id;
1423         if (p->tb_time) {
1424                 params = s->pipe_profiles + p->profile;
1425
1426                 double subport_tc_be_rate =
1427                 (double)sp->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE]
1428                         / (double) sp->tc_period;
1429                 double pipe_tc_be_rate =
1430                         (double) params->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE]
1431                         / (double) params->tc_period;
1432                 uint32_t tc_be_ov = s->tc_ov;
1433
1434                 /* Unplug pipe from its subport */
1435                 s->tc_ov_n -= params->tc_ov_weight;
1436                 s->tc_ov_rate -= pipe_tc_be_rate;
1437                 s->tc_ov = s->tc_ov_rate > subport_tc_be_rate;
1438
1439                 if (s->tc_ov != tc_be_ov) {
1440                         RTE_LOG(DEBUG, SCHED,
1441                                 "Subport %u Best-effort TC oversubscription is OFF (%.4lf >= %.4lf)\n",
1442                                 subport_id, subport_tc_be_rate, s->tc_ov_rate);
1443                 }
1444
1445                 /* Reset the pipe */
1446                 memset(p, 0, sizeof(struct rte_sched_pipe));
1447         }
1448
1449         if (deactivate)
1450                 return 0;
1451
1452         /* Apply the new pipe configuration */
1453         p->profile = profile;
1454         params = s->pipe_profiles + p->profile;
1455
1456         /* Token Bucket (TB) */
1457         p->tb_time = port->time;
1458         p->tb_credits = params->tb_size / 2;
1459
1460         /* Traffic Classes (TCs) */
1461         p->tc_time = port->time + params->tc_period;
1462
1463         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
1464                 if (s->qsize[i])
1465                         p->tc_credits[i] = params->tc_credits_per_period[i];
1466
1467         {
1468                 /* Subport best effort tc oversubscription */
1469                 double subport_tc_be_rate =
1470                 (double)sp->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE]
1471                         / (double) sp->tc_period;
1472                 double pipe_tc_be_rate =
1473                         (double) params->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE]
1474                         / (double) params->tc_period;
1475                 uint32_t tc_be_ov = s->tc_ov;
1476
1477                 s->tc_ov_n += params->tc_ov_weight;
1478                 s->tc_ov_rate += pipe_tc_be_rate;
1479                 s->tc_ov = s->tc_ov_rate > subport_tc_be_rate;
1480
1481                 if (s->tc_ov != tc_be_ov) {
1482                         RTE_LOG(DEBUG, SCHED,
1483                                 "Subport %u Best effort TC oversubscription is ON (%.4lf < %.4lf)\n",
1484                                 subport_id, subport_tc_be_rate, s->tc_ov_rate);
1485                 }
1486                 p->tc_ov_period_id = s->tc_ov_period_id;
1487                 p->tc_ov_credits = s->tc_ov_wm;
1488         }
1489
1490         return 0;
1491
1492 out:
1493         rte_sched_free_memory(port, n_subports);
1494
1495         return ret;
1496 }
1497
1498 int
1499 rte_sched_subport_pipe_profile_add(struct rte_sched_port *port,
1500         uint32_t subport_id,
1501         struct rte_sched_pipe_params *params,
1502         uint32_t *pipe_profile_id)
1503 {
1504         struct rte_sched_subport *s;
1505         struct rte_sched_pipe_profile *pp;
1506         uint32_t i;
1507         int status;
1508
1509         /* Port */
1510         if (port == NULL) {
1511                 RTE_LOG(ERR, SCHED,
1512                         "%s: Incorrect value for parameter port\n", __func__);
1513                 return -EINVAL;
1514         }
1515
1516         /* Subport id not exceeds the max limit */
1517         if (subport_id > port->n_subports_per_port) {
1518                 RTE_LOG(ERR, SCHED,
1519                         "%s: Incorrect value for subport id\n", __func__);
1520                 return -EINVAL;
1521         }
1522
1523         s = port->subports[subport_id];
1524
1525         /* Pipe profiles exceeds the max limit */
1526         if (s->n_pipe_profiles >= s->n_max_pipe_profiles) {
1527                 RTE_LOG(ERR, SCHED,
1528                         "%s: Number of pipe profiles exceeds the max limit\n", __func__);
1529                 return -EINVAL;
1530         }
1531
1532         /* Pipe params */
1533         status = pipe_profile_check(params, port->rate, &s->qsize[0]);
1534         if (status != 0) {
1535                 RTE_LOG(ERR, SCHED,
1536                         "%s: Pipe profile check failed(%d)\n", __func__, status);
1537                 return -EINVAL;
1538         }
1539
1540         pp = &s->pipe_profiles[s->n_pipe_profiles];
1541         rte_sched_pipe_profile_convert(s, params, pp, port->rate);
1542
1543         /* Pipe profile should not exists */
1544         for (i = 0; i < s->n_pipe_profiles; i++)
1545                 if (memcmp(s->pipe_profiles + i, pp, sizeof(*pp)) == 0) {
1546                         RTE_LOG(ERR, SCHED,
1547                                 "%s: Pipe profile exists\n", __func__);
1548                         return -EINVAL;
1549                 }
1550
1551         /* Pipe profile commit */
1552         *pipe_profile_id = s->n_pipe_profiles;
1553         s->n_pipe_profiles++;
1554
1555         if (s->pipe_tc_be_rate_max < params->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE])
1556                 s->pipe_tc_be_rate_max = params->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE];
1557
1558         rte_sched_port_log_pipe_profile(s, *pipe_profile_id);
1559
1560         return 0;
1561 }
1562
1563 int
1564 rte_sched_port_subport_profile_add(struct rte_sched_port *port,
1565         struct rte_sched_subport_profile_params *params,
1566         uint32_t *subport_profile_id)
1567 {
1568         int status;
1569         uint32_t i;
1570         struct rte_sched_subport_profile *dst;
1571
1572         /* Port */
1573         if (port == NULL) {
1574                 RTE_LOG(ERR, SCHED, "%s: "
1575                 "Incorrect value for parameter port\n", __func__);
1576                 return -EINVAL;
1577         }
1578
1579         if (params == NULL) {
1580                 RTE_LOG(ERR, SCHED, "%s: "
1581                 "Incorrect value for parameter profile\n", __func__);
1582                 return -EINVAL;
1583         }
1584
1585         if (subport_profile_id == NULL) {
1586                 RTE_LOG(ERR, SCHED, "%s: "
1587                 "Incorrect value for parameter subport_profile_id\n",
1588                 __func__);
1589                 return -EINVAL;
1590         }
1591
1592         dst = port->subport_profiles + port->n_subport_profiles;
1593
1594         /* Subport profiles exceeds the max limit */
1595         if (port->n_subport_profiles >= port->n_max_subport_profiles) {
1596                 RTE_LOG(ERR, SCHED, "%s: "
1597                 "Number of subport profiles exceeds the max limit\n",
1598                  __func__);
1599                 return -EINVAL;
1600         }
1601
1602         status = subport_profile_check(params, port->rate);
1603         if (status != 0) {
1604                 RTE_LOG(ERR, SCHED,
1605                 "%s: subport profile check failed(%d)\n", __func__, status);
1606                 return -EINVAL;
1607         }
1608
1609         rte_sched_subport_profile_convert(params, dst, port->rate);
1610
1611         /* Subport profile should not exists */
1612         for (i = 0; i < port->n_subport_profiles; i++)
1613                 if (memcmp(port->subport_profiles + i,
1614                     dst, sizeof(*dst)) == 0) {
1615                         RTE_LOG(ERR, SCHED,
1616                         "%s: subport profile exists\n", __func__);
1617                         return -EINVAL;
1618                 }
1619
1620         /* Subport profile commit */
1621         *subport_profile_id = port->n_subport_profiles;
1622         port->n_subport_profiles++;
1623
1624         rte_sched_port_log_subport_profile(port, *subport_profile_id);
1625
1626         return 0;
1627 }
1628
1629 static inline uint32_t
1630 rte_sched_port_qindex(struct rte_sched_port *port,
1631         uint32_t subport,
1632         uint32_t pipe,
1633         uint32_t traffic_class,
1634         uint32_t queue)
1635 {
1636         return ((subport & (port->n_subports_per_port - 1)) <<
1637                 (port->n_pipes_per_subport_log2 + 4)) |
1638                 ((pipe &
1639                 (port->subports[subport]->n_pipes_per_subport_enabled - 1)) << 4) |
1640                 ((rte_sched_port_pipe_queue(port, traffic_class) + queue) &
1641                 (RTE_SCHED_QUEUES_PER_PIPE - 1));
1642 }
1643
1644 void
1645 rte_sched_port_pkt_write(struct rte_sched_port *port,
1646                          struct rte_mbuf *pkt,
1647                          uint32_t subport, uint32_t pipe,
1648                          uint32_t traffic_class,
1649                          uint32_t queue, enum rte_color color)
1650 {
1651         uint32_t queue_id =
1652                 rte_sched_port_qindex(port, subport, pipe, traffic_class, queue);
1653
1654         rte_mbuf_sched_set(pkt, queue_id, traffic_class, (uint8_t)color);
1655 }
1656
1657 void
1658 rte_sched_port_pkt_read_tree_path(struct rte_sched_port *port,
1659                                   const struct rte_mbuf *pkt,
1660                                   uint32_t *subport, uint32_t *pipe,
1661                                   uint32_t *traffic_class, uint32_t *queue)
1662 {
1663         uint32_t queue_id = rte_mbuf_sched_queue_get(pkt);
1664
1665         *subport = queue_id >> (port->n_pipes_per_subport_log2 + 4);
1666         *pipe = (queue_id >> 4) &
1667                 (port->subports[*subport]->n_pipes_per_subport_enabled - 1);
1668         *traffic_class = rte_sched_port_pipe_tc(port, queue_id);
1669         *queue = rte_sched_port_tc_queue(port, queue_id);
1670 }
1671
1672 enum rte_color
1673 rte_sched_port_pkt_read_color(const struct rte_mbuf *pkt)
1674 {
1675         return (enum rte_color)rte_mbuf_sched_color_get(pkt);
1676 }
1677
1678 int
1679 rte_sched_subport_read_stats(struct rte_sched_port *port,
1680                              uint32_t subport_id,
1681                              struct rte_sched_subport_stats *stats,
1682                              uint32_t *tc_ov)
1683 {
1684         struct rte_sched_subport *s;
1685
1686         /* Check user parameters */
1687         if (port == NULL) {
1688                 RTE_LOG(ERR, SCHED,
1689                         "%s: Incorrect value for parameter port\n", __func__);
1690                 return -EINVAL;
1691         }
1692
1693         if (subport_id >= port->n_subports_per_port) {
1694                 RTE_LOG(ERR, SCHED,
1695                         "%s: Incorrect value for subport id\n", __func__);
1696                 return -EINVAL;
1697         }
1698
1699         if (stats == NULL) {
1700                 RTE_LOG(ERR, SCHED,
1701                         "%s: Incorrect value for parameter stats\n", __func__);
1702                 return -EINVAL;
1703         }
1704
1705         if (tc_ov == NULL) {
1706                 RTE_LOG(ERR, SCHED,
1707                         "%s: Incorrect value for tc_ov\n", __func__);
1708                 return -EINVAL;
1709         }
1710
1711         s = port->subports[subport_id];
1712
1713         /* Copy subport stats and clear */
1714         memcpy(stats, &s->stats, sizeof(struct rte_sched_subport_stats));
1715         memset(&s->stats, 0, sizeof(struct rte_sched_subport_stats));
1716
1717         /* Subport TC oversubscription status */
1718         *tc_ov = s->tc_ov;
1719
1720         return 0;
1721 }
1722
1723 int
1724 rte_sched_queue_read_stats(struct rte_sched_port *port,
1725         uint32_t queue_id,
1726         struct rte_sched_queue_stats *stats,
1727         uint16_t *qlen)
1728 {
1729         struct rte_sched_subport *s;
1730         struct rte_sched_queue *q;
1731         struct rte_sched_queue_extra *qe;
1732         uint32_t subport_id, subport_qmask, subport_qindex;
1733
1734         /* Check user parameters */
1735         if (port == NULL) {
1736                 RTE_LOG(ERR, SCHED,
1737                         "%s: Incorrect value for parameter port\n", __func__);
1738                 return -EINVAL;
1739         }
1740
1741         if (queue_id >= rte_sched_port_queues_per_port(port)) {
1742                 RTE_LOG(ERR, SCHED,
1743                         "%s: Incorrect value for queue id\n", __func__);
1744                 return -EINVAL;
1745         }
1746
1747         if (stats == NULL) {
1748                 RTE_LOG(ERR, SCHED,
1749                         "%s: Incorrect value for parameter stats\n", __func__);
1750                 return -EINVAL;
1751         }
1752
1753         if (qlen == NULL) {
1754                 RTE_LOG(ERR, SCHED,
1755                         "%s: Incorrect value for parameter qlen\n", __func__);
1756                 return -EINVAL;
1757         }
1758         subport_qmask = port->n_pipes_per_subport_log2 + 4;
1759         subport_id = (queue_id >> subport_qmask) & (port->n_subports_per_port - 1);
1760
1761         s = port->subports[subport_id];
1762         subport_qindex = ((1 << subport_qmask) - 1) & queue_id;
1763         q = s->queue + subport_qindex;
1764         qe = s->queue_extra + subport_qindex;
1765
1766         /* Copy queue stats and clear */
1767         memcpy(stats, &qe->stats, sizeof(struct rte_sched_queue_stats));
1768         memset(&qe->stats, 0, sizeof(struct rte_sched_queue_stats));
1769
1770         /* Queue length */
1771         *qlen = q->qw - q->qr;
1772
1773         return 0;
1774 }
1775
1776 #ifdef RTE_SCHED_DEBUG
1777
1778 static inline int
1779 rte_sched_port_queue_is_empty(struct rte_sched_subport *subport,
1780         uint32_t qindex)
1781 {
1782         struct rte_sched_queue *queue = subport->queue + qindex;
1783
1784         return queue->qr == queue->qw;
1785 }
1786
1787 #endif /* RTE_SCHED_DEBUG */
1788
1789 #ifdef RTE_SCHED_COLLECT_STATS
1790
1791 static inline void
1792 rte_sched_port_update_subport_stats(struct rte_sched_port *port,
1793         struct rte_sched_subport *subport,
1794         uint32_t qindex,
1795         struct rte_mbuf *pkt)
1796 {
1797         uint32_t tc_index = rte_sched_port_pipe_tc(port, qindex);
1798         uint32_t pkt_len = pkt->pkt_len;
1799
1800         subport->stats.n_pkts_tc[tc_index] += 1;
1801         subport->stats.n_bytes_tc[tc_index] += pkt_len;
1802 }
1803
1804 static inline void
1805 rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port,
1806         struct rte_sched_subport *subport,
1807         uint32_t qindex,
1808         struct rte_mbuf *pkt,
1809         __rte_unused uint32_t n_pkts_cman_dropped)
1810 {
1811         uint32_t tc_index = rte_sched_port_pipe_tc(port, qindex);
1812         uint32_t pkt_len = pkt->pkt_len;
1813
1814         subport->stats.n_pkts_tc_dropped[tc_index] += 1;
1815         subport->stats.n_bytes_tc_dropped[tc_index] += pkt_len;
1816         subport->stats.n_pkts_cman_dropped[tc_index] += n_pkts_cman_dropped;
1817 }
1818
1819 static inline void
1820 rte_sched_port_update_queue_stats(struct rte_sched_subport *subport,
1821         uint32_t qindex,
1822         struct rte_mbuf *pkt)
1823 {
1824         struct rte_sched_queue_extra *qe = subport->queue_extra + qindex;
1825         uint32_t pkt_len = pkt->pkt_len;
1826
1827         qe->stats.n_pkts += 1;
1828         qe->stats.n_bytes += pkt_len;
1829 }
1830
1831 static inline void
1832 rte_sched_port_update_queue_stats_on_drop(struct rte_sched_subport *subport,
1833         uint32_t qindex,
1834         struct rte_mbuf *pkt,
1835         __rte_unused uint32_t n_pkts_cman_dropped)
1836 {
1837         struct rte_sched_queue_extra *qe = subport->queue_extra + qindex;
1838         uint32_t pkt_len = pkt->pkt_len;
1839
1840         qe->stats.n_pkts_dropped += 1;
1841         qe->stats.n_bytes_dropped += pkt_len;
1842 #ifdef RTE_SCHED_CMAN
1843         if (subport->cman_enabled)
1844                 qe->stats.n_pkts_cman_dropped += n_pkts_cman_dropped;
1845 #endif
1846 }
1847
1848 #endif /* RTE_SCHED_COLLECT_STATS */
1849
1850 #ifdef RTE_SCHED_CMAN
1851
1852 static inline int
1853 rte_sched_port_cman_drop(struct rte_sched_port *port,
1854         struct rte_sched_subport *subport,
1855         struct rte_mbuf *pkt,
1856         uint32_t qindex,
1857         uint16_t qlen)
1858 {
1859         if (!subport->cman_enabled)
1860                 return 0;
1861
1862         struct rte_sched_queue_extra *qe;
1863         uint32_t tc_index;
1864
1865         tc_index = rte_sched_port_pipe_tc(port, qindex);
1866         qe = subport->queue_extra + qindex;
1867
1868         /* RED */
1869         if (subport->cman == RTE_SCHED_CMAN_RED) {
1870                 struct rte_red_config *red_cfg;
1871                 struct rte_red *red;
1872                 enum rte_color color;
1873
1874                 color = rte_sched_port_pkt_read_color(pkt);
1875                 red_cfg = &subport->red_config[tc_index][color];
1876
1877                 if ((red_cfg->min_th | red_cfg->max_th) == 0)
1878                         return 0;
1879
1880                 red = &qe->red;
1881
1882                 return rte_red_enqueue(red_cfg, red, qlen, port->time);
1883         }
1884
1885         /* PIE */
1886         struct rte_pie_config *pie_cfg = &subport->pie_config[tc_index];
1887         struct rte_pie *pie = &qe->pie;
1888
1889         return rte_pie_enqueue(pie_cfg, pie, qlen, pkt->pkt_len, port->time_cpu_cycles);
1890 }
1891
1892 static inline void
1893 rte_sched_port_red_set_queue_empty_timestamp(struct rte_sched_port *port,
1894         struct rte_sched_subport *subport, uint32_t qindex)
1895 {
1896         if (subport->cman_enabled) {
1897                 struct rte_sched_queue_extra *qe = subport->queue_extra + qindex;
1898                 if (subport->cman == RTE_SCHED_CMAN_RED) {
1899                         struct rte_red *red = &qe->red;
1900
1901                         rte_red_mark_queue_empty(red, port->time);
1902                 }
1903         }
1904 }
1905
1906 static inline void
1907 rte_sched_port_pie_dequeue(struct rte_sched_subport *subport,
1908 uint32_t qindex, uint32_t pkt_len, uint64_t time) {
1909         if (subport->cman_enabled && subport->cman == RTE_SCHED_CMAN_PIE) {
1910                 struct rte_sched_queue_extra *qe = subport->queue_extra + qindex;
1911                 struct rte_pie *pie = &qe->pie;
1912
1913                 /* Update queue length */
1914                 pie->qlen -= 1;
1915                 pie->qlen_bytes -= pkt_len;
1916
1917                 rte_pie_dequeue(pie, pkt_len, time);
1918         }
1919 }
1920
1921 #else
1922
1923 static inline int rte_sched_port_cman_drop(struct rte_sched_port *port __rte_unused,
1924         struct rte_sched_subport *subport __rte_unused,
1925         struct rte_mbuf *pkt __rte_unused,
1926         uint32_t qindex __rte_unused,
1927         uint16_t qlen __rte_unused)
1928 {
1929         return 0;
1930 }
1931
1932 #define rte_sched_port_red_set_queue_empty_timestamp(port, subport, qindex)
1933
1934 static inline void
1935 rte_sched_port_pie_dequeue(struct rte_sched_subport *subport __rte_unused,
1936         uint32_t qindex __rte_unused,
1937         uint32_t pkt_len __rte_unused,
1938         uint64_t time __rte_unused) {
1939         /* do-nothing when RTE_SCHED_CMAN not defined */
1940 }
1941
1942 #endif /* RTE_SCHED_CMAN */
1943
1944 #ifdef RTE_SCHED_DEBUG
1945
1946 static inline void
1947 debug_check_queue_slab(struct rte_sched_subport *subport, uint32_t bmp_pos,
1948                        uint64_t bmp_slab)
1949 {
1950         uint64_t mask;
1951         uint32_t i, panic;
1952
1953         if (bmp_slab == 0)
1954                 rte_panic("Empty slab at position %u\n", bmp_pos);
1955
1956         panic = 0;
1957         for (i = 0, mask = 1; i < 64; i++, mask <<= 1) {
1958                 if (mask & bmp_slab) {
1959                         if (rte_sched_port_queue_is_empty(subport, bmp_pos + i)) {
1960                                 printf("Queue %u (slab offset %u) is empty\n", bmp_pos + i, i);
1961                                 panic = 1;
1962                         }
1963                 }
1964         }
1965
1966         if (panic)
1967                 rte_panic("Empty queues in slab 0x%" PRIx64 "starting at position %u\n",
1968                         bmp_slab, bmp_pos);
1969 }
1970
1971 #endif /* RTE_SCHED_DEBUG */
1972
1973 static inline struct rte_sched_subport *
1974 rte_sched_port_subport(struct rte_sched_port *port,
1975         struct rte_mbuf *pkt)
1976 {
1977         uint32_t queue_id = rte_mbuf_sched_queue_get(pkt);
1978         uint32_t subport_id = queue_id >> (port->n_pipes_per_subport_log2 + 4);
1979
1980         return port->subports[subport_id];
1981 }
1982
1983 static inline uint32_t
1984 rte_sched_port_enqueue_qptrs_prefetch0(struct rte_sched_subport *subport,
1985         struct rte_mbuf *pkt, uint32_t subport_qmask)
1986 {
1987         struct rte_sched_queue *q;
1988 #ifdef RTE_SCHED_COLLECT_STATS
1989         struct rte_sched_queue_extra *qe;
1990 #endif
1991         uint32_t qindex = rte_mbuf_sched_queue_get(pkt);
1992         uint32_t subport_queue_id = subport_qmask & qindex;
1993
1994         q = subport->queue + subport_queue_id;
1995         rte_prefetch0(q);
1996 #ifdef RTE_SCHED_COLLECT_STATS
1997         qe = subport->queue_extra + subport_queue_id;
1998         rte_prefetch0(qe);
1999 #endif
2000
2001         return subport_queue_id;
2002 }
2003
2004 static inline void
2005 rte_sched_port_enqueue_qwa_prefetch0(struct rte_sched_port *port,
2006         struct rte_sched_subport *subport,
2007         uint32_t qindex,
2008         struct rte_mbuf **qbase)
2009 {
2010         struct rte_sched_queue *q;
2011         struct rte_mbuf **q_qw;
2012         uint16_t qsize;
2013
2014         q = subport->queue + qindex;
2015         qsize = rte_sched_subport_pipe_qsize(port, subport, qindex);
2016         q_qw = qbase + (q->qw & (qsize - 1));
2017
2018         rte_prefetch0(q_qw);
2019         rte_bitmap_prefetch0(subport->bmp, qindex);
2020 }
2021
2022 static inline int
2023 rte_sched_port_enqueue_qwa(struct rte_sched_port *port,
2024         struct rte_sched_subport *subport,
2025         uint32_t qindex,
2026         struct rte_mbuf **qbase,
2027         struct rte_mbuf *pkt)
2028 {
2029         struct rte_sched_queue *q;
2030         uint16_t qsize;
2031         uint16_t qlen;
2032
2033         q = subport->queue + qindex;
2034         qsize = rte_sched_subport_pipe_qsize(port, subport, qindex);
2035         qlen = q->qw - q->qr;
2036
2037         /* Drop the packet (and update drop stats) when queue is full */
2038         if (unlikely(rte_sched_port_cman_drop(port, subport, pkt, qindex, qlen) ||
2039                      (qlen >= qsize))) {
2040                 rte_pktmbuf_free(pkt);
2041 #ifdef RTE_SCHED_COLLECT_STATS
2042                 rte_sched_port_update_subport_stats_on_drop(port, subport,
2043                         qindex, pkt, qlen < qsize);
2044                 rte_sched_port_update_queue_stats_on_drop(subport, qindex, pkt,
2045                         qlen < qsize);
2046 #endif
2047                 return 0;
2048         }
2049
2050         /* Enqueue packet */
2051         qbase[q->qw & (qsize - 1)] = pkt;
2052         q->qw++;
2053
2054         /* Activate queue in the subport bitmap */
2055         rte_bitmap_set(subport->bmp, qindex);
2056
2057         /* Statistics */
2058 #ifdef RTE_SCHED_COLLECT_STATS
2059         rte_sched_port_update_subport_stats(port, subport, qindex, pkt);
2060         rte_sched_port_update_queue_stats(subport, qindex, pkt);
2061 #endif
2062
2063         return 1;
2064 }
2065
2066
2067 /*
2068  * The enqueue function implements a 4-level pipeline with each stage
2069  * processing two different packets. The purpose of using a pipeline
2070  * is to hide the latency of prefetching the data structures. The
2071  * naming convention is presented in the diagram below:
2072  *
2073  *   p00  _______   p10  _______   p20  _______   p30  _______
2074  * ----->|       |----->|       |----->|       |----->|       |----->
2075  *       |   0   |      |   1   |      |   2   |      |   3   |
2076  * ----->|_______|----->|_______|----->|_______|----->|_______|----->
2077  *   p01            p11            p21            p31
2078  *
2079  */
2080 int
2081 rte_sched_port_enqueue(struct rte_sched_port *port, struct rte_mbuf **pkts,
2082                        uint32_t n_pkts)
2083 {
2084         struct rte_mbuf *pkt00, *pkt01, *pkt10, *pkt11, *pkt20, *pkt21,
2085                 *pkt30, *pkt31, *pkt_last;
2086         struct rte_mbuf **q00_base, **q01_base, **q10_base, **q11_base,
2087                 **q20_base, **q21_base, **q30_base, **q31_base, **q_last_base;
2088         struct rte_sched_subport *subport00, *subport01, *subport10, *subport11,
2089                 *subport20, *subport21, *subport30, *subport31, *subport_last;
2090         uint32_t q00, q01, q10, q11, q20, q21, q30, q31, q_last;
2091         uint32_t r00, r01, r10, r11, r20, r21, r30, r31, r_last;
2092         uint32_t subport_qmask;
2093         uint32_t result, i;
2094
2095         result = 0;
2096         subport_qmask = (1 << (port->n_pipes_per_subport_log2 + 4)) - 1;
2097
2098         /*
2099          * Less then 6 input packets available, which is not enough to
2100          * feed the pipeline
2101          */
2102         if (unlikely(n_pkts < 6)) {
2103                 struct rte_sched_subport *subports[5];
2104                 struct rte_mbuf **q_base[5];
2105                 uint32_t q[5];
2106
2107                 /* Prefetch the mbuf structure of each packet */
2108                 for (i = 0; i < n_pkts; i++)
2109                         rte_prefetch0(pkts[i]);
2110
2111                 /* Prefetch the subport structure for each packet */
2112                 for (i = 0; i < n_pkts; i++)
2113                         subports[i] = rte_sched_port_subport(port, pkts[i]);
2114
2115                 /* Prefetch the queue structure for each queue */
2116                 for (i = 0; i < n_pkts; i++)
2117                         q[i] = rte_sched_port_enqueue_qptrs_prefetch0(subports[i],
2118                                         pkts[i], subport_qmask);
2119
2120                 /* Prefetch the write pointer location of each queue */
2121                 for (i = 0; i < n_pkts; i++) {
2122                         q_base[i] = rte_sched_subport_pipe_qbase(subports[i], q[i]);
2123                         rte_sched_port_enqueue_qwa_prefetch0(port, subports[i],
2124                                 q[i], q_base[i]);
2125                 }
2126
2127                 /* Write each packet to its queue */
2128                 for (i = 0; i < n_pkts; i++)
2129                         result += rte_sched_port_enqueue_qwa(port, subports[i],
2130                                                 q[i], q_base[i], pkts[i]);
2131
2132                 return result;
2133         }
2134
2135         /* Feed the first 3 stages of the pipeline (6 packets needed) */
2136         pkt20 = pkts[0];
2137         pkt21 = pkts[1];
2138         rte_prefetch0(pkt20);
2139         rte_prefetch0(pkt21);
2140
2141         pkt10 = pkts[2];
2142         pkt11 = pkts[3];
2143         rte_prefetch0(pkt10);
2144         rte_prefetch0(pkt11);
2145
2146         subport20 = rte_sched_port_subport(port, pkt20);
2147         subport21 = rte_sched_port_subport(port, pkt21);
2148         q20 = rte_sched_port_enqueue_qptrs_prefetch0(subport20,
2149                         pkt20, subport_qmask);
2150         q21 = rte_sched_port_enqueue_qptrs_prefetch0(subport21,
2151                         pkt21, subport_qmask);
2152
2153         pkt00 = pkts[4];
2154         pkt01 = pkts[5];
2155         rte_prefetch0(pkt00);
2156         rte_prefetch0(pkt01);
2157
2158         subport10 = rte_sched_port_subport(port, pkt10);
2159         subport11 = rte_sched_port_subport(port, pkt11);
2160         q10 = rte_sched_port_enqueue_qptrs_prefetch0(subport10,
2161                         pkt10, subport_qmask);
2162         q11 = rte_sched_port_enqueue_qptrs_prefetch0(subport11,
2163                         pkt11, subport_qmask);
2164
2165         q20_base = rte_sched_subport_pipe_qbase(subport20, q20);
2166         q21_base = rte_sched_subport_pipe_qbase(subport21, q21);
2167         rte_sched_port_enqueue_qwa_prefetch0(port, subport20, q20, q20_base);
2168         rte_sched_port_enqueue_qwa_prefetch0(port, subport21, q21, q21_base);
2169
2170         /* Run the pipeline */
2171         for (i = 6; i < (n_pkts & (~1)); i += 2) {
2172                 /* Propagate stage inputs */
2173                 pkt30 = pkt20;
2174                 pkt31 = pkt21;
2175                 pkt20 = pkt10;
2176                 pkt21 = pkt11;
2177                 pkt10 = pkt00;
2178                 pkt11 = pkt01;
2179                 q30 = q20;
2180                 q31 = q21;
2181                 q20 = q10;
2182                 q21 = q11;
2183                 subport30 = subport20;
2184                 subport31 = subport21;
2185                 subport20 = subport10;
2186                 subport21 = subport11;
2187                 q30_base = q20_base;
2188                 q31_base = q21_base;
2189
2190                 /* Stage 0: Get packets in */
2191                 pkt00 = pkts[i];
2192                 pkt01 = pkts[i + 1];
2193                 rte_prefetch0(pkt00);
2194                 rte_prefetch0(pkt01);
2195
2196                 /* Stage 1: Prefetch subport and queue structure storing queue pointers */
2197                 subport10 = rte_sched_port_subport(port, pkt10);
2198                 subport11 = rte_sched_port_subport(port, pkt11);
2199                 q10 = rte_sched_port_enqueue_qptrs_prefetch0(subport10,
2200                                 pkt10, subport_qmask);
2201                 q11 = rte_sched_port_enqueue_qptrs_prefetch0(subport11,
2202                                 pkt11, subport_qmask);
2203
2204                 /* Stage 2: Prefetch queue write location */
2205                 q20_base = rte_sched_subport_pipe_qbase(subport20, q20);
2206                 q21_base = rte_sched_subport_pipe_qbase(subport21, q21);
2207                 rte_sched_port_enqueue_qwa_prefetch0(port, subport20, q20, q20_base);
2208                 rte_sched_port_enqueue_qwa_prefetch0(port, subport21, q21, q21_base);
2209
2210                 /* Stage 3: Write packet to queue and activate queue */
2211                 r30 = rte_sched_port_enqueue_qwa(port, subport30,
2212                                 q30, q30_base, pkt30);
2213                 r31 = rte_sched_port_enqueue_qwa(port, subport31,
2214                                 q31, q31_base, pkt31);
2215                 result += r30 + r31;
2216         }
2217
2218         /*
2219          * Drain the pipeline (exactly 6 packets).
2220          * Handle the last packet in the case
2221          * of an odd number of input packets.
2222          */
2223         pkt_last = pkts[n_pkts - 1];
2224         rte_prefetch0(pkt_last);
2225
2226         subport00 = rte_sched_port_subport(port, pkt00);
2227         subport01 = rte_sched_port_subport(port, pkt01);
2228         q00 = rte_sched_port_enqueue_qptrs_prefetch0(subport00,
2229                         pkt00, subport_qmask);
2230         q01 = rte_sched_port_enqueue_qptrs_prefetch0(subport01,
2231                         pkt01, subport_qmask);
2232
2233         q10_base = rte_sched_subport_pipe_qbase(subport10, q10);
2234         q11_base = rte_sched_subport_pipe_qbase(subport11, q11);
2235         rte_sched_port_enqueue_qwa_prefetch0(port, subport10, q10, q10_base);
2236         rte_sched_port_enqueue_qwa_prefetch0(port, subport11, q11, q11_base);
2237
2238         r20 = rte_sched_port_enqueue_qwa(port, subport20,
2239                         q20, q20_base, pkt20);
2240         r21 = rte_sched_port_enqueue_qwa(port, subport21,
2241                         q21, q21_base, pkt21);
2242         result += r20 + r21;
2243
2244         subport_last = rte_sched_port_subport(port, pkt_last);
2245         q_last = rte_sched_port_enqueue_qptrs_prefetch0(subport_last,
2246                                 pkt_last, subport_qmask);
2247
2248         q00_base = rte_sched_subport_pipe_qbase(subport00, q00);
2249         q01_base = rte_sched_subport_pipe_qbase(subport01, q01);
2250         rte_sched_port_enqueue_qwa_prefetch0(port, subport00, q00, q00_base);
2251         rte_sched_port_enqueue_qwa_prefetch0(port, subport01, q01, q01_base);
2252
2253         r10 = rte_sched_port_enqueue_qwa(port, subport10, q10,
2254                         q10_base, pkt10);
2255         r11 = rte_sched_port_enqueue_qwa(port, subport11, q11,
2256                         q11_base, pkt11);
2257         result += r10 + r11;
2258
2259         q_last_base = rte_sched_subport_pipe_qbase(subport_last, q_last);
2260         rte_sched_port_enqueue_qwa_prefetch0(port, subport_last,
2261                 q_last, q_last_base);
2262
2263         r00 = rte_sched_port_enqueue_qwa(port, subport00, q00,
2264                         q00_base, pkt00);
2265         r01 = rte_sched_port_enqueue_qwa(port, subport01, q01,
2266                         q01_base, pkt01);
2267         result += r00 + r01;
2268
2269         if (n_pkts & 1) {
2270                 r_last = rte_sched_port_enqueue_qwa(port, subport_last,
2271                                         q_last, q_last_base, pkt_last);
2272                 result += r_last;
2273         }
2274
2275         return result;
2276 }
2277
2278 #ifndef RTE_SCHED_SUBPORT_TC_OV
2279
2280 static inline void
2281 grinder_credits_update(struct rte_sched_port *port,
2282         struct rte_sched_subport *subport, uint32_t pos)
2283 {
2284         struct rte_sched_grinder *grinder = subport->grinder + pos;
2285         struct rte_sched_pipe *pipe = grinder->pipe;
2286         struct rte_sched_pipe_profile *params = grinder->pipe_params;
2287         struct rte_sched_subport_profile *sp = grinder->subport_params;
2288         uint64_t n_periods;
2289         uint32_t i;
2290
2291         /* Subport TB */
2292         n_periods = (port->time - subport->tb_time) / sp->tb_period;
2293         subport->tb_credits += n_periods * sp->tb_credits_per_period;
2294         subport->tb_credits = RTE_MIN(subport->tb_credits, sp->tb_size);
2295         subport->tb_time += n_periods * sp->tb_period;
2296
2297         /* Pipe TB */
2298         n_periods = (port->time - pipe->tb_time) / params->tb_period;
2299         pipe->tb_credits += n_periods * params->tb_credits_per_period;
2300         pipe->tb_credits = RTE_MIN(pipe->tb_credits, params->tb_size);
2301         pipe->tb_time += n_periods * params->tb_period;
2302
2303         /* Subport TCs */
2304         if (unlikely(port->time >= subport->tc_time)) {
2305                 for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
2306                         subport->tc_credits[i] = sp->tc_credits_per_period[i];
2307
2308                 subport->tc_time = port->time + sp->tc_period;
2309         }
2310
2311         /* Pipe TCs */
2312         if (unlikely(port->time >= pipe->tc_time)) {
2313                 for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
2314                         pipe->tc_credits[i] = params->tc_credits_per_period[i];
2315
2316                 pipe->tc_time = port->time + params->tc_period;
2317         }
2318 }
2319
2320 #else
2321
2322 static inline uint64_t
2323 grinder_tc_ov_credits_update(struct rte_sched_port *port,
2324         struct rte_sched_subport *subport, uint32_t pos)
2325 {
2326         struct rte_sched_grinder *grinder = subport->grinder + pos;
2327         struct rte_sched_subport_profile *sp = grinder->subport_params;
2328         uint64_t tc_ov_consumption[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
2329         uint64_t tc_consumption = 0, tc_ov_consumption_max;
2330         uint64_t tc_ov_wm = subport->tc_ov_wm;
2331         uint32_t i;
2332
2333         if (subport->tc_ov == 0)
2334                 return subport->tc_ov_wm_max;
2335
2336         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASS_BE; i++) {
2337                 tc_ov_consumption[i] = sp->tc_credits_per_period[i]
2338                                         -  subport->tc_credits[i];
2339                 tc_consumption += tc_ov_consumption[i];
2340         }
2341
2342         tc_ov_consumption[RTE_SCHED_TRAFFIC_CLASS_BE] =
2343         sp->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE] -
2344                 subport->tc_credits[RTE_SCHED_TRAFFIC_CLASS_BE];
2345
2346         tc_ov_consumption_max =
2347         sp->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE] -
2348                         tc_consumption;
2349
2350         if (tc_ov_consumption[RTE_SCHED_TRAFFIC_CLASS_BE] >
2351                 (tc_ov_consumption_max - port->mtu)) {
2352                 tc_ov_wm  -= tc_ov_wm >> 7;
2353                 if (tc_ov_wm < subport->tc_ov_wm_min)
2354                         tc_ov_wm = subport->tc_ov_wm_min;
2355
2356                 return tc_ov_wm;
2357         }
2358
2359         tc_ov_wm += (tc_ov_wm >> 7) + 1;
2360         if (tc_ov_wm > subport->tc_ov_wm_max)
2361                 tc_ov_wm = subport->tc_ov_wm_max;
2362
2363         return tc_ov_wm;
2364 }
2365
2366 static inline void
2367 grinder_credits_update(struct rte_sched_port *port,
2368         struct rte_sched_subport *subport, uint32_t pos)
2369 {
2370         struct rte_sched_grinder *grinder = subport->grinder + pos;
2371         struct rte_sched_pipe *pipe = grinder->pipe;
2372         struct rte_sched_pipe_profile *params = grinder->pipe_params;
2373         struct rte_sched_subport_profile *sp = grinder->subport_params;
2374         uint64_t n_periods;
2375         uint32_t i;
2376
2377         /* Subport TB */
2378         n_periods = (port->time - subport->tb_time) / sp->tb_period;
2379         subport->tb_credits += n_periods * sp->tb_credits_per_period;
2380         subport->tb_credits = RTE_MIN(subport->tb_credits, sp->tb_size);
2381         subport->tb_time += n_periods * sp->tb_period;
2382
2383         /* Pipe TB */
2384         n_periods = (port->time - pipe->tb_time) / params->tb_period;
2385         pipe->tb_credits += n_periods * params->tb_credits_per_period;
2386         pipe->tb_credits = RTE_MIN(pipe->tb_credits, params->tb_size);
2387         pipe->tb_time += n_periods * params->tb_period;
2388
2389         /* Subport TCs */
2390         if (unlikely(port->time >= subport->tc_time)) {
2391                 subport->tc_ov_wm =
2392                         grinder_tc_ov_credits_update(port, subport, pos);
2393
2394                 for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
2395                         subport->tc_credits[i] = sp->tc_credits_per_period[i];
2396
2397                 subport->tc_time = port->time + sp->tc_period;
2398                 subport->tc_ov_period_id++;
2399         }
2400
2401         /* Pipe TCs */
2402         if (unlikely(port->time >= pipe->tc_time)) {
2403                 for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
2404                         pipe->tc_credits[i] = params->tc_credits_per_period[i];
2405                 pipe->tc_time = port->time + params->tc_period;
2406         }
2407
2408         /* Pipe TCs - Oversubscription */
2409         if (unlikely(pipe->tc_ov_period_id != subport->tc_ov_period_id)) {
2410                 pipe->tc_ov_credits = subport->tc_ov_wm * params->tc_ov_weight;
2411
2412                 pipe->tc_ov_period_id = subport->tc_ov_period_id;
2413         }
2414 }
2415
2416 #endif /* RTE_SCHED_TS_CREDITS_UPDATE, RTE_SCHED_SUBPORT_TC_OV */
2417
2418
2419 #ifndef RTE_SCHED_SUBPORT_TC_OV
2420
2421 static inline int
2422 grinder_credits_check(struct rte_sched_port *port,
2423         struct rte_sched_subport *subport, uint32_t pos)
2424 {
2425         struct rte_sched_grinder *grinder = subport->grinder + pos;
2426         struct rte_sched_pipe *pipe = grinder->pipe;
2427         struct rte_mbuf *pkt = grinder->pkt;
2428         uint32_t tc_index = grinder->tc_index;
2429         uint64_t pkt_len = pkt->pkt_len + port->frame_overhead;
2430         uint64_t subport_tb_credits = subport->tb_credits;
2431         uint64_t subport_tc_credits = subport->tc_credits[tc_index];
2432         uint64_t pipe_tb_credits = pipe->tb_credits;
2433         uint64_t pipe_tc_credits = pipe->tc_credits[tc_index];
2434         int enough_credits;
2435
2436         /* Check queue credits */
2437         enough_credits = (pkt_len <= subport_tb_credits) &&
2438                 (pkt_len <= subport_tc_credits) &&
2439                 (pkt_len <= pipe_tb_credits) &&
2440                 (pkt_len <= pipe_tc_credits);
2441
2442         if (!enough_credits)
2443                 return 0;
2444
2445         /* Update port credits */
2446         subport->tb_credits -= pkt_len;
2447         subport->tc_credits[tc_index] -= pkt_len;
2448         pipe->tb_credits -= pkt_len;
2449         pipe->tc_credits[tc_index] -= pkt_len;
2450
2451         return 1;
2452 }
2453
2454 #else
2455
2456 static inline int
2457 grinder_credits_check(struct rte_sched_port *port,
2458         struct rte_sched_subport *subport, uint32_t pos)
2459 {
2460         struct rte_sched_grinder *grinder = subport->grinder + pos;
2461         struct rte_sched_pipe *pipe = grinder->pipe;
2462         struct rte_mbuf *pkt = grinder->pkt;
2463         uint32_t tc_index = grinder->tc_index;
2464         uint64_t pkt_len = pkt->pkt_len + port->frame_overhead;
2465         uint64_t subport_tb_credits = subport->tb_credits;
2466         uint64_t subport_tc_credits = subport->tc_credits[tc_index];
2467         uint64_t pipe_tb_credits = pipe->tb_credits;
2468         uint64_t pipe_tc_credits = pipe->tc_credits[tc_index];
2469         uint64_t pipe_tc_ov_mask1[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
2470         uint64_t pipe_tc_ov_mask2[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE] = {0};
2471         uint64_t pipe_tc_ov_credits;
2472         uint32_t i;
2473         int enough_credits;
2474
2475         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
2476                 pipe_tc_ov_mask1[i] = ~0LLU;
2477
2478         pipe_tc_ov_mask1[RTE_SCHED_TRAFFIC_CLASS_BE] = pipe->tc_ov_credits;
2479         pipe_tc_ov_mask2[RTE_SCHED_TRAFFIC_CLASS_BE] = ~0LLU;
2480         pipe_tc_ov_credits = pipe_tc_ov_mask1[tc_index];
2481
2482         /* Check pipe and subport credits */
2483         enough_credits = (pkt_len <= subport_tb_credits) &&
2484                 (pkt_len <= subport_tc_credits) &&
2485                 (pkt_len <= pipe_tb_credits) &&
2486                 (pkt_len <= pipe_tc_credits) &&
2487                 (pkt_len <= pipe_tc_ov_credits);
2488
2489         if (!enough_credits)
2490                 return 0;
2491
2492         /* Update pipe and subport credits */
2493         subport->tb_credits -= pkt_len;
2494         subport->tc_credits[tc_index] -= pkt_len;
2495         pipe->tb_credits -= pkt_len;
2496         pipe->tc_credits[tc_index] -= pkt_len;
2497         pipe->tc_ov_credits -= pipe_tc_ov_mask2[tc_index] & pkt_len;
2498
2499         return 1;
2500 }
2501
2502 #endif /* RTE_SCHED_SUBPORT_TC_OV */
2503
2504
2505 static inline int
2506 grinder_schedule(struct rte_sched_port *port,
2507         struct rte_sched_subport *subport, uint32_t pos)
2508 {
2509         struct rte_sched_grinder *grinder = subport->grinder + pos;
2510         struct rte_sched_queue *queue = grinder->queue[grinder->qpos];
2511         uint32_t qindex = grinder->qindex[grinder->qpos];
2512         struct rte_mbuf *pkt = grinder->pkt;
2513         uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
2514         uint32_t be_tc_active;
2515
2516         if (!grinder_credits_check(port, subport, pos))
2517                 return 0;
2518
2519         /* Advance port time */
2520         port->time += pkt_len;
2521
2522         /* Send packet */
2523         port->pkts_out[port->n_pkts_out++] = pkt;
2524         queue->qr++;
2525
2526         be_tc_active = (grinder->tc_index == RTE_SCHED_TRAFFIC_CLASS_BE) ? ~0x0 : 0x0;
2527         grinder->wrr_tokens[grinder->qpos] +=
2528                 (pkt_len * grinder->wrr_cost[grinder->qpos]) & be_tc_active;
2529
2530         if (queue->qr == queue->qw) {
2531                 rte_bitmap_clear(subport->bmp, qindex);
2532                 grinder->qmask &= ~(1 << grinder->qpos);
2533                 if (be_tc_active)
2534                         grinder->wrr_mask[grinder->qpos] = 0;
2535
2536                 rte_sched_port_red_set_queue_empty_timestamp(port, subport, qindex);
2537         }
2538
2539         rte_sched_port_pie_dequeue(subport, qindex, pkt_len, port->time_cpu_cycles);
2540
2541         /* Reset pipe loop detection */
2542         subport->pipe_loop = RTE_SCHED_PIPE_INVALID;
2543         grinder->productive = 1;
2544
2545         return 1;
2546 }
2547
2548 #ifdef SCHED_VECTOR_SSE4
2549
2550 static inline int
2551 grinder_pipe_exists(struct rte_sched_subport *subport, uint32_t base_pipe)
2552 {
2553         __m128i index = _mm_set1_epi32(base_pipe);
2554         __m128i pipes = _mm_load_si128((__m128i *)subport->grinder_base_bmp_pos);
2555         __m128i res = _mm_cmpeq_epi32(pipes, index);
2556
2557         pipes = _mm_load_si128((__m128i *)(subport->grinder_base_bmp_pos + 4));
2558         pipes = _mm_cmpeq_epi32(pipes, index);
2559         res = _mm_or_si128(res, pipes);
2560
2561         if (_mm_testz_si128(res, res))
2562                 return 0;
2563
2564         return 1;
2565 }
2566
2567 #elif defined(SCHED_VECTOR_NEON)
2568
2569 static inline int
2570 grinder_pipe_exists(struct rte_sched_subport *subport, uint32_t base_pipe)
2571 {
2572         uint32x4_t index, pipes;
2573         uint32_t *pos = (uint32_t *)subport->grinder_base_bmp_pos;
2574
2575         index = vmovq_n_u32(base_pipe);
2576         pipes = vld1q_u32(pos);
2577         if (!vminvq_u32(veorq_u32(pipes, index)))
2578                 return 1;
2579
2580         pipes = vld1q_u32(pos + 4);
2581         if (!vminvq_u32(veorq_u32(pipes, index)))
2582                 return 1;
2583
2584         return 0;
2585 }
2586
2587 #else
2588
2589 static inline int
2590 grinder_pipe_exists(struct rte_sched_subport *subport, uint32_t base_pipe)
2591 {
2592         uint32_t i;
2593
2594         for (i = 0; i < RTE_SCHED_PORT_N_GRINDERS; i++) {
2595                 if (subport->grinder_base_bmp_pos[i] == base_pipe)
2596                         return 1;
2597         }
2598
2599         return 0;
2600 }
2601
2602 #endif /* RTE_SCHED_OPTIMIZATIONS */
2603
2604 static inline void
2605 grinder_pcache_populate(struct rte_sched_subport *subport,
2606         uint32_t pos, uint32_t bmp_pos, uint64_t bmp_slab)
2607 {
2608         struct rte_sched_grinder *grinder = subport->grinder + pos;
2609         uint16_t w[4];
2610
2611         grinder->pcache_w = 0;
2612         grinder->pcache_r = 0;
2613
2614         w[0] = (uint16_t) bmp_slab;
2615         w[1] = (uint16_t) (bmp_slab >> 16);
2616         w[2] = (uint16_t) (bmp_slab >> 32);
2617         w[3] = (uint16_t) (bmp_slab >> 48);
2618
2619         grinder->pcache_qmask[grinder->pcache_w] = w[0];
2620         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos;
2621         grinder->pcache_w += (w[0] != 0);
2622
2623         grinder->pcache_qmask[grinder->pcache_w] = w[1];
2624         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos + 16;
2625         grinder->pcache_w += (w[1] != 0);
2626
2627         grinder->pcache_qmask[grinder->pcache_w] = w[2];
2628         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos + 32;
2629         grinder->pcache_w += (w[2] != 0);
2630
2631         grinder->pcache_qmask[grinder->pcache_w] = w[3];
2632         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos + 48;
2633         grinder->pcache_w += (w[3] != 0);
2634 }
2635
2636 static inline void
2637 grinder_tccache_populate(struct rte_sched_subport *subport,
2638         uint32_t pos, uint32_t qindex, uint16_t qmask)
2639 {
2640         struct rte_sched_grinder *grinder = subport->grinder + pos;
2641         uint8_t b, i;
2642
2643         grinder->tccache_w = 0;
2644         grinder->tccache_r = 0;
2645
2646         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASS_BE; i++) {
2647                 b = (uint8_t) ((qmask >> i) & 0x1);
2648                 grinder->tccache_qmask[grinder->tccache_w] = b;
2649                 grinder->tccache_qindex[grinder->tccache_w] = qindex + i;
2650                 grinder->tccache_w += (b != 0);
2651         }
2652
2653         b = (uint8_t) (qmask >> (RTE_SCHED_TRAFFIC_CLASS_BE));
2654         grinder->tccache_qmask[grinder->tccache_w] = b;
2655         grinder->tccache_qindex[grinder->tccache_w] = qindex +
2656                 RTE_SCHED_TRAFFIC_CLASS_BE;
2657         grinder->tccache_w += (b != 0);
2658 }
2659
2660 static inline int
2661 grinder_next_tc(struct rte_sched_port *port,
2662         struct rte_sched_subport *subport, uint32_t pos)
2663 {
2664         struct rte_sched_grinder *grinder = subport->grinder + pos;
2665         struct rte_mbuf **qbase;
2666         uint32_t qindex;
2667         uint16_t qsize;
2668
2669         if (grinder->tccache_r == grinder->tccache_w)
2670                 return 0;
2671
2672         qindex = grinder->tccache_qindex[grinder->tccache_r];
2673         qbase = rte_sched_subport_pipe_qbase(subport, qindex);
2674         qsize = rte_sched_subport_pipe_qsize(port, subport, qindex);
2675
2676         grinder->tc_index = rte_sched_port_pipe_tc(port, qindex);
2677         grinder->qmask = grinder->tccache_qmask[grinder->tccache_r];
2678         grinder->qsize = qsize;
2679
2680         if (grinder->tc_index < RTE_SCHED_TRAFFIC_CLASS_BE) {
2681                 grinder->queue[0] = subport->queue + qindex;
2682                 grinder->qbase[0] = qbase;
2683                 grinder->qindex[0] = qindex;
2684                 grinder->tccache_r++;
2685
2686                 return 1;
2687         }
2688
2689         grinder->queue[0] = subport->queue + qindex;
2690         grinder->queue[1] = subport->queue + qindex + 1;
2691         grinder->queue[2] = subport->queue + qindex + 2;
2692         grinder->queue[3] = subport->queue + qindex + 3;
2693
2694         grinder->qbase[0] = qbase;
2695         grinder->qbase[1] = qbase + qsize;
2696         grinder->qbase[2] = qbase + 2 * qsize;
2697         grinder->qbase[3] = qbase + 3 * qsize;
2698
2699         grinder->qindex[0] = qindex;
2700         grinder->qindex[1] = qindex + 1;
2701         grinder->qindex[2] = qindex + 2;
2702         grinder->qindex[3] = qindex + 3;
2703
2704         grinder->tccache_r++;
2705         return 1;
2706 }
2707
2708 static inline int
2709 grinder_next_pipe(struct rte_sched_port *port,
2710         struct rte_sched_subport *subport, uint32_t pos)
2711 {
2712         struct rte_sched_grinder *grinder = subport->grinder + pos;
2713         uint32_t pipe_qindex;
2714         uint16_t pipe_qmask;
2715
2716         if (grinder->pcache_r < grinder->pcache_w) {
2717                 pipe_qmask = grinder->pcache_qmask[grinder->pcache_r];
2718                 pipe_qindex = grinder->pcache_qindex[grinder->pcache_r];
2719                 grinder->pcache_r++;
2720         } else {
2721                 uint64_t bmp_slab = 0;
2722                 uint32_t bmp_pos = 0;
2723
2724                 /* Get another non-empty pipe group */
2725                 if (unlikely(rte_bitmap_scan(subport->bmp, &bmp_pos, &bmp_slab) <= 0))
2726                         return 0;
2727
2728 #ifdef RTE_SCHED_DEBUG
2729                 debug_check_queue_slab(subport, bmp_pos, bmp_slab);
2730 #endif
2731
2732                 /* Return if pipe group already in one of the other grinders */
2733                 subport->grinder_base_bmp_pos[pos] = RTE_SCHED_BMP_POS_INVALID;
2734                 if (unlikely(grinder_pipe_exists(subport, bmp_pos)))
2735                         return 0;
2736
2737                 subport->grinder_base_bmp_pos[pos] = bmp_pos;
2738
2739                 /* Install new pipe group into grinder's pipe cache */
2740                 grinder_pcache_populate(subport, pos, bmp_pos, bmp_slab);
2741
2742                 pipe_qmask = grinder->pcache_qmask[0];
2743                 pipe_qindex = grinder->pcache_qindex[0];
2744                 grinder->pcache_r = 1;
2745         }
2746
2747         /* Install new pipe in the grinder */
2748         grinder->pindex = pipe_qindex >> 4;
2749         grinder->subport = subport;
2750         grinder->pipe = subport->pipe + grinder->pindex;
2751         grinder->pipe_params = NULL; /* to be set after the pipe structure is prefetched */
2752         grinder->productive = 0;
2753
2754         grinder_tccache_populate(subport, pos, pipe_qindex, pipe_qmask);
2755         grinder_next_tc(port, subport, pos);
2756
2757         /* Check for pipe exhaustion */
2758         if (grinder->pindex == subport->pipe_loop) {
2759                 subport->pipe_exhaustion = 1;
2760                 subport->pipe_loop = RTE_SCHED_PIPE_INVALID;
2761         }
2762
2763         return 1;
2764 }
2765
2766
2767 static inline void
2768 grinder_wrr_load(struct rte_sched_subport *subport, uint32_t pos)
2769 {
2770         struct rte_sched_grinder *grinder = subport->grinder + pos;
2771         struct rte_sched_pipe *pipe = grinder->pipe;
2772         struct rte_sched_pipe_profile *pipe_params = grinder->pipe_params;
2773         uint32_t qmask = grinder->qmask;
2774
2775         grinder->wrr_tokens[0] =
2776                 ((uint16_t) pipe->wrr_tokens[0]) << RTE_SCHED_WRR_SHIFT;
2777         grinder->wrr_tokens[1] =
2778                 ((uint16_t) pipe->wrr_tokens[1]) << RTE_SCHED_WRR_SHIFT;
2779         grinder->wrr_tokens[2] =
2780                 ((uint16_t) pipe->wrr_tokens[2]) << RTE_SCHED_WRR_SHIFT;
2781         grinder->wrr_tokens[3] =
2782                 ((uint16_t) pipe->wrr_tokens[3]) << RTE_SCHED_WRR_SHIFT;
2783
2784         grinder->wrr_mask[0] = (qmask & 0x1) * 0xFFFF;
2785         grinder->wrr_mask[1] = ((qmask >> 1) & 0x1) * 0xFFFF;
2786         grinder->wrr_mask[2] = ((qmask >> 2) & 0x1) * 0xFFFF;
2787         grinder->wrr_mask[3] = ((qmask >> 3) & 0x1) * 0xFFFF;
2788
2789         grinder->wrr_cost[0] = pipe_params->wrr_cost[0];
2790         grinder->wrr_cost[1] = pipe_params->wrr_cost[1];
2791         grinder->wrr_cost[2] = pipe_params->wrr_cost[2];
2792         grinder->wrr_cost[3] = pipe_params->wrr_cost[3];
2793 }
2794
2795 static inline void
2796 grinder_wrr_store(struct rte_sched_subport *subport, uint32_t pos)
2797 {
2798         struct rte_sched_grinder *grinder = subport->grinder + pos;
2799         struct rte_sched_pipe *pipe = grinder->pipe;
2800
2801         pipe->wrr_tokens[0] =
2802                         (grinder->wrr_tokens[0] & grinder->wrr_mask[0]) >>
2803                                 RTE_SCHED_WRR_SHIFT;
2804         pipe->wrr_tokens[1] =
2805                         (grinder->wrr_tokens[1] & grinder->wrr_mask[1]) >>
2806                                 RTE_SCHED_WRR_SHIFT;
2807         pipe->wrr_tokens[2] =
2808                         (grinder->wrr_tokens[2] & grinder->wrr_mask[2]) >>
2809                                 RTE_SCHED_WRR_SHIFT;
2810         pipe->wrr_tokens[3] =
2811                         (grinder->wrr_tokens[3] & grinder->wrr_mask[3]) >>
2812                                 RTE_SCHED_WRR_SHIFT;
2813 }
2814
2815 static inline void
2816 grinder_wrr(struct rte_sched_subport *subport, uint32_t pos)
2817 {
2818         struct rte_sched_grinder *grinder = subport->grinder + pos;
2819         uint16_t wrr_tokens_min;
2820
2821         grinder->wrr_tokens[0] |= ~grinder->wrr_mask[0];
2822         grinder->wrr_tokens[1] |= ~grinder->wrr_mask[1];
2823         grinder->wrr_tokens[2] |= ~grinder->wrr_mask[2];
2824         grinder->wrr_tokens[3] |= ~grinder->wrr_mask[3];
2825
2826         grinder->qpos = rte_min_pos_4_u16(grinder->wrr_tokens);
2827         wrr_tokens_min = grinder->wrr_tokens[grinder->qpos];
2828
2829         grinder->wrr_tokens[0] -= wrr_tokens_min;
2830         grinder->wrr_tokens[1] -= wrr_tokens_min;
2831         grinder->wrr_tokens[2] -= wrr_tokens_min;
2832         grinder->wrr_tokens[3] -= wrr_tokens_min;
2833 }
2834
2835
2836 #define grinder_evict(subport, pos)
2837
2838 static inline void
2839 grinder_prefetch_pipe(struct rte_sched_subport *subport, uint32_t pos)
2840 {
2841         struct rte_sched_grinder *grinder = subport->grinder + pos;
2842
2843         rte_prefetch0(grinder->pipe);
2844         rte_prefetch0(grinder->queue[0]);
2845 }
2846
2847 static inline void
2848 grinder_prefetch_tc_queue_arrays(struct rte_sched_subport *subport, uint32_t pos)
2849 {
2850         struct rte_sched_grinder *grinder = subport->grinder + pos;
2851         uint16_t qsize, qr[RTE_SCHED_MAX_QUEUES_PER_TC];
2852
2853         qsize = grinder->qsize;
2854         grinder->qpos = 0;
2855
2856         if (grinder->tc_index < RTE_SCHED_TRAFFIC_CLASS_BE) {
2857                 qr[0] = grinder->queue[0]->qr & (qsize - 1);
2858
2859                 rte_prefetch0(grinder->qbase[0] + qr[0]);
2860                 return;
2861         }
2862
2863         qr[0] = grinder->queue[0]->qr & (qsize - 1);
2864         qr[1] = grinder->queue[1]->qr & (qsize - 1);
2865         qr[2] = grinder->queue[2]->qr & (qsize - 1);
2866         qr[3] = grinder->queue[3]->qr & (qsize - 1);
2867
2868         rte_prefetch0(grinder->qbase[0] + qr[0]);
2869         rte_prefetch0(grinder->qbase[1] + qr[1]);
2870
2871         grinder_wrr_load(subport, pos);
2872         grinder_wrr(subport, pos);
2873
2874         rte_prefetch0(grinder->qbase[2] + qr[2]);
2875         rte_prefetch0(grinder->qbase[3] + qr[3]);
2876 }
2877
2878 static inline void
2879 grinder_prefetch_mbuf(struct rte_sched_subport *subport, uint32_t pos)
2880 {
2881         struct rte_sched_grinder *grinder = subport->grinder + pos;
2882         uint32_t qpos = grinder->qpos;
2883         struct rte_mbuf **qbase = grinder->qbase[qpos];
2884         uint16_t qsize = grinder->qsize;
2885         uint16_t qr = grinder->queue[qpos]->qr & (qsize - 1);
2886
2887         grinder->pkt = qbase[qr];
2888         rte_prefetch0(grinder->pkt);
2889
2890         if (unlikely((qr & 0x7) == 7)) {
2891                 uint16_t qr_next = (grinder->queue[qpos]->qr + 1) & (qsize - 1);
2892
2893                 rte_prefetch0(qbase + qr_next);
2894         }
2895 }
2896
2897 static inline uint32_t
2898 grinder_handle(struct rte_sched_port *port,
2899         struct rte_sched_subport *subport, uint32_t pos)
2900 {
2901         struct rte_sched_grinder *grinder = subport->grinder + pos;
2902
2903         switch (grinder->state) {
2904         case e_GRINDER_PREFETCH_PIPE:
2905         {
2906                 if (grinder_next_pipe(port, subport, pos)) {
2907                         grinder_prefetch_pipe(subport, pos);
2908                         subport->busy_grinders++;
2909
2910                         grinder->state = e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS;
2911                         return 0;
2912                 }
2913
2914                 return 0;
2915         }
2916
2917         case e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS:
2918         {
2919                 struct rte_sched_pipe *pipe = grinder->pipe;
2920
2921                 grinder->pipe_params = subport->pipe_profiles + pipe->profile;
2922                 grinder->subport_params = port->subport_profiles +
2923                                                 subport->profile;
2924
2925                 grinder_prefetch_tc_queue_arrays(subport, pos);
2926                 grinder_credits_update(port, subport, pos);
2927
2928                 grinder->state = e_GRINDER_PREFETCH_MBUF;
2929                 return 0;
2930         }
2931
2932         case e_GRINDER_PREFETCH_MBUF:
2933         {
2934                 grinder_prefetch_mbuf(subport, pos);
2935
2936                 grinder->state = e_GRINDER_READ_MBUF;
2937                 return 0;
2938         }
2939
2940         case e_GRINDER_READ_MBUF:
2941         {
2942                 uint32_t wrr_active, result = 0;
2943
2944                 result = grinder_schedule(port, subport, pos);
2945
2946                 wrr_active = (grinder->tc_index == RTE_SCHED_TRAFFIC_CLASS_BE);
2947
2948                 /* Look for next packet within the same TC */
2949                 if (result && grinder->qmask) {
2950                         if (wrr_active)
2951                                 grinder_wrr(subport, pos);
2952
2953                         grinder_prefetch_mbuf(subport, pos);
2954
2955                         return 1;
2956                 }
2957
2958                 if (wrr_active)
2959                         grinder_wrr_store(subport, pos);
2960
2961                 /* Look for another active TC within same pipe */
2962                 if (grinder_next_tc(port, subport, pos)) {
2963                         grinder_prefetch_tc_queue_arrays(subport, pos);
2964
2965                         grinder->state = e_GRINDER_PREFETCH_MBUF;
2966                         return result;
2967                 }
2968
2969                 if (grinder->productive == 0 &&
2970                     subport->pipe_loop == RTE_SCHED_PIPE_INVALID)
2971                         subport->pipe_loop = grinder->pindex;
2972
2973                 grinder_evict(subport, pos);
2974
2975                 /* Look for another active pipe */
2976                 if (grinder_next_pipe(port, subport, pos)) {
2977                         grinder_prefetch_pipe(subport, pos);
2978
2979                         grinder->state = e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS;
2980                         return result;
2981                 }
2982
2983                 /* No active pipe found */
2984                 subport->busy_grinders--;
2985
2986                 grinder->state = e_GRINDER_PREFETCH_PIPE;
2987                 return result;
2988         }
2989
2990         default:
2991                 rte_panic("Algorithmic error (invalid state)\n");
2992                 return 0;
2993         }
2994 }
2995
2996 static inline void
2997 rte_sched_port_time_resync(struct rte_sched_port *port)
2998 {
2999         uint64_t cycles = rte_get_tsc_cycles();
3000         uint64_t cycles_diff;
3001         uint64_t bytes_diff;
3002         uint32_t i;
3003
3004         if (cycles < port->time_cpu_cycles)
3005                 port->time_cpu_cycles = 0;
3006
3007         cycles_diff = cycles - port->time_cpu_cycles;
3008         /* Compute elapsed time in bytes */
3009         bytes_diff = rte_reciprocal_divide(cycles_diff << RTE_SCHED_TIME_SHIFT,
3010                                            port->inv_cycles_per_byte);
3011
3012         /* Advance port time */
3013         port->time_cpu_cycles +=
3014                 (bytes_diff * port->cycles_per_byte) >> RTE_SCHED_TIME_SHIFT;
3015         port->time_cpu_bytes += bytes_diff;
3016         if (port->time < port->time_cpu_bytes)
3017                 port->time = port->time_cpu_bytes;
3018
3019         /* Reset pipe loop detection */
3020         for (i = 0; i < port->n_subports_per_port; i++)
3021                 port->subports[i]->pipe_loop = RTE_SCHED_PIPE_INVALID;
3022 }
3023
3024 static inline int
3025 rte_sched_port_exceptions(struct rte_sched_subport *subport, int second_pass)
3026 {
3027         int exceptions;
3028
3029         /* Check if any exception flag is set */
3030         exceptions = (second_pass && subport->busy_grinders == 0) ||
3031                 (subport->pipe_exhaustion == 1);
3032
3033         /* Clear exception flags */
3034         subport->pipe_exhaustion = 0;
3035
3036         return exceptions;
3037 }
3038
3039 int
3040 rte_sched_port_dequeue(struct rte_sched_port *port, struct rte_mbuf **pkts, uint32_t n_pkts)
3041 {
3042         struct rte_sched_subport *subport;
3043         uint32_t subport_id = port->subport_id;
3044         uint32_t i, n_subports = 0, count;
3045
3046         port->pkts_out = pkts;
3047         port->n_pkts_out = 0;
3048
3049         rte_sched_port_time_resync(port);
3050
3051         /* Take each queue in the grinder one step further */
3052         for (i = 0, count = 0; ; i++)  {
3053                 subport = port->subports[subport_id];
3054
3055                 count += grinder_handle(port, subport,
3056                                 i & (RTE_SCHED_PORT_N_GRINDERS - 1));
3057
3058                 if (count == n_pkts) {
3059                         subport_id++;
3060
3061                         if (subport_id == port->n_subports_per_port)
3062                                 subport_id = 0;
3063
3064                         port->subport_id = subport_id;
3065                         break;
3066                 }
3067
3068                 if (rte_sched_port_exceptions(subport, i >= RTE_SCHED_PORT_N_GRINDERS)) {
3069                         i = 0;
3070                         subport_id++;
3071                         n_subports++;
3072                 }
3073
3074                 if (subport_id == port->n_subports_per_port)
3075                         subport_id = 0;
3076
3077                 if (n_subports == port->n_subports_per_port) {
3078                         port->subport_id = subport_id;
3079                         break;
3080                 }
3081         }
3082
3083         return count;
3084 }