lib/librte_sched/rte_sched.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <string.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_log.h>
  10 #include <rte_memory.h>
  11 #include <rte_malloc.h>
  12 #include <rte_cycles.h>
  13 #include <rte_prefetch.h>
  14 #include <rte_branch_prediction.h>
  15 #include <rte_mbuf.h>
  16 #include <rte_bitmap.h>
  17 #include <rte_reciprocal.h>
  18
  19 #include "rte_sched.h"
  20 #include "rte_sched_common.h"
  21 #include "rte_approx.h"
  22
  23 #ifdef __INTEL_COMPILER
  24 #pragma warning(disable:2259) /* conversion may lose significant bits */
  25 #endif
  26
  27 #ifdef RTE_SCHED_VECTOR
  28 #include <rte_vect.h>
  29
  30 #ifdef RTE_ARCH_X86
  31 #define SCHED_VECTOR_SSE4
  32 #elif defined(RTE_MACHINE_CPUFLAG_NEON)
  33 #define SCHED_VECTOR_NEON
  34 #endif
  35
  36 #endif
  37
  38 #define RTE_SCHED_TB_RATE_CONFIG_ERR          (1e-7)
  39 #define RTE_SCHED_WRR_SHIFT                   3
  40 #define RTE_SCHED_MAX_QUEUES_PER_TC           RTE_SCHED_BE_QUEUES_PER_PIPE
  41 #define RTE_SCHED_GRINDER_PCACHE_SIZE         (64 / RTE_SCHED_QUEUES_PER_PIPE)
  42 #define RTE_SCHED_PIPE_INVALID                UINT32_MAX
  43 #define RTE_SCHED_BMP_POS_INVALID             UINT32_MAX
  44
  45 /* Scaling for cycles_per_byte calculation
  46  * Chosen so that minimum rate is 480 bit/sec
  47  */
  48 #define RTE_SCHED_TIME_SHIFT                  8
  49
  50 struct rte_sched_subport {
  51         /* Token bucket (TB) */
  52         uint64_t tb_time; /* time of last update */
  53         uint32_t tb_period;
  54         uint32_t tb_credits_per_period;
  55         uint32_t tb_size;
  56         uint32_t tb_credits;
  57
  58         /* Traffic classes (TCs) */
  59         uint64_t tc_time; /* time of next update */
  60         uint32_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
  61         uint32_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
  62         uint32_t tc_period;
  63
  64         /* TC oversubscription */
  65         uint32_t tc_ov_wm;
  66         uint32_t tc_ov_wm_min;
  67         uint32_t tc_ov_wm_max;
  68         uint8_t tc_ov_period_id;
  69         uint8_t tc_ov;
  70         uint32_t tc_ov_n;
  71         double tc_ov_rate;
  72
  73         /* Statistics */
  74         struct rte_sched_subport_stats stats;
  75 };
  76
  77 struct rte_sched_pipe_profile {
  78         /* Token bucket (TB) */
  79         uint32_t tb_period;
  80         uint32_t tb_credits_per_period;
  81         uint32_t tb_size;
  82
  83         /* Pipe traffic classes */
  84         uint32_t tc_period;
  85         uint32_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
  86         uint8_t tc_ov_weight;
  87
  88         /* Pipe best-effort traffic class queues */
  89         uint8_t  wrr_cost[RTE_SCHED_BE_QUEUES_PER_PIPE];
  90 };
  91
  92 struct rte_sched_pipe {
  93         /* Token bucket (TB) */
  94         uint64_t tb_time; /* time of last update */
  95         uint32_t tb_credits;
  96
  97         /* Pipe profile and flags */
  98         uint32_t profile;
  99
 100         /* Traffic classes (TCs) */
 101         uint64_t tc_time; /* time of next update */
 102         uint32_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 103
 104         /* Weighted Round Robin (WRR) */
 105         uint8_t wrr_tokens[RTE_SCHED_BE_QUEUES_PER_PIPE];
 106
 107         /* TC oversubscription */
 108         uint32_t tc_ov_credits;
 109         uint8_t tc_ov_period_id;
 110         uint8_t reserved[3];
 111 } __rte_cache_aligned;
 112
 113 struct rte_sched_queue {
 114         uint16_t qw;
 115         uint16_t qr;
 116 };
 117
 118 struct rte_sched_queue_extra {
 119         struct rte_sched_queue_stats stats;
 120 #ifdef RTE_SCHED_RED
 121         struct rte_red red;
 122 #endif
 123 };
 124
 125 enum grinder_state {
 126         e_GRINDER_PREFETCH_PIPE = 0,
 127         e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS,
 128         e_GRINDER_PREFETCH_MBUF,
 129         e_GRINDER_READ_MBUF
 130 };
 131
 132 struct rte_sched_grinder {
 133         /* Pipe cache */
 134         uint16_t pcache_qmask[RTE_SCHED_GRINDER_PCACHE_SIZE];
 135         uint32_t pcache_qindex[RTE_SCHED_GRINDER_PCACHE_SIZE];
 136         uint32_t pcache_w;
 137         uint32_t pcache_r;
 138
 139         /* Current pipe */
 140         enum grinder_state state;
 141         uint32_t productive;
 142         uint32_t pindex;
 143         struct rte_sched_subport *subport;
 144         struct rte_sched_pipe *pipe;
 145         struct rte_sched_pipe_profile *pipe_params;
 146
 147         /* TC cache */
 148         uint8_t tccache_qmask[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 149         uint32_t tccache_qindex[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 150         uint32_t tccache_w;
 151         uint32_t tccache_r;
 152
 153         /* Current TC */
 154         uint32_t tc_index;
 155         struct rte_sched_queue *queue[RTE_SCHED_MAX_QUEUES_PER_TC];
 156         struct rte_mbuf **qbase[RTE_SCHED_MAX_QUEUES_PER_TC];
 157         uint32_t qindex[RTE_SCHED_MAX_QUEUES_PER_TC];
 158         uint16_t qsize;
 159         uint32_t qmask;
 160         uint32_t qpos;
 161         struct rte_mbuf *pkt;
 162
 163         /* WRR */
 164         uint16_t wrr_tokens[RTE_SCHED_BE_QUEUES_PER_PIPE];
 165         uint16_t wrr_mask[RTE_SCHED_BE_QUEUES_PER_PIPE];
 166         uint8_t wrr_cost[RTE_SCHED_BE_QUEUES_PER_PIPE];
 167 };
 168
 169 struct rte_sched_port {
 170         /* User parameters */
 171         uint32_t n_subports_per_port;
 172         uint32_t n_pipes_per_subport;
 173         uint32_t n_pipes_per_subport_log2;
 174         uint16_t pipe_queue[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 175         uint8_t pipe_tc[RTE_SCHED_QUEUES_PER_PIPE];
 176         uint8_t tc_queue[RTE_SCHED_QUEUES_PER_PIPE];
 177         uint32_t rate;
 178         uint32_t mtu;
 179         uint32_t frame_overhead;
 180         uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
 181         uint32_t n_pipe_profiles;
 182         uint32_t n_max_pipe_profiles;
 183         uint32_t pipe_tc_be_rate_max;
 184 #ifdef RTE_SCHED_RED
 185         struct rte_red_config red_config[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][RTE_COLORS];
 186 #endif
 187
 188         /* Timing */
 189         uint64_t time_cpu_cycles;     /* Current CPU time measured in CPU cyles */
 190         uint64_t time_cpu_bytes;      /* Current CPU time measured in bytes */
 191         uint64_t time;                /* Current NIC TX time measured in bytes */
 192         struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */
 193
 194         /* Scheduling loop detection */
 195         uint32_t pipe_loop;
 196         uint32_t pipe_exhaustion;
 197
 198         /* Bitmap */
 199         struct rte_bitmap *bmp;
 200         uint32_t grinder_base_bmp_pos[RTE_SCHED_PORT_N_GRINDERS] __rte_aligned_16;
 201
 202         /* Grinders */
 203         struct rte_sched_grinder grinder[RTE_SCHED_PORT_N_GRINDERS];
 204         uint32_t busy_grinders;
 205         struct rte_mbuf **pkts_out;
 206         uint32_t n_pkts_out;
 207
 208         /* Queue base calculation */
 209         uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE];
 210         uint32_t qsize_sum;
 211
 212         /* Large data structures */
 213         struct rte_sched_subport *subport;
 214         struct rte_sched_pipe *pipe;
 215         struct rte_sched_queue *queue;
 216         struct rte_sched_queue_extra *queue_extra;
 217         struct rte_sched_pipe_profile *pipe_profiles;
 218         uint8_t *bmp_array;
 219         struct rte_mbuf **queue_array;
 220         uint8_t memory[0] __rte_cache_aligned;
 221 } __rte_cache_aligned;
 222
 223 enum rte_sched_port_array {
 224         e_RTE_SCHED_PORT_ARRAY_SUBPORT = 0,
 225         e_RTE_SCHED_PORT_ARRAY_PIPE,
 226         e_RTE_SCHED_PORT_ARRAY_QUEUE,
 227         e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA,
 228         e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES,
 229         e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY,
 230         e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY,
 231         e_RTE_SCHED_PORT_ARRAY_TOTAL,
 232 };
 233
 234 #ifdef RTE_SCHED_COLLECT_STATS
 235
 236 static inline uint32_t
 237 rte_sched_port_queues_per_subport(struct rte_sched_port *port)
 238 {
 239         return RTE_SCHED_QUEUES_PER_PIPE * port->n_pipes_per_subport;
 240 }
 241
 242 #endif
 243
 244 static inline uint32_t
 245 rte_sched_port_queues_per_port(struct rte_sched_port *port)
 246 {
 247         return RTE_SCHED_QUEUES_PER_PIPE * port->n_pipes_per_subport * port->n_subports_per_port;
 248 }
 249
 250 static inline struct rte_mbuf **
 251 rte_sched_port_qbase(struct rte_sched_port *port, uint32_t qindex)
 252 {
 253         uint32_t pindex = qindex >> 4;
 254         uint32_t qpos = qindex & 0xF;
 255
 256         return (port->queue_array + pindex *
 257                 port->qsize_sum + port->qsize_add[qpos]);
 258 }
 259
 260 static inline uint16_t
 261 rte_sched_port_qsize(struct rte_sched_port *port, uint32_t qindex)
 262 {
 263         uint32_t tc = port->pipe_tc[qindex & (RTE_SCHED_QUEUES_PER_PIPE - 1)];
 264
 265         return port->qsize[tc];
 266 }
 267
 268 static inline uint16_t
 269 rte_sched_port_pipe_queue(struct rte_sched_port *port, uint32_t traffic_class)
 270 {
 271         uint16_t pipe_queue = port->pipe_queue[traffic_class];
 272
 273         return pipe_queue;
 274 }
 275
 276 static inline uint8_t
 277 rte_sched_port_pipe_tc(struct rte_sched_port *port, uint32_t qindex)
 278 {
 279         uint8_t pipe_tc = port->pipe_tc[qindex & (RTE_SCHED_QUEUES_PER_PIPE - 1)];
 280
 281         return pipe_tc;
 282 }
 283
 284 static inline uint8_t
 285 rte_sched_port_tc_queue(struct rte_sched_port *port, uint32_t qindex)
 286 {
 287         uint8_t tc_queue = port->tc_queue[qindex & (RTE_SCHED_QUEUES_PER_PIPE - 1)];
 288
 289         return tc_queue;
 290 }
 291
 292 static int
 293 pipe_profile_check(struct rte_sched_pipe_params *params,
 294         uint32_t rate, uint16_t *qsize)
 295 {
 296         uint32_t i;
 297
 298         /* Pipe parameters */
 299         if (params == NULL) {
 300                 RTE_LOG(ERR, SCHED,
 301                         "%s: Incorrect value for parameter params\n", __func__);
 302                 return -EINVAL;
 303         }
 304
 305         /* TB rate: non-zero, not greater than port rate */
 306         if (params->tb_rate == 0 ||
 307                 params->tb_rate > rate) {
 308                 RTE_LOG(ERR, SCHED,
 309                         "%s: Incorrect value for tb rate\n", __func__);
 310                 return -EINVAL;
 311         }
 312
 313         /* TB size: non-zero */
 314         if (params->tb_size == 0) {
 315                 RTE_LOG(ERR, SCHED,
 316                         "%s: Incorrect value for tb size\n", __func__);
 317                 return -EINVAL;
 318         }
 319
 320         /* TC rate: non-zero if qsize non-zero, less than pipe rate */
 321         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 322                 if ((qsize[i] == 0 && params->tc_rate[i] != 0) ||
 323                         (qsize[i] != 0 && (params->tc_rate[i] == 0 ||
 324                         params->tc_rate[i] > params->tb_rate))) {
 325                         RTE_LOG(ERR, SCHED,
 326                                 "%s: Incorrect value for qsize or tc_rate\n", __func__);
 327                         return -EINVAL;
 328                 }
 329         }
 330
 331         if (params->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE] == 0 ||
 332                 qsize[RTE_SCHED_TRAFFIC_CLASS_BE] == 0) {
 333                 RTE_LOG(ERR, SCHED,
 334                         "%s: Incorrect value for be traffic class rate\n", __func__);
 335                 return -EINVAL;
 336         }
 337
 338         /* TC period: non-zero */
 339         if (params->tc_period == 0) {
 340                 RTE_LOG(ERR, SCHED,
 341                         "%s: Incorrect value for tc period\n", __func__);
 342                 return -EINVAL;
 343         }
 344
 345         /*  Best effort tc oversubscription weight: non-zero */
 346         if (params->tc_ov_weight == 0) {
 347                 RTE_LOG(ERR, SCHED,
 348                         "%s: Incorrect value for tc ov weight\n", __func__);
 349                 return -EINVAL;
 350         }
 351
 352         /* Queue WRR weights: non-zero */
 353         for (i = 0; i < RTE_SCHED_BE_QUEUES_PER_PIPE; i++) {
 354                 if (params->wrr_weights[i] == 0) {
 355                         RTE_LOG(ERR, SCHED,
 356                                 "%s: Incorrect value for wrr weight\n", __func__);
 357                         return -EINVAL;
 358                 }
 359         }
 360
 361         return 0;
 362 }
 363
 364 static int
 365 rte_sched_port_check_params(struct rte_sched_port_params *params)
 366 {
 367         uint32_t i;
 368
 369         if (params == NULL) {
 370                 RTE_LOG(ERR, SCHED,
 371                         "%s: Incorrect value for parameter params\n", __func__);
 372                 return -EINVAL;
 373         }
 374
 375         /* socket */
 376         if (params->socket < 0) {
 377                 RTE_LOG(ERR, SCHED,
 378                         "%s: Incorrect value for socket id\n", __func__);
 379                 return -EINVAL;
 380         }
 381
 382         /* rate */
 383         if (params->rate == 0) {
 384                 RTE_LOG(ERR, SCHED,
 385                         "%s: Incorrect value for rate\n", __func__);
 386                 return -EINVAL;
 387         }
 388
 389         /* mtu */
 390         if (params->mtu == 0) {
 391                 RTE_LOG(ERR, SCHED,
 392                         "%s: Incorrect value for mtu\n", __func__);
 393                 return -EINVAL;
 394         }
 395
 396         /* n_subports_per_port: non-zero, limited to 16 bits, power of 2 */
 397         if (params->n_subports_per_port == 0 ||
 398             params->n_subports_per_port > 1u << 16 ||
 399             !rte_is_power_of_2(params->n_subports_per_port)) {
 400                 RTE_LOG(ERR, SCHED,
 401                         "%s: Incorrect value for number of subports\n", __func__);
 402                 return -EINVAL;
 403         }
 404
 405         /* n_pipes_per_subport: non-zero, power of 2 */
 406         if (params->n_pipes_per_subport == 0 ||
 407             !rte_is_power_of_2(params->n_pipes_per_subport)) {
 408                 RTE_LOG(ERR, SCHED,
 409                         "%s: Incorrect value for pipes number\n", __func__);
 410                 return -EINVAL;
 411         }
 412
 413         /* qsize: if non-zero, power of 2,
 414          * no bigger than 32K (due to 16-bit read/write pointers)
 415          */
 416         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 417                 uint16_t qsize = params->qsize[i];
 418
 419                 if ((qsize != 0 && !rte_is_power_of_2(qsize)) ||
 420                         ((i == RTE_SCHED_TRAFFIC_CLASS_BE) && (qsize == 0))) {
 421                         RTE_LOG(ERR, SCHED,
 422                                 "%s: Incorrect value for tc rate\n", __func__);
 423                         return -EINVAL;
 424                 }
 425         }
 426
 427         /* pipe_profiles and n_pipe_profiles */
 428         if (params->pipe_profiles == NULL ||
 429             params->n_pipe_profiles == 0 ||
 430                  params->n_pipe_profiles > params->n_max_pipe_profiles) {
 431                 RTE_LOG(ERR, SCHED,
 432                         "%s: Incorrect value for number of pipe profiles\n", __func__);
 433                 return -EINVAL;
 434         }
 435
 436         for (i = 0; i < params->n_pipe_profiles; i++) {
 437                 struct rte_sched_pipe_params *p = params->pipe_profiles + i;
 438                 int status;
 439
 440                 status = pipe_profile_check(p, params->rate, &params->qsize[0]);
 441                 if (status != 0) {
 442                         RTE_LOG(ERR, SCHED,
 443                                 "%s: Pipe profile check failed(%d)\n", __func__, status);
 444                         return -EINVAL;
 445                 }
 446         }
 447
 448         return 0;
 449 }
 450
 451 static uint32_t
 452 rte_sched_port_get_array_base(struct rte_sched_port_params *params, enum rte_sched_port_array array)
 453 {
 454         uint32_t n_subports_per_port = params->n_subports_per_port;
 455         uint32_t n_pipes_per_subport = params->n_pipes_per_subport;
 456         uint32_t n_pipes_per_port = n_pipes_per_subport * n_subports_per_port;
 457         uint32_t n_queues_per_port = RTE_SCHED_QUEUES_PER_PIPE * n_pipes_per_subport * n_subports_per_port;
 458
 459         uint32_t size_subport = n_subports_per_port * sizeof(struct rte_sched_subport);
 460         uint32_t size_pipe = n_pipes_per_port * sizeof(struct rte_sched_pipe);
 461         uint32_t size_queue = n_queues_per_port * sizeof(struct rte_sched_queue);
 462         uint32_t size_queue_extra
 463                 = n_queues_per_port * sizeof(struct rte_sched_queue_extra);
 464         uint32_t size_pipe_profiles
 465                 = params->n_max_pipe_profiles * sizeof(struct rte_sched_pipe_profile);
 466         uint32_t size_bmp_array = rte_bitmap_get_memory_footprint(n_queues_per_port);
 467         uint32_t size_per_pipe_queue_array, size_queue_array;
 468
 469         uint32_t base, i;
 470
 471         size_per_pipe_queue_array = 0;
 472         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 473                 if (i < RTE_SCHED_TRAFFIC_CLASS_BE)
 474                         size_per_pipe_queue_array +=
 475                                 params->qsize[i] * sizeof(struct rte_mbuf *);
 476                 else
 477                         size_per_pipe_queue_array += RTE_SCHED_MAX_QUEUES_PER_TC *
 478                                 params->qsize[i] * sizeof(struct rte_mbuf *);
 479         }
 480         size_queue_array = n_pipes_per_port * size_per_pipe_queue_array;
 481
 482         base = 0;
 483
 484         if (array == e_RTE_SCHED_PORT_ARRAY_SUBPORT)
 485                 return base;
 486         base += RTE_CACHE_LINE_ROUNDUP(size_subport);
 487
 488         if (array == e_RTE_SCHED_PORT_ARRAY_PIPE)
 489                 return base;
 490         base += RTE_CACHE_LINE_ROUNDUP(size_pipe);
 491
 492         if (array == e_RTE_SCHED_PORT_ARRAY_QUEUE)
 493                 return base;
 494         base += RTE_CACHE_LINE_ROUNDUP(size_queue);
 495
 496         if (array == e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA)
 497                 return base;
 498         base += RTE_CACHE_LINE_ROUNDUP(size_queue_extra);
 499
 500         if (array == e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES)
 501                 return base;
 502         base += RTE_CACHE_LINE_ROUNDUP(size_pipe_profiles);
 503
 504         if (array == e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY)
 505                 return base;
 506         base += RTE_CACHE_LINE_ROUNDUP(size_bmp_array);
 507
 508         if (array == e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY)
 509                 return base;
 510         base += RTE_CACHE_LINE_ROUNDUP(size_queue_array);
 511
 512         return base;
 513 }
 514
 515 uint32_t
 516 rte_sched_port_get_memory_footprint(struct rte_sched_port_params *params)
 517 {
 518         uint32_t size0, size1;
 519         int status;
 520
 521         status = rte_sched_port_check_params(params);
 522         if (status != 0) {
 523                 RTE_LOG(NOTICE, SCHED,
 524                         "Port scheduler params check failed (%d)\n", status);
 525
 526                 return 0;
 527         }
 528
 529         size0 = sizeof(struct rte_sched_port);
 530         size1 = rte_sched_port_get_array_base(params, e_RTE_SCHED_PORT_ARRAY_TOTAL);
 531
 532         return size0 + size1;
 533 }
 534
 535 static void
 536 rte_sched_port_config_qsize(struct rte_sched_port *port)
 537 {
 538         uint32_t i;
 539
 540         port->qsize_add[0] = 0;
 541
 542         /* Strict prority traffic class */
 543         for (i = 1; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 544                 port->qsize_add[i] = port->qsize_add[i-1] + port->qsize[i-1];
 545
 546         /* Best-effort traffic class */
 547         port->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 1] =
 548                 port->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE] +
 549                 port->qsize[RTE_SCHED_TRAFFIC_CLASS_BE];
 550         port->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 2] =
 551                 port->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 1] +
 552                 port->qsize[RTE_SCHED_TRAFFIC_CLASS_BE];
 553         port->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 3] =
 554                 port->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 2] +
 555                 port->qsize[RTE_SCHED_TRAFFIC_CLASS_BE];
 556
 557         port->qsize_sum = port->qsize_add[RTE_SCHED_TRAFFIC_CLASS_BE + 3] +
 558                 port->qsize[RTE_SCHED_TRAFFIC_CLASS_BE];
 559 }
 560
 561 static void
 562 rte_sched_port_log_pipe_profile(struct rte_sched_port *port, uint32_t i)
 563 {
 564         struct rte_sched_pipe_profile *p = port->pipe_profiles + i;
 565
 566         RTE_LOG(DEBUG, SCHED, "Low level config for pipe profile %u:\n"
 567                 "       Token bucket: period = %u, credits per period = %u, size = %u\n"
 568                 "       Traffic classes: period = %u,\n"
 569                 "       credits per period = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u]\n"
 570                 "       Best-effort traffic class oversubscription: weight = %hhu\n"
 571                 "       WRR cost: [%hhu, %hhu, %hhu, %hhu]\n",
 572                 i,
 573
 574                 /* Token bucket */
 575                 p->tb_period,
 576                 p->tb_credits_per_period,
 577                 p->tb_size,
 578
 579                 /* Traffic classes */
 580                 p->tc_period,
 581                 p->tc_credits_per_period[0],
 582                 p->tc_credits_per_period[1],
 583                 p->tc_credits_per_period[2],
 584                 p->tc_credits_per_period[3],
 585                 p->tc_credits_per_period[4],
 586                 p->tc_credits_per_period[5],
 587                 p->tc_credits_per_period[6],
 588                 p->tc_credits_per_period[7],
 589                 p->tc_credits_per_period[8],
 590                 p->tc_credits_per_period[9],
 591                 p->tc_credits_per_period[10],
 592                 p->tc_credits_per_period[11],
 593                 p->tc_credits_per_period[12],
 594
 595                 /* Best-effort traffic class oversubscription */
 596                 p->tc_ov_weight,
 597
 598                 /* WRR */
 599                 p->wrr_cost[0], p->wrr_cost[1], p->wrr_cost[2], p->wrr_cost[3]);
 600 }
 601
 602 static inline uint64_t
 603 rte_sched_time_ms_to_bytes(uint32_t time_ms, uint32_t rate)
 604 {
 605         uint64_t time = time_ms;
 606
 607         time = (time * rate) / 1000;
 608
 609         return time;
 610 }
 611
 612 static void
 613 rte_sched_pipe_profile_convert(struct rte_sched_port *port,
 614         struct rte_sched_pipe_params *src,
 615         struct rte_sched_pipe_profile *dst,
 616         uint32_t rate)
 617 {
 618         uint32_t wrr_cost[RTE_SCHED_BE_QUEUES_PER_PIPE];
 619         uint32_t lcd1, lcd2, lcd;
 620         uint32_t i;
 621
 622         /* Token Bucket */
 623         if (src->tb_rate == rate) {
 624                 dst->tb_credits_per_period = 1;
 625                 dst->tb_period = 1;
 626         } else {
 627                 double tb_rate = (double) src->tb_rate
 628                                 / (double) rate;
 629                 double d = RTE_SCHED_TB_RATE_CONFIG_ERR;
 630
 631                 rte_approx(tb_rate, d,
 632                         &dst->tb_credits_per_period, &dst->tb_period);
 633         }
 634
 635         dst->tb_size = src->tb_size;
 636
 637         /* Traffic Classes */
 638         dst->tc_period = rte_sched_time_ms_to_bytes(src->tc_period,
 639                                                 rate);
 640
 641         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 642                 if (port->qsize[i])
 643                         dst->tc_credits_per_period[i]
 644                                 = rte_sched_time_ms_to_bytes(src->tc_period,
 645                                         src->tc_rate[i]);
 646
 647         dst->tc_ov_weight = src->tc_ov_weight;
 648
 649         /* WRR queues */
 650         wrr_cost[0] = src->wrr_weights[0];
 651         wrr_cost[1] = src->wrr_weights[1];
 652         wrr_cost[2] = src->wrr_weights[2];
 653         wrr_cost[3] = src->wrr_weights[3];
 654
 655         lcd1 = rte_get_lcd(wrr_cost[0], wrr_cost[1]);
 656         lcd2 = rte_get_lcd(wrr_cost[2], wrr_cost[3]);
 657         lcd = rte_get_lcd(lcd1, lcd2);
 658
 659         wrr_cost[0] = lcd / wrr_cost[0];
 660         wrr_cost[1] = lcd / wrr_cost[1];
 661         wrr_cost[2] = lcd / wrr_cost[2];
 662         wrr_cost[3] = lcd / wrr_cost[3];
 663
 664         dst->wrr_cost[0] = (uint8_t) wrr_cost[0];
 665         dst->wrr_cost[1] = (uint8_t) wrr_cost[1];
 666         dst->wrr_cost[2] = (uint8_t) wrr_cost[2];
 667         dst->wrr_cost[3] = (uint8_t) wrr_cost[3];
 668 }
 669
 670 static void
 671 rte_sched_port_config_pipe_profile_table(struct rte_sched_port *port,
 672         struct rte_sched_port_params *params)
 673 {
 674         uint32_t i;
 675
 676         for (i = 0; i < port->n_pipe_profiles; i++) {
 677                 struct rte_sched_pipe_params *src = params->pipe_profiles + i;
 678                 struct rte_sched_pipe_profile *dst = port->pipe_profiles + i;
 679
 680                 rte_sched_pipe_profile_convert(port, src, dst, params->rate);
 681                 rte_sched_port_log_pipe_profile(port, i);
 682         }
 683
 684         port->pipe_tc_be_rate_max = 0;
 685         for (i = 0; i < port->n_pipe_profiles; i++) {
 686                 struct rte_sched_pipe_params *src = params->pipe_profiles + i;
 687                 uint32_t pipe_tc_be_rate = src->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE];
 688
 689                 if (port->pipe_tc_be_rate_max < pipe_tc_be_rate)
 690                         port->pipe_tc_be_rate_max = pipe_tc_be_rate;
 691         }
 692 }
 693
 694 struct rte_sched_port *
 695 rte_sched_port_config(struct rte_sched_port_params *params)
 696 {
 697         struct rte_sched_port *port = NULL;
 698         uint32_t mem_size, bmp_mem_size, n_queues_per_port, i, j, cycles_per_byte;
 699
 700         /* Check user parameters. Determine the amount of memory to allocate */
 701         mem_size = rte_sched_port_get_memory_footprint(params);
 702         if (mem_size == 0)
 703                 return NULL;
 704
 705         /* Allocate memory to store the data structures */
 706         port = rte_zmalloc_socket("qos_params", mem_size, RTE_CACHE_LINE_SIZE,
 707                 params->socket);
 708         if (port == NULL)
 709                 return NULL;
 710
 711         /* compile time checks */
 712         RTE_BUILD_BUG_ON(RTE_SCHED_PORT_N_GRINDERS == 0);
 713         RTE_BUILD_BUG_ON(RTE_SCHED_PORT_N_GRINDERS & (RTE_SCHED_PORT_N_GRINDERS - 1));
 714
 715         /* User parameters */
 716         port->n_subports_per_port = params->n_subports_per_port;
 717         port->n_pipes_per_subport = params->n_pipes_per_subport;
 718         port->n_pipes_per_subport_log2 =
 719                         __builtin_ctz(params->n_pipes_per_subport);
 720
 721         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 722                 port->pipe_queue[i] = i;
 723
 724         for (i = 0, j = 0; i < RTE_SCHED_QUEUES_PER_PIPE; i++) {
 725                 port->pipe_tc[i] = j;
 726
 727                 if (j < RTE_SCHED_TRAFFIC_CLASS_BE)
 728                         j++;
 729         }
 730
 731         for (i = 0, j = 0; i < RTE_SCHED_QUEUES_PER_PIPE; i++) {
 732                 port->tc_queue[i] = j;
 733
 734                 if (i >= RTE_SCHED_TRAFFIC_CLASS_BE)
 735                         j++;
 736         }
 737         port->rate = params->rate;
 738         port->mtu = params->mtu + params->frame_overhead;
 739         port->frame_overhead = params->frame_overhead;
 740         memcpy(port->qsize, params->qsize, sizeof(params->qsize));
 741         port->n_pipe_profiles = params->n_pipe_profiles;
 742         port->n_max_pipe_profiles = params->n_max_pipe_profiles;
 743
 744 #ifdef RTE_SCHED_RED
 745         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 746                 uint32_t j;
 747
 748                 for (j = 0; j < RTE_COLORS; j++) {
 749                         /* if min/max are both zero, then RED is disabled */
 750                         if ((params->red_params[i][j].min_th |
 751                              params->red_params[i][j].max_th) == 0) {
 752                                 continue;
 753                         }
 754
 755                         if (rte_red_config_init(&port->red_config[i][j],
 756                                 params->red_params[i][j].wq_log2,
 757                                 params->red_params[i][j].min_th,
 758                                 params->red_params[i][j].max_th,
 759                                 params->red_params[i][j].maxp_inv) != 0) {
 760                                 rte_free(port);
 761                                 return NULL;
 762                         }
 763                 }
 764         }
 765 #endif
 766
 767         /* Timing */
 768         port->time_cpu_cycles = rte_get_tsc_cycles();
 769         port->time_cpu_bytes = 0;
 770         port->time = 0;
 771
 772         cycles_per_byte = (rte_get_tsc_hz() << RTE_SCHED_TIME_SHIFT)
 773                 / params->rate;
 774         port->inv_cycles_per_byte = rte_reciprocal_value(cycles_per_byte);
 775
 776         /* Scheduling loop detection */
 777         port->pipe_loop = RTE_SCHED_PIPE_INVALID;
 778         port->pipe_exhaustion = 0;
 779
 780         /* Grinders */
 781         port->busy_grinders = 0;
 782         port->pkts_out = NULL;
 783         port->n_pkts_out = 0;
 784
 785         /* Queue base calculation */
 786         rte_sched_port_config_qsize(port);
 787
 788         /* Large data structures */
 789         port->subport = (struct rte_sched_subport *)
 790                 (port->memory + rte_sched_port_get_array_base(params,
 791                                                               e_RTE_SCHED_PORT_ARRAY_SUBPORT));
 792         port->pipe = (struct rte_sched_pipe *)
 793                 (port->memory + rte_sched_port_get_array_base(params,
 794                                                               e_RTE_SCHED_PORT_ARRAY_PIPE));
 795         port->queue = (struct rte_sched_queue *)
 796                 (port->memory + rte_sched_port_get_array_base(params,
 797                                                               e_RTE_SCHED_PORT_ARRAY_QUEUE));
 798         port->queue_extra = (struct rte_sched_queue_extra *)
 799                 (port->memory + rte_sched_port_get_array_base(params,
 800                                                               e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA));
 801         port->pipe_profiles = (struct rte_sched_pipe_profile *)
 802                 (port->memory + rte_sched_port_get_array_base(params,
 803                                                               e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES));
 804         port->bmp_array =  port->memory
 805                 + rte_sched_port_get_array_base(params, e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY);
 806         port->queue_array = (struct rte_mbuf **)
 807                 (port->memory + rte_sched_port_get_array_base(params,
 808                                                               e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY));
 809
 810         /* Pipe profile table */
 811         rte_sched_port_config_pipe_profile_table(port, params);
 812
 813         /* Bitmap */
 814         n_queues_per_port = rte_sched_port_queues_per_port(port);
 815         bmp_mem_size = rte_bitmap_get_memory_footprint(n_queues_per_port);
 816         port->bmp = rte_bitmap_init(n_queues_per_port, port->bmp_array,
 817                                     bmp_mem_size);
 818         if (port->bmp == NULL) {
 819                 RTE_LOG(ERR, SCHED, "Bitmap init error\n");
 820                 rte_free(port);
 821                 return NULL;
 822         }
 823
 824         for (i = 0; i < RTE_SCHED_PORT_N_GRINDERS; i++)
 825                 port->grinder_base_bmp_pos[i] = RTE_SCHED_PIPE_INVALID;
 826
 827
 828         return port;
 829 }
 830
 831 void
 832 rte_sched_port_free(struct rte_sched_port *port)
 833 {
 834         uint32_t qindex;
 835         uint32_t n_queues_per_port;
 836
 837         /* Check user parameters */
 838         if (port == NULL)
 839                 return;
 840
 841         n_queues_per_port = rte_sched_port_queues_per_port(port);
 842
 843         /* Free enqueued mbufs */
 844         for (qindex = 0; qindex < n_queues_per_port; qindex++) {
 845                 struct rte_mbuf **mbufs = rte_sched_port_qbase(port, qindex);
 846                 uint16_t qsize = rte_sched_port_qsize(port, qindex);
 847                 if (qsize != 0) {
 848                         struct rte_sched_queue *queue = port->queue + qindex;
 849                         uint16_t qr = queue->qr & (qsize - 1);
 850                         uint16_t qw = queue->qw & (qsize - 1);
 851
 852                         for (; qr != qw; qr = (qr + 1) & (qsize - 1))
 853                                 rte_pktmbuf_free(mbufs[qr]);
 854                 }
 855         }
 856
 857         rte_bitmap_free(port->bmp);
 858         rte_free(port);
 859 }
 860
 861 static void
 862 rte_sched_port_log_subport_config(struct rte_sched_port *port, uint32_t i)
 863 {
 864         struct rte_sched_subport *s = port->subport + i;
 865
 866         RTE_LOG(DEBUG, SCHED, "Low level config for subport %u:\n"
 867                 "       Token bucket: period = %u, credits per period = %u, size = %u\n"
 868                 "       Traffic classes: period = %u\n"
 869                 "       credits per period = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u]\n"
 870                 "       Best effort traffic class oversubscription: wm min = %u, wm max = %u\n",
 871                 i,
 872
 873                 /* Token bucket */
 874                 s->tb_period,
 875                 s->tb_credits_per_period,
 876                 s->tb_size,
 877
 878                 /* Traffic classes */
 879                 s->tc_period,
 880                 s->tc_credits_per_period[0],
 881                 s->tc_credits_per_period[1],
 882                 s->tc_credits_per_period[2],
 883                 s->tc_credits_per_period[3],
 884                 s->tc_credits_per_period[4],
 885                 s->tc_credits_per_period[5],
 886                 s->tc_credits_per_period[6],
 887                 s->tc_credits_per_period[7],
 888                 s->tc_credits_per_period[8],
 889                 s->tc_credits_per_period[9],
 890                 s->tc_credits_per_period[10],
 891                 s->tc_credits_per_period[11],
 892                 s->tc_credits_per_period[12],
 893
 894                 /* Best effort traffic class oversubscription */
 895                 s->tc_ov_wm_min,
 896                 s->tc_ov_wm_max);
 897 }
 898
 899 int
 900 rte_sched_subport_config(struct rte_sched_port *port,
 901         uint32_t subport_id,
 902         struct rte_sched_subport_params *params)
 903 {
 904         struct rte_sched_subport *s;
 905         uint32_t i;
 906
 907         /* Check user parameters */
 908         if (port == NULL) {
 909                 RTE_LOG(ERR, SCHED,
 910                         "%s: Incorrect value for parameter port\n", __func__);
 911                 return -EINVAL;
 912         }
 913
 914         if (subport_id >= port->n_subports_per_port) {
 915                 RTE_LOG(ERR, SCHED,
 916                         "%s: Incorrect value for subport id\n", __func__);
 917                 return -EINVAL;
 918         }
 919
 920         if (params == NULL) {
 921                 RTE_LOG(ERR, SCHED,
 922                         "%s: Incorrect value for parameter params\n", __func__);
 923                 return -EINVAL;
 924         }
 925
 926         if (params->tb_rate == 0 || params->tb_rate > port->rate) {
 927                 RTE_LOG(ERR, SCHED,
 928                         "%s: Incorrect value for tb rate\n", __func__);
 929                 return -EINVAL;
 930         }
 931
 932         if (params->tb_size == 0) {
 933                 RTE_LOG(ERR, SCHED,
 934                         "%s: Incorrect value for tb size\n", __func__);
 935                 return -EINVAL;
 936         }
 937
 938         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 939                 uint32_t tc_rate = params->tc_rate[i];
 940                 uint16_t qsize = port->qsize[i];
 941
 942                 if ((qsize == 0 && tc_rate != 0) ||
 943                         (qsize != 0 && tc_rate == 0) ||
 944                         (tc_rate > params->tb_rate)) {
 945                         RTE_LOG(ERR, SCHED,
 946                                 "%s: Incorrect value for tc rate\n", __func__);
 947                         return -EINVAL;
 948                 }
 949         }
 950
 951         if (port->qsize[RTE_SCHED_TRAFFIC_CLASS_BE] == 0 ||
 952                 params->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE] == 0) {
 953                 RTE_LOG(ERR, SCHED,
 954                         "%s: Incorrect value for tc rate(best effort)\n", __func__);
 955                 return -EINVAL;
 956         }
 957
 958         if (params->tc_period == 0) {
 959                 RTE_LOG(ERR, SCHED,
 960                         "%s: Incorrect value for tc period\n", __func__);
 961                 return -EINVAL;
 962         }
 963
 964         s = port->subport + subport_id;
 965
 966         /* Token Bucket (TB) */
 967         if (params->tb_rate == port->rate) {
 968                 s->tb_credits_per_period = 1;
 969                 s->tb_period = 1;
 970         } else {
 971                 double tb_rate = ((double) params->tb_rate) / ((double) port->rate);
 972                 double d = RTE_SCHED_TB_RATE_CONFIG_ERR;
 973
 974                 rte_approx(tb_rate, d, &s->tb_credits_per_period, &s->tb_period);
 975         }
 976
 977         s->tb_size = params->tb_size;
 978         s->tb_time = port->time;
 979         s->tb_credits = s->tb_size / 2;
 980
 981         /* Traffic Classes (TCs) */
 982         s->tc_period = rte_sched_time_ms_to_bytes(params->tc_period, port->rate);
 983         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) {
 984                 if (port->qsize[i])
 985                         s->tc_credits_per_period[i]
 986                                 = rte_sched_time_ms_to_bytes(params->tc_period,
 987                                                                  params->tc_rate[i]);
 988
 989         }
 990         s->tc_time = port->time + s->tc_period;
 991         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
 992                 if (port->qsize[i])
 993                         s->tc_credits[i] = s->tc_credits_per_period[i];
 994
 995         /* TC oversubscription */
 996         s->tc_ov_wm_min = port->mtu;
 997         s->tc_ov_wm_max = rte_sched_time_ms_to_bytes(params->tc_period,
 998                                                      port->pipe_tc_be_rate_max);
 999         s->tc_ov_wm = s->tc_ov_wm_max;
1000         s->tc_ov_period_id = 0;
1001         s->tc_ov = 0;
1002         s->tc_ov_n = 0;
1003         s->tc_ov_rate = 0;
1004
1005         rte_sched_port_log_subport_config(port, subport_id);
1006
1007         return 0;
1008 }
1009
1010 int
1011 rte_sched_pipe_config(struct rte_sched_port *port,
1012         uint32_t subport_id,
1013         uint32_t pipe_id,
1014         int32_t pipe_profile)
1015 {
1016         struct rte_sched_subport *s;
1017         struct rte_sched_pipe *p;
1018         struct rte_sched_pipe_profile *params;
1019         uint32_t deactivate, profile, i;
1020
1021         /* Check user parameters */
1022         profile = (uint32_t) pipe_profile;
1023         deactivate = (pipe_profile < 0);
1024
1025         if (port == NULL) {
1026                 RTE_LOG(ERR, SCHED,
1027                         "%s: Incorrect value for parameter port\n", __func__);
1028                 return -EINVAL;
1029         }
1030
1031         if (subport_id >= port->n_subports_per_port) {
1032                 RTE_LOG(ERR, SCHED,
1033                         "%s: Incorrect value for parameter subport id\n", __func__);
1034                 return -EINVAL;
1035         }
1036
1037         if (pipe_id >= port->n_pipes_per_subport) {
1038                 RTE_LOG(ERR, SCHED,
1039                         "%s: Incorrect value for parameter pipe id\n", __func__);
1040                 return -EINVAL;
1041         }
1042
1043         if (!deactivate && profile >= port->n_pipe_profiles) {
1044                 RTE_LOG(ERR, SCHED,
1045                         "%s: Incorrect value for parameter pipe profile\n", __func__);
1046                 return -EINVAL;
1047         }
1048
1049         /* Check that subport configuration is valid */
1050         s = port->subport + subport_id;
1051         if (s->tb_period == 0) {
1052                 RTE_LOG(ERR, SCHED,
1053                         "%s: Subport configuration invalid\n", __func__);
1054                 return -EINVAL;
1055         }
1056
1057         p = port->pipe + (subport_id * port->n_pipes_per_subport + pipe_id);
1058
1059         /* Handle the case when pipe already has a valid configuration */
1060         if (p->tb_time) {
1061                 params = port->pipe_profiles + p->profile;
1062
1063                 double subport_tc_be_rate =
1064                         (double) s->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE]
1065                         / (double) s->tc_period;
1066                 double pipe_tc_be_rate =
1067                         (double) params->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE]
1068                         / (double) params->tc_period;
1069                 uint32_t tc_be_ov = s->tc_ov;
1070
1071                 /* Unplug pipe from its subport */
1072                 s->tc_ov_n -= params->tc_ov_weight;
1073                 s->tc_ov_rate -= pipe_tc_be_rate;
1074                 s->tc_ov = s->tc_ov_rate > subport_tc_be_rate;
1075
1076                 if (s->tc_ov != tc_be_ov) {
1077                         RTE_LOG(DEBUG, SCHED,
1078                                 "Subport %u Best-effort TC oversubscription is OFF (%.4lf >= %.4lf)\n",
1079                                 subport_id, subport_tc_be_rate, s->tc_ov_rate);
1080                 }
1081
1082                 /* Reset the pipe */
1083                 memset(p, 0, sizeof(struct rte_sched_pipe));
1084         }
1085
1086         if (deactivate)
1087                 return 0;
1088
1089         /* Apply the new pipe configuration */
1090         p->profile = profile;
1091         params = port->pipe_profiles + p->profile;
1092
1093         /* Token Bucket (TB) */
1094         p->tb_time = port->time;
1095         p->tb_credits = params->tb_size / 2;
1096
1097         /* Traffic Classes (TCs) */
1098         p->tc_time = port->time + params->tc_period;
1099
1100         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
1101                 if (port->qsize[i])
1102                         p->tc_credits[i] = params->tc_credits_per_period[i];
1103
1104         {
1105                 /* Subport best effort tc oversubscription */
1106                 double subport_tc_be_rate =
1107                         (double) s->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE]
1108                         / (double) s->tc_period;
1109                 double pipe_tc_be_rate =
1110                         (double) params->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE]
1111                         / (double) params->tc_period;
1112                 uint32_t tc_be_ov = s->tc_ov;
1113
1114                 s->tc_ov_n += params->tc_ov_weight;
1115                 s->tc_ov_rate += pipe_tc_be_rate;
1116                 s->tc_ov = s->tc_ov_rate > subport_tc_be_rate;
1117
1118                 if (s->tc_ov != tc_be_ov) {
1119                         RTE_LOG(DEBUG, SCHED,
1120                                 "Subport %u Best effort TC oversubscription is ON (%.4lf < %.4lf)\n",
1121                                 subport_id, subport_tc_be_rate, s->tc_ov_rate);
1122                 }
1123                 p->tc_ov_period_id = s->tc_ov_period_id;
1124                 p->tc_ov_credits = s->tc_ov_wm;
1125         }
1126
1127         return 0;
1128 }
1129
1130 int
1131 rte_sched_port_pipe_profile_add(struct rte_sched_port *port,
1132         struct rte_sched_pipe_params *params,
1133         uint32_t *pipe_profile_id)
1134 {
1135         struct rte_sched_pipe_profile *pp;
1136         uint32_t i;
1137         int status;
1138
1139         /* Port */
1140         if (port == NULL) {
1141                 RTE_LOG(ERR, SCHED,
1142                         "%s: Incorrect value for parameter port\n", __func__);
1143                 return -EINVAL;
1144         }
1145
1146         /* Pipe profiles not exceeds the max limit */
1147         if (port->n_pipe_profiles >= port->n_max_pipe_profiles) {
1148                 RTE_LOG(ERR, SCHED,
1149                         "%s: Number of pipe profiles exceeds the max limit\n", __func__);
1150                 return -EINVAL;
1151         }
1152
1153         /* Pipe params */
1154         status = pipe_profile_check(params, port->rate, &port->qsize[0]);
1155         if (status != 0) {
1156                 RTE_LOG(ERR, SCHED,
1157                         "%s: Pipe profile check failed(%d)\n", __func__, status);
1158                 return -EINVAL;
1159         }
1160
1161         pp = &port->pipe_profiles[port->n_pipe_profiles];
1162         rte_sched_pipe_profile_convert(port, params, pp, port->rate);
1163
1164         /* Pipe profile not exists */
1165         for (i = 0; i < port->n_pipe_profiles; i++)
1166                 if (memcmp(port->pipe_profiles + i, pp, sizeof(*pp)) == 0) {
1167                         RTE_LOG(ERR, SCHED,
1168                                 "%s: Pipe profile doesn't exist\n", __func__);
1169                         return -EINVAL;
1170                 }
1171
1172         /* Pipe profile commit */
1173         *pipe_profile_id = port->n_pipe_profiles;
1174         port->n_pipe_profiles++;
1175
1176         if (port->pipe_tc_be_rate_max < params->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE])
1177                 port->pipe_tc_be_rate_max = params->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE];
1178
1179         rte_sched_port_log_pipe_profile(port, *pipe_profile_id);
1180
1181         return 0;
1182 }
1183
1184 static inline uint32_t
1185 rte_sched_port_qindex(struct rte_sched_port *port,
1186         uint32_t subport,
1187         uint32_t pipe,
1188         uint32_t traffic_class,
1189         uint32_t queue)
1190 {
1191         return ((subport & (port->n_subports_per_port - 1)) <<
1192                         (port->n_pipes_per_subport_log2 + 4)) |
1193                         ((pipe & (port->n_pipes_per_subport - 1)) << 4) |
1194                         ((rte_sched_port_pipe_queue(port, traffic_class) + queue) &
1195                         (RTE_SCHED_QUEUES_PER_PIPE - 1));
1196 }
1197
1198 void
1199 rte_sched_port_pkt_write(struct rte_sched_port *port,
1200                          struct rte_mbuf *pkt,
1201                          uint32_t subport, uint32_t pipe,
1202                          uint32_t traffic_class,
1203                          uint32_t queue, enum rte_color color)
1204 {
1205         uint32_t queue_id =
1206                 rte_sched_port_qindex(port, subport, pipe, traffic_class, queue);
1207
1208         rte_mbuf_sched_set(pkt, queue_id, traffic_class, (uint8_t)color);
1209 }
1210
1211 void
1212 rte_sched_port_pkt_read_tree_path(struct rte_sched_port *port,
1213                                   const struct rte_mbuf *pkt,
1214                                   uint32_t *subport, uint32_t *pipe,
1215                                   uint32_t *traffic_class, uint32_t *queue)
1216 {
1217         uint32_t queue_id = rte_mbuf_sched_queue_get(pkt);
1218
1219         *subport = queue_id >> (port->n_pipes_per_subport_log2 + 4);
1220         *pipe = (queue_id >> 4) & (port->n_pipes_per_subport - 1);
1221         *traffic_class = rte_sched_port_pipe_tc(port, queue_id);
1222         *queue = rte_sched_port_tc_queue(port, queue_id);
1223 }
1224
1225 enum rte_color
1226 rte_sched_port_pkt_read_color(const struct rte_mbuf *pkt)
1227 {
1228         return (enum rte_color)rte_mbuf_sched_color_get(pkt);
1229 }
1230
1231 int
1232 rte_sched_subport_read_stats(struct rte_sched_port *port,
1233                              uint32_t subport_id,
1234                              struct rte_sched_subport_stats *stats,
1235                              uint32_t *tc_ov)
1236 {
1237         struct rte_sched_subport *s;
1238
1239         /* Check user parameters */
1240         if (port == NULL) {
1241                 RTE_LOG(ERR, SCHED,
1242                         "%s: Incorrect value for parameter port\n", __func__);
1243                 return -EINVAL;
1244         }
1245
1246         if (subport_id >= port->n_subports_per_port) {
1247                 RTE_LOG(ERR, SCHED,
1248                         "%s: Incorrect value for subport id\n", __func__);
1249                 return -EINVAL;
1250         }
1251
1252         if (stats == NULL) {
1253                 RTE_LOG(ERR, SCHED,
1254                         "%s: Incorrect value for parameter stats\n", __func__);
1255                 return -EINVAL;
1256         }
1257
1258         if (tc_ov == NULL) {
1259                 RTE_LOG(ERR, SCHED,
1260                         "%s: Incorrect value for tc_ov\n", __func__);
1261                 return -EINVAL;
1262         }
1263
1264         s = port->subport + subport_id;
1265
1266         /* Copy subport stats and clear */
1267         memcpy(stats, &s->stats, sizeof(struct rte_sched_subport_stats));
1268         memset(&s->stats, 0, sizeof(struct rte_sched_subport_stats));
1269
1270         /* Subport TC oversubscription status */
1271         *tc_ov = s->tc_ov;
1272
1273         return 0;
1274 }
1275
1276 int
1277 rte_sched_queue_read_stats(struct rte_sched_port *port,
1278         uint32_t queue_id,
1279         struct rte_sched_queue_stats *stats,
1280         uint16_t *qlen)
1281 {
1282         struct rte_sched_queue *q;
1283         struct rte_sched_queue_extra *qe;
1284
1285         /* Check user parameters */
1286         if (port == NULL) {
1287                 RTE_LOG(ERR, SCHED,
1288                         "%s: Incorrect value for parameter port\n", __func__);
1289                 return -EINVAL;
1290         }
1291
1292         if (queue_id >= rte_sched_port_queues_per_port(port)) {
1293                 RTE_LOG(ERR, SCHED,
1294                         "%s: Incorrect value for queue id\n", __func__);
1295                 return -EINVAL;
1296         }
1297
1298         if (stats == NULL) {
1299                 RTE_LOG(ERR, SCHED,
1300                         "%s: Incorrect value for parameter stats\n", __func__);
1301                 return -EINVAL;
1302         }
1303
1304         if (qlen == NULL) {
1305                 RTE_LOG(ERR, SCHED,
1306                         "%s: Incorrect value for parameter qlen\n", __func__);
1307                 return -EINVAL;
1308         }
1309         q = port->queue + queue_id;
1310         qe = port->queue_extra + queue_id;
1311
1312         /* Copy queue stats and clear */
1313         memcpy(stats, &qe->stats, sizeof(struct rte_sched_queue_stats));
1314         memset(&qe->stats, 0, sizeof(struct rte_sched_queue_stats));
1315
1316         /* Queue length */
1317         *qlen = q->qw - q->qr;
1318
1319         return 0;
1320 }
1321
1322 #ifdef RTE_SCHED_DEBUG
1323
1324 static inline int
1325 rte_sched_port_queue_is_empty(struct rte_sched_port *port, uint32_t qindex)
1326 {
1327         struct rte_sched_queue *queue = port->queue + qindex;
1328
1329         return queue->qr == queue->qw;
1330 }
1331
1332 #endif /* RTE_SCHED_DEBUG */
1333
1334 #ifdef RTE_SCHED_COLLECT_STATS
1335
1336 static inline void
1337 rte_sched_port_update_subport_stats(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
1338 {
1339         struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port));
1340         uint32_t tc_index = rte_sched_port_pipe_tc(port, qindex);
1341         uint32_t pkt_len = pkt->pkt_len;
1342
1343         s->stats.n_pkts_tc[tc_index] += 1;
1344         s->stats.n_bytes_tc[tc_index] += pkt_len;
1345 }
1346
1347 #ifdef RTE_SCHED_RED
1348 static inline void
1349 rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port,
1350                                                 uint32_t qindex,
1351                                                 struct rte_mbuf *pkt, uint32_t red)
1352 #else
1353 static inline void
1354 rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port,
1355                                                 uint32_t qindex,
1356                                                 struct rte_mbuf *pkt, __rte_unused uint32_t red)
1357 #endif
1358 {
1359         struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port));
1360         uint32_t tc_index = rte_sched_port_pipe_tc(port, qindex);
1361         uint32_t pkt_len = pkt->pkt_len;
1362
1363         s->stats.n_pkts_tc_dropped[tc_index] += 1;
1364         s->stats.n_bytes_tc_dropped[tc_index] += pkt_len;
1365 #ifdef RTE_SCHED_RED
1366         s->stats.n_pkts_red_dropped[tc_index] += red;
1367 #endif
1368 }
1369
1370 static inline void
1371 rte_sched_port_update_queue_stats(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
1372 {
1373         struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
1374         uint32_t pkt_len = pkt->pkt_len;
1375
1376         qe->stats.n_pkts += 1;
1377         qe->stats.n_bytes += pkt_len;
1378 }
1379
1380 #ifdef RTE_SCHED_RED
1381 static inline void
1382 rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port,
1383                                                 uint32_t qindex,
1384                                                 struct rte_mbuf *pkt, uint32_t red)
1385 #else
1386 static inline void
1387 rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port,
1388                                                 uint32_t qindex,
1389                                                 struct rte_mbuf *pkt, __rte_unused uint32_t red)
1390 #endif
1391 {
1392         struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
1393         uint32_t pkt_len = pkt->pkt_len;
1394
1395         qe->stats.n_pkts_dropped += 1;
1396         qe->stats.n_bytes_dropped += pkt_len;
1397 #ifdef RTE_SCHED_RED
1398         qe->stats.n_pkts_red_dropped += red;
1399 #endif
1400 }
1401
1402 #endif /* RTE_SCHED_COLLECT_STATS */
1403
1404 #ifdef RTE_SCHED_RED
1405
1406 static inline int
1407 rte_sched_port_red_drop(struct rte_sched_port *port, struct rte_mbuf *pkt, uint32_t qindex, uint16_t qlen)
1408 {
1409         struct rte_sched_queue_extra *qe;
1410         struct rte_red_config *red_cfg;
1411         struct rte_red *red;
1412         uint32_t tc_index;
1413         enum rte_color color;
1414
1415         tc_index = rte_sched_port_pipe_tc(port, qindex);
1416         color = rte_sched_port_pkt_read_color(pkt);
1417         red_cfg = &port->red_config[tc_index][color];
1418
1419         if ((red_cfg->min_th | red_cfg->max_th) == 0)
1420                 return 0;
1421
1422         qe = port->queue_extra + qindex;
1423         red = &qe->red;
1424
1425         return rte_red_enqueue(red_cfg, red, qlen, port->time);
1426 }
1427
1428 static inline void
1429 rte_sched_port_set_queue_empty_timestamp(struct rte_sched_port *port, uint32_t qindex)
1430 {
1431         struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
1432         struct rte_red *red = &qe->red;
1433
1434         rte_red_mark_queue_empty(red, port->time);
1435 }
1436
1437 #else
1438
1439 #define rte_sched_port_red_drop(port, pkt, qindex, qlen)             0
1440
1441 #define rte_sched_port_set_queue_empty_timestamp(port, qindex)
1442
1443 #endif /* RTE_SCHED_RED */
1444
1445 #ifdef RTE_SCHED_DEBUG
1446
1447 static inline void
1448 debug_check_queue_slab(struct rte_sched_port *port, uint32_t bmp_pos,
1449                        uint64_t bmp_slab)
1450 {
1451         uint64_t mask;
1452         uint32_t i, panic;
1453
1454         if (bmp_slab == 0)
1455                 rte_panic("Empty slab at position %u\n", bmp_pos);
1456
1457         panic = 0;
1458         for (i = 0, mask = 1; i < 64; i++, mask <<= 1) {
1459                 if (mask & bmp_slab) {
1460                         if (rte_sched_port_queue_is_empty(port, bmp_pos + i)) {
1461                                 printf("Queue %u (slab offset %u) is empty\n", bmp_pos + i, i);
1462                                 panic = 1;
1463                         }
1464                 }
1465         }
1466
1467         if (panic)
1468                 rte_panic("Empty queues in slab 0x%" PRIx64 "starting at position %u\n",
1469                         bmp_slab, bmp_pos);
1470 }
1471
1472 #endif /* RTE_SCHED_DEBUG */
1473
1474 static inline uint32_t
1475 rte_sched_port_enqueue_qptrs_prefetch0(struct rte_sched_port *port,
1476                                        struct rte_mbuf *pkt)
1477 {
1478         struct rte_sched_queue *q;
1479 #ifdef RTE_SCHED_COLLECT_STATS
1480         struct rte_sched_queue_extra *qe;
1481 #endif
1482         uint32_t qindex = rte_mbuf_sched_queue_get(pkt);
1483
1484         q = port->queue + qindex;
1485         rte_prefetch0(q);
1486 #ifdef RTE_SCHED_COLLECT_STATS
1487         qe = port->queue_extra + qindex;
1488         rte_prefetch0(qe);
1489 #endif
1490
1491         return qindex;
1492 }
1493
1494 static inline void
1495 rte_sched_port_enqueue_qwa_prefetch0(struct rte_sched_port *port,
1496                                      uint32_t qindex, struct rte_mbuf **qbase)
1497 {
1498         struct rte_sched_queue *q;
1499         struct rte_mbuf **q_qw;
1500         uint16_t qsize;
1501
1502         q = port->queue + qindex;
1503         qsize = rte_sched_port_qsize(port, qindex);
1504         q_qw = qbase + (q->qw & (qsize - 1));
1505
1506         rte_prefetch0(q_qw);
1507         rte_bitmap_prefetch0(port->bmp, qindex);
1508 }
1509
1510 static inline int
1511 rte_sched_port_enqueue_qwa(struct rte_sched_port *port, uint32_t qindex,
1512                            struct rte_mbuf **qbase, struct rte_mbuf *pkt)
1513 {
1514         struct rte_sched_queue *q;
1515         uint16_t qsize;
1516         uint16_t qlen;
1517
1518         q = port->queue + qindex;
1519         qsize = rte_sched_port_qsize(port, qindex);
1520         qlen = q->qw - q->qr;
1521
1522         /* Drop the packet (and update drop stats) when queue is full */
1523         if (unlikely(rte_sched_port_red_drop(port, pkt, qindex, qlen) ||
1524                      (qlen >= qsize))) {
1525                 rte_pktmbuf_free(pkt);
1526 #ifdef RTE_SCHED_COLLECT_STATS
1527                 rte_sched_port_update_subport_stats_on_drop(port, qindex, pkt,
1528                                                             qlen < qsize);
1529                 rte_sched_port_update_queue_stats_on_drop(port, qindex, pkt,
1530                                                           qlen < qsize);
1531 #endif
1532                 return 0;
1533         }
1534
1535         /* Enqueue packet */
1536         qbase[q->qw & (qsize - 1)] = pkt;
1537         q->qw++;
1538
1539         /* Activate queue in the port bitmap */
1540         rte_bitmap_set(port->bmp, qindex);
1541
1542         /* Statistics */
1543 #ifdef RTE_SCHED_COLLECT_STATS
1544         rte_sched_port_update_subport_stats(port, qindex, pkt);
1545         rte_sched_port_update_queue_stats(port, qindex, pkt);
1546 #endif
1547
1548         return 1;
1549 }
1550
1551
1552 /*
1553  * The enqueue function implements a 4-level pipeline with each stage
1554  * processing two different packets. The purpose of using a pipeline
1555  * is to hide the latency of prefetching the data structures. The
1556  * naming convention is presented in the diagram below:
1557  *
1558  *   p00  _______   p10  _______   p20  _______   p30  _______
1559  * ----->|       |----->|       |----->|       |----->|       |----->
1560  *       |   0   |      |   1   |      |   2   |      |   3   |
1561  * ----->|_______|----->|_______|----->|_______|----->|_______|----->
1562  *   p01            p11            p21            p31
1563  *
1564  */
1565 int
1566 rte_sched_port_enqueue(struct rte_sched_port *port, struct rte_mbuf **pkts,
1567                        uint32_t n_pkts)
1568 {
1569         struct rte_mbuf *pkt00, *pkt01, *pkt10, *pkt11, *pkt20, *pkt21,
1570                 *pkt30, *pkt31, *pkt_last;
1571         struct rte_mbuf **q00_base, **q01_base, **q10_base, **q11_base,
1572                 **q20_base, **q21_base, **q30_base, **q31_base, **q_last_base;
1573         uint32_t q00, q01, q10, q11, q20, q21, q30, q31, q_last;
1574         uint32_t r00, r01, r10, r11, r20, r21, r30, r31, r_last;
1575         uint32_t result, i;
1576
1577         result = 0;
1578
1579         /*
1580          * Less then 6 input packets available, which is not enough to
1581          * feed the pipeline
1582          */
1583         if (unlikely(n_pkts < 6)) {
1584                 struct rte_mbuf **q_base[5];
1585                 uint32_t q[5];
1586
1587                 /* Prefetch the mbuf structure of each packet */
1588                 for (i = 0; i < n_pkts; i++)
1589                         rte_prefetch0(pkts[i]);
1590
1591                 /* Prefetch the queue structure for each queue */
1592                 for (i = 0; i < n_pkts; i++)
1593                         q[i] = rte_sched_port_enqueue_qptrs_prefetch0(port,
1594                                                                       pkts[i]);
1595
1596                 /* Prefetch the write pointer location of each queue */
1597                 for (i = 0; i < n_pkts; i++) {
1598                         q_base[i] = rte_sched_port_qbase(port, q[i]);
1599                         rte_sched_port_enqueue_qwa_prefetch0(port, q[i],
1600                                                              q_base[i]);
1601                 }
1602
1603                 /* Write each packet to its queue */
1604                 for (i = 0; i < n_pkts; i++)
1605                         result += rte_sched_port_enqueue_qwa(port, q[i],
1606                                                              q_base[i], pkts[i]);
1607
1608                 return result;
1609         }
1610
1611         /* Feed the first 3 stages of the pipeline (6 packets needed) */
1612         pkt20 = pkts[0];
1613         pkt21 = pkts[1];
1614         rte_prefetch0(pkt20);
1615         rte_prefetch0(pkt21);
1616
1617         pkt10 = pkts[2];
1618         pkt11 = pkts[3];
1619         rte_prefetch0(pkt10);
1620         rte_prefetch0(pkt11);
1621
1622         q20 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt20);
1623         q21 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt21);
1624
1625         pkt00 = pkts[4];
1626         pkt01 = pkts[5];
1627         rte_prefetch0(pkt00);
1628         rte_prefetch0(pkt01);
1629
1630         q10 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt10);
1631         q11 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt11);
1632
1633         q20_base = rte_sched_port_qbase(port, q20);
1634         q21_base = rte_sched_port_qbase(port, q21);
1635         rte_sched_port_enqueue_qwa_prefetch0(port, q20, q20_base);
1636         rte_sched_port_enqueue_qwa_prefetch0(port, q21, q21_base);
1637
1638         /* Run the pipeline */
1639         for (i = 6; i < (n_pkts & (~1)); i += 2) {
1640                 /* Propagate stage inputs */
1641                 pkt30 = pkt20;
1642                 pkt31 = pkt21;
1643                 pkt20 = pkt10;
1644                 pkt21 = pkt11;
1645                 pkt10 = pkt00;
1646                 pkt11 = pkt01;
1647                 q30 = q20;
1648                 q31 = q21;
1649                 q20 = q10;
1650                 q21 = q11;
1651                 q30_base = q20_base;
1652                 q31_base = q21_base;
1653
1654                 /* Stage 0: Get packets in */
1655                 pkt00 = pkts[i];
1656                 pkt01 = pkts[i + 1];
1657                 rte_prefetch0(pkt00);
1658                 rte_prefetch0(pkt01);
1659
1660                 /* Stage 1: Prefetch queue structure storing queue pointers */
1661                 q10 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt10);
1662                 q11 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt11);
1663
1664                 /* Stage 2: Prefetch queue write location */
1665                 q20_base = rte_sched_port_qbase(port, q20);
1666                 q21_base = rte_sched_port_qbase(port, q21);
1667                 rte_sched_port_enqueue_qwa_prefetch0(port, q20, q20_base);
1668                 rte_sched_port_enqueue_qwa_prefetch0(port, q21, q21_base);
1669
1670                 /* Stage 3: Write packet to queue and activate queue */
1671                 r30 = rte_sched_port_enqueue_qwa(port, q30, q30_base, pkt30);
1672                 r31 = rte_sched_port_enqueue_qwa(port, q31, q31_base, pkt31);
1673                 result += r30 + r31;
1674         }
1675
1676         /*
1677          * Drain the pipeline (exactly 6 packets).
1678          * Handle the last packet in the case
1679          * of an odd number of input packets.
1680          */
1681         pkt_last = pkts[n_pkts - 1];
1682         rte_prefetch0(pkt_last);
1683
1684         q00 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt00);
1685         q01 = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt01);
1686
1687         q10_base = rte_sched_port_qbase(port, q10);
1688         q11_base = rte_sched_port_qbase(port, q11);
1689         rte_sched_port_enqueue_qwa_prefetch0(port, q10, q10_base);
1690         rte_sched_port_enqueue_qwa_prefetch0(port, q11, q11_base);
1691
1692         r20 = rte_sched_port_enqueue_qwa(port, q20, q20_base, pkt20);
1693         r21 = rte_sched_port_enqueue_qwa(port, q21, q21_base, pkt21);
1694         result += r20 + r21;
1695
1696         q_last = rte_sched_port_enqueue_qptrs_prefetch0(port, pkt_last);
1697
1698         q00_base = rte_sched_port_qbase(port, q00);
1699         q01_base = rte_sched_port_qbase(port, q01);
1700         rte_sched_port_enqueue_qwa_prefetch0(port, q00, q00_base);
1701         rte_sched_port_enqueue_qwa_prefetch0(port, q01, q01_base);
1702
1703         r10 = rte_sched_port_enqueue_qwa(port, q10, q10_base, pkt10);
1704         r11 = rte_sched_port_enqueue_qwa(port, q11, q11_base, pkt11);
1705         result += r10 + r11;
1706
1707         q_last_base = rte_sched_port_qbase(port, q_last);
1708         rte_sched_port_enqueue_qwa_prefetch0(port, q_last, q_last_base);
1709
1710         r00 = rte_sched_port_enqueue_qwa(port, q00, q00_base, pkt00);
1711         r01 = rte_sched_port_enqueue_qwa(port, q01, q01_base, pkt01);
1712         result += r00 + r01;
1713
1714         if (n_pkts & 1) {
1715                 r_last = rte_sched_port_enqueue_qwa(port, q_last, q_last_base, pkt_last);
1716                 result += r_last;
1717         }
1718
1719         return result;
1720 }
1721
1722 #ifndef RTE_SCHED_SUBPORT_TC_OV
1723
1724 static inline void
1725 grinder_credits_update(struct rte_sched_port *port, uint32_t pos)
1726 {
1727         struct rte_sched_grinder *grinder = port->grinder + pos;
1728         struct rte_sched_subport *subport = grinder->subport;
1729         struct rte_sched_pipe *pipe = grinder->pipe;
1730         struct rte_sched_pipe_profile *params = grinder->pipe_params;
1731         uint64_t n_periods;
1732         uint32_t i;
1733
1734         /* Subport TB */
1735         n_periods = (port->time - subport->tb_time) / subport->tb_period;
1736         subport->tb_credits += n_periods * subport->tb_credits_per_period;
1737         subport->tb_credits = rte_sched_min_val_2_u32(subport->tb_credits, subport->tb_size);
1738         subport->tb_time += n_periods * subport->tb_period;
1739
1740         /* Pipe TB */
1741         n_periods = (port->time - pipe->tb_time) / params->tb_period;
1742         pipe->tb_credits += n_periods * params->tb_credits_per_period;
1743         pipe->tb_credits = rte_sched_min_val_2_u32(pipe->tb_credits, params->tb_size);
1744         pipe->tb_time += n_periods * params->tb_period;
1745
1746         /* Subport TCs */
1747         if (unlikely(port->time >= subport->tc_time)) {
1748                 for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
1749                         subport->tc_credits[i] = subport->tc_credits_per_period[i];
1750
1751                 subport->tc_time = port->time + subport->tc_period;
1752         }
1753
1754         /* Pipe TCs */
1755         if (unlikely(port->time >= pipe->tc_time)) {
1756                 for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
1757                         pipe->tc_credits[i] = params->tc_credits_per_period[i];
1758
1759                 pipe->tc_time = port->time + params->tc_period;
1760         }
1761 }
1762
1763 #else
1764
1765 static inline uint32_t
1766 grinder_tc_ov_credits_update(struct rte_sched_port *port, uint32_t pos)
1767 {
1768         struct rte_sched_grinder *grinder = port->grinder + pos;
1769         struct rte_sched_subport *subport = grinder->subport;
1770         uint32_t tc_ov_consumption[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
1771         uint32_t tc_consumption = 0, tc_ov_consumption_max;
1772         uint32_t tc_ov_wm = subport->tc_ov_wm;
1773         uint32_t i;
1774
1775         if (subport->tc_ov == 0)
1776                 return subport->tc_ov_wm_max;
1777
1778         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASS_BE; i++) {
1779                 tc_ov_consumption[i] =
1780                         subport->tc_credits_per_period[i] - subport->tc_credits[i];
1781                 tc_consumption += tc_ov_consumption[i];
1782         }
1783
1784         tc_ov_consumption[RTE_SCHED_TRAFFIC_CLASS_BE] =
1785                 subport->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE] -
1786                 subport->tc_credits[RTE_SCHED_TRAFFIC_CLASS_BE];
1787
1788         tc_ov_consumption_max =
1789                 subport->tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASS_BE] -
1790                         tc_consumption;
1791
1792         if (tc_ov_consumption[RTE_SCHED_TRAFFIC_CLASS_BE] >
1793                 (tc_ov_consumption_max - port->mtu)) {
1794                 tc_ov_wm  -= tc_ov_wm >> 7;
1795                 if (tc_ov_wm < subport->tc_ov_wm_min)
1796                         tc_ov_wm = subport->tc_ov_wm_min;
1797
1798                 return tc_ov_wm;
1799         }
1800
1801         tc_ov_wm += (tc_ov_wm >> 7) + 1;
1802         if (tc_ov_wm > subport->tc_ov_wm_max)
1803                 tc_ov_wm = subport->tc_ov_wm_max;
1804
1805         return tc_ov_wm;
1806 }
1807
1808 static inline void
1809 grinder_credits_update(struct rte_sched_port *port, uint32_t pos)
1810 {
1811         struct rte_sched_grinder *grinder = port->grinder + pos;
1812         struct rte_sched_subport *subport = grinder->subport;
1813         struct rte_sched_pipe *pipe = grinder->pipe;
1814         struct rte_sched_pipe_profile *params = grinder->pipe_params;
1815         uint64_t n_periods;
1816         uint32_t i;
1817
1818         /* Subport TB */
1819         n_periods = (port->time - subport->tb_time) / subport->tb_period;
1820         subport->tb_credits += n_periods * subport->tb_credits_per_period;
1821         subport->tb_credits = rte_sched_min_val_2_u32(subport->tb_credits, subport->tb_size);
1822         subport->tb_time += n_periods * subport->tb_period;
1823
1824         /* Pipe TB */
1825         n_periods = (port->time - pipe->tb_time) / params->tb_period;
1826         pipe->tb_credits += n_periods * params->tb_credits_per_period;
1827         pipe->tb_credits = rte_sched_min_val_2_u32(pipe->tb_credits, params->tb_size);
1828         pipe->tb_time += n_periods * params->tb_period;
1829
1830         /* Subport TCs */
1831         if (unlikely(port->time >= subport->tc_time)) {
1832                 subport->tc_ov_wm = grinder_tc_ov_credits_update(port, pos);
1833
1834                 for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
1835                         subport->tc_credits[i] = subport->tc_credits_per_period[i];
1836
1837                 subport->tc_time = port->time + subport->tc_period;
1838                 subport->tc_ov_period_id++;
1839         }
1840
1841         /* Pipe TCs */
1842         if (unlikely(port->time >= pipe->tc_time)) {
1843                 for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
1844                         pipe->tc_credits[i] = params->tc_credits_per_period[i];
1845                 pipe->tc_time = port->time + params->tc_period;
1846         }
1847
1848         /* Pipe TCs - Oversubscription */
1849         if (unlikely(pipe->tc_ov_period_id != subport->tc_ov_period_id)) {
1850                 pipe->tc_ov_credits = subport->tc_ov_wm * params->tc_ov_weight;
1851
1852                 pipe->tc_ov_period_id = subport->tc_ov_period_id;
1853         }
1854 }
1855
1856 #endif /* RTE_SCHED_TS_CREDITS_UPDATE, RTE_SCHED_SUBPORT_TC_OV */
1857
1858
1859 #ifndef RTE_SCHED_SUBPORT_TC_OV
1860
1861 static inline int
1862 grinder_credits_check(struct rte_sched_port *port, uint32_t pos)
1863 {
1864         struct rte_sched_grinder *grinder = port->grinder + pos;
1865         struct rte_sched_subport *subport = grinder->subport;
1866         struct rte_sched_pipe *pipe = grinder->pipe;
1867         struct rte_mbuf *pkt = grinder->pkt;
1868         uint32_t tc_index = grinder->tc_index;
1869         uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
1870         uint32_t subport_tb_credits = subport->tb_credits;
1871         uint32_t subport_tc_credits = subport->tc_credits[tc_index];
1872         uint32_t pipe_tb_credits = pipe->tb_credits;
1873         uint32_t pipe_tc_credits = pipe->tc_credits[tc_index];
1874         int enough_credits;
1875
1876         /* Check queue credits */
1877         enough_credits = (pkt_len <= subport_tb_credits) &&
1878                 (pkt_len <= subport_tc_credits) &&
1879                 (pkt_len <= pipe_tb_credits) &&
1880                 (pkt_len <= pipe_tc_credits);
1881
1882         if (!enough_credits)
1883                 return 0;
1884
1885         /* Update port credits */
1886         subport->tb_credits -= pkt_len;
1887         subport->tc_credits[tc_index] -= pkt_len;
1888         pipe->tb_credits -= pkt_len;
1889         pipe->tc_credits[tc_index] -= pkt_len;
1890
1891         return 1;
1892 }
1893
1894 #else
1895
1896 static inline int
1897 grinder_credits_check(struct rte_sched_port *port, uint32_t pos)
1898 {
1899         struct rte_sched_grinder *grinder = port->grinder + pos;
1900         struct rte_sched_subport *subport = grinder->subport;
1901         struct rte_sched_pipe *pipe = grinder->pipe;
1902         struct rte_mbuf *pkt = grinder->pkt;
1903         uint32_t tc_index = grinder->tc_index;
1904         uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
1905         uint32_t subport_tb_credits = subport->tb_credits;
1906         uint32_t subport_tc_credits = subport->tc_credits[tc_index];
1907         uint32_t pipe_tb_credits = pipe->tb_credits;
1908         uint32_t pipe_tc_credits = pipe->tc_credits[tc_index];
1909         uint32_t pipe_tc_ov_mask1[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE];
1910         uint32_t pipe_tc_ov_mask2[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE] = {0};
1911         uint32_t pipe_tc_ov_credits, i;
1912         int enough_credits;
1913
1914         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
1915                 pipe_tc_ov_mask1[i] = UINT32_MAX;
1916
1917         pipe_tc_ov_mask1[RTE_SCHED_TRAFFIC_CLASS_BE] = pipe->tc_ov_credits;
1918         pipe_tc_ov_mask2[RTE_SCHED_TRAFFIC_CLASS_BE] = UINT32_MAX;
1919         pipe_tc_ov_credits = pipe_tc_ov_mask1[tc_index];
1920
1921         /* Check pipe and subport credits */
1922         enough_credits = (pkt_len <= subport_tb_credits) &&
1923                 (pkt_len <= subport_tc_credits) &&
1924                 (pkt_len <= pipe_tb_credits) &&
1925                 (pkt_len <= pipe_tc_credits) &&
1926                 (pkt_len <= pipe_tc_ov_credits);
1927
1928         if (!enough_credits)
1929                 return 0;
1930
1931         /* Update pipe and subport credits */
1932         subport->tb_credits -= pkt_len;
1933         subport->tc_credits[tc_index] -= pkt_len;
1934         pipe->tb_credits -= pkt_len;
1935         pipe->tc_credits[tc_index] -= pkt_len;
1936         pipe->tc_ov_credits -= pipe_tc_ov_mask2[tc_index] & pkt_len;
1937
1938         return 1;
1939 }
1940
1941 #endif /* RTE_SCHED_SUBPORT_TC_OV */
1942
1943
1944 static inline int
1945 grinder_schedule(struct rte_sched_port *port, uint32_t pos)
1946 {
1947         struct rte_sched_grinder *grinder = port->grinder + pos;
1948         struct rte_sched_queue *queue = grinder->queue[grinder->qpos];
1949         struct rte_mbuf *pkt = grinder->pkt;
1950         uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
1951         uint32_t be_tc_active;
1952
1953         if (!grinder_credits_check(port, pos))
1954                 return 0;
1955
1956         /* Advance port time */
1957         port->time += pkt_len;
1958
1959         /* Send packet */
1960         port->pkts_out[port->n_pkts_out++] = pkt;
1961         queue->qr++;
1962
1963         be_tc_active = (grinder->tc_index == RTE_SCHED_TRAFFIC_CLASS_BE) ? ~0x0 : 0x0;
1964         grinder->wrr_tokens[grinder->qpos] +=
1965                 (pkt_len * grinder->wrr_cost[grinder->qpos]) & be_tc_active;
1966
1967         if (queue->qr == queue->qw) {
1968                 uint32_t qindex = grinder->qindex[grinder->qpos];
1969
1970                 rte_bitmap_clear(port->bmp, qindex);
1971                 grinder->qmask &= ~(1 << grinder->qpos);
1972                 if (be_tc_active)
1973                         grinder->wrr_mask[grinder->qpos] = 0;
1974                 rte_sched_port_set_queue_empty_timestamp(port, qindex);
1975         }
1976
1977         /* Reset pipe loop detection */
1978         port->pipe_loop = RTE_SCHED_PIPE_INVALID;
1979         grinder->productive = 1;
1980
1981         return 1;
1982 }
1983
1984 #ifdef SCHED_VECTOR_SSE4
1985
1986 static inline int
1987 grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe)
1988 {
1989         __m128i index = _mm_set1_epi32(base_pipe);
1990         __m128i pipes = _mm_load_si128((__m128i *)port->grinder_base_bmp_pos);
1991         __m128i res = _mm_cmpeq_epi32(pipes, index);
1992
1993         pipes = _mm_load_si128((__m128i *)(port->grinder_base_bmp_pos + 4));
1994         pipes = _mm_cmpeq_epi32(pipes, index);
1995         res = _mm_or_si128(res, pipes);
1996
1997         if (_mm_testz_si128(res, res))
1998                 return 0;
1999
2000         return 1;
2001 }
2002
2003 #elif defined(SCHED_VECTOR_NEON)
2004
2005 static inline int
2006 grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe)
2007 {
2008         uint32x4_t index, pipes;
2009         uint32_t *pos = (uint32_t *)port->grinder_base_bmp_pos;
2010
2011         index = vmovq_n_u32(base_pipe);
2012         pipes = vld1q_u32(pos);
2013         if (!vminvq_u32(veorq_u32(pipes, index)))
2014                 return 1;
2015
2016         pipes = vld1q_u32(pos + 4);
2017         if (!vminvq_u32(veorq_u32(pipes, index)))
2018                 return 1;
2019
2020         return 0;
2021 }
2022
2023 #else
2024
2025 static inline int
2026 grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe)
2027 {
2028         uint32_t i;
2029
2030         for (i = 0; i < RTE_SCHED_PORT_N_GRINDERS; i++) {
2031                 if (port->grinder_base_bmp_pos[i] == base_pipe)
2032                         return 1;
2033         }
2034
2035         return 0;
2036 }
2037
2038 #endif /* RTE_SCHED_OPTIMIZATIONS */
2039
2040 static inline void
2041 grinder_pcache_populate(struct rte_sched_port *port, uint32_t pos, uint32_t bmp_pos, uint64_t bmp_slab)
2042 {
2043         struct rte_sched_grinder *grinder = port->grinder + pos;
2044         uint16_t w[4];
2045
2046         grinder->pcache_w = 0;
2047         grinder->pcache_r = 0;
2048
2049         w[0] = (uint16_t) bmp_slab;
2050         w[1] = (uint16_t) (bmp_slab >> 16);
2051         w[2] = (uint16_t) (bmp_slab >> 32);
2052         w[3] = (uint16_t) (bmp_slab >> 48);
2053
2054         grinder->pcache_qmask[grinder->pcache_w] = w[0];
2055         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos;
2056         grinder->pcache_w += (w[0] != 0);
2057
2058         grinder->pcache_qmask[grinder->pcache_w] = w[1];
2059         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos + 16;
2060         grinder->pcache_w += (w[1] != 0);
2061
2062         grinder->pcache_qmask[grinder->pcache_w] = w[2];
2063         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos + 32;
2064         grinder->pcache_w += (w[2] != 0);
2065
2066         grinder->pcache_qmask[grinder->pcache_w] = w[3];
2067         grinder->pcache_qindex[grinder->pcache_w] = bmp_pos + 48;
2068         grinder->pcache_w += (w[3] != 0);
2069 }
2070
2071 static inline void
2072 grinder_tccache_populate(struct rte_sched_port *port, uint32_t pos, uint32_t qindex, uint16_t qmask)
2073 {
2074         struct rte_sched_grinder *grinder = port->grinder + pos;
2075         uint8_t b, i;
2076
2077         grinder->tccache_w = 0;
2078         grinder->tccache_r = 0;
2079
2080         for (i = 0; i < RTE_SCHED_TRAFFIC_CLASS_BE; i++) {
2081                 b = (uint8_t) ((qmask >> i) & 0x1);
2082                 grinder->tccache_qmask[grinder->tccache_w] = b;
2083                 grinder->tccache_qindex[grinder->tccache_w] = qindex + i;
2084                 grinder->tccache_w += (b != 0);
2085         }
2086
2087         b = (uint8_t) (qmask >> (RTE_SCHED_TRAFFIC_CLASS_BE));
2088         grinder->tccache_qmask[grinder->tccache_w] = b;
2089         grinder->tccache_qindex[grinder->tccache_w] = qindex +
2090                 RTE_SCHED_TRAFFIC_CLASS_BE;
2091         grinder->tccache_w += (b != 0);
2092 }
2093
2094 static inline int
2095 grinder_next_tc(struct rte_sched_port *port, uint32_t pos)
2096 {
2097         struct rte_sched_grinder *grinder = port->grinder + pos;
2098         struct rte_mbuf **qbase;
2099         uint32_t qindex;
2100         uint16_t qsize;
2101
2102         if (grinder->tccache_r == grinder->tccache_w)
2103                 return 0;
2104
2105         qindex = grinder->tccache_qindex[grinder->tccache_r];
2106         qbase = rte_sched_port_qbase(port, qindex);
2107         qsize = rte_sched_port_qsize(port, qindex);
2108
2109         grinder->tc_index = rte_sched_port_pipe_tc(port, qindex);
2110         grinder->qmask = grinder->tccache_qmask[grinder->tccache_r];
2111         grinder->qsize = qsize;
2112
2113         if (grinder->tc_index < RTE_SCHED_TRAFFIC_CLASS_BE) {
2114                 grinder->queue[0] = port->queue + qindex;
2115                 grinder->qbase[0] = qbase;
2116                 grinder->qindex[0] = qindex;
2117                 grinder->tccache_r++;
2118
2119                 return 1;
2120         }
2121
2122         grinder->queue[0] = port->queue + qindex;
2123         grinder->queue[1] = port->queue + qindex + 1;
2124         grinder->queue[2] = port->queue + qindex + 2;
2125         grinder->queue[3] = port->queue + qindex + 3;
2126
2127         grinder->qbase[0] = qbase;
2128         grinder->qbase[1] = qbase + qsize;
2129         grinder->qbase[2] = qbase + 2 * qsize;
2130         grinder->qbase[3] = qbase + 3 * qsize;
2131
2132         grinder->qindex[0] = qindex;
2133         grinder->qindex[1] = qindex + 1;
2134         grinder->qindex[2] = qindex + 2;
2135         grinder->qindex[3] = qindex + 3;
2136
2137         grinder->tccache_r++;
2138         return 1;
2139 }
2140
2141 static inline int
2142 grinder_next_pipe(struct rte_sched_port *port, uint32_t pos)
2143 {
2144         struct rte_sched_grinder *grinder = port->grinder + pos;
2145         uint32_t pipe_qindex;
2146         uint16_t pipe_qmask;
2147
2148         if (grinder->pcache_r < grinder->pcache_w) {
2149                 pipe_qmask = grinder->pcache_qmask[grinder->pcache_r];
2150                 pipe_qindex = grinder->pcache_qindex[grinder->pcache_r];
2151                 grinder->pcache_r++;
2152         } else {
2153                 uint64_t bmp_slab = 0;
2154                 uint32_t bmp_pos = 0;
2155
2156                 /* Get another non-empty pipe group */
2157                 if (unlikely(rte_bitmap_scan(port->bmp, &bmp_pos, &bmp_slab) <= 0))
2158                         return 0;
2159
2160 #ifdef RTE_SCHED_DEBUG
2161                 debug_check_queue_slab(port, bmp_pos, bmp_slab);
2162 #endif
2163
2164                 /* Return if pipe group already in one of the other grinders */
2165                 port->grinder_base_bmp_pos[pos] = RTE_SCHED_BMP_POS_INVALID;
2166                 if (unlikely(grinder_pipe_exists(port, bmp_pos)))
2167                         return 0;
2168
2169                 port->grinder_base_bmp_pos[pos] = bmp_pos;
2170
2171                 /* Install new pipe group into grinder's pipe cache */
2172                 grinder_pcache_populate(port, pos, bmp_pos, bmp_slab);
2173
2174                 pipe_qmask = grinder->pcache_qmask[0];
2175                 pipe_qindex = grinder->pcache_qindex[0];
2176                 grinder->pcache_r = 1;
2177         }
2178
2179         /* Install new pipe in the grinder */
2180         grinder->pindex = pipe_qindex >> 4;
2181         grinder->subport = port->subport + (grinder->pindex / port->n_pipes_per_subport);
2182         grinder->pipe = port->pipe + grinder->pindex;
2183         grinder->pipe_params = NULL; /* to be set after the pipe structure is prefetched */
2184         grinder->productive = 0;
2185
2186         grinder_tccache_populate(port, pos, pipe_qindex, pipe_qmask);
2187         grinder_next_tc(port, pos);
2188
2189         /* Check for pipe exhaustion */
2190         if (grinder->pindex == port->pipe_loop) {
2191                 port->pipe_exhaustion = 1;
2192                 port->pipe_loop = RTE_SCHED_PIPE_INVALID;
2193         }
2194
2195         return 1;
2196 }
2197
2198
2199 static inline void
2200 grinder_wrr_load(struct rte_sched_port *port, uint32_t pos)
2201 {
2202         struct rte_sched_grinder *grinder = port->grinder + pos;
2203         struct rte_sched_pipe *pipe = grinder->pipe;
2204         struct rte_sched_pipe_profile *pipe_params = grinder->pipe_params;
2205         uint32_t qmask = grinder->qmask;
2206
2207         grinder->wrr_tokens[0] =
2208                 ((uint16_t) pipe->wrr_tokens[0]) << RTE_SCHED_WRR_SHIFT;
2209         grinder->wrr_tokens[1] =
2210                 ((uint16_t) pipe->wrr_tokens[1]) << RTE_SCHED_WRR_SHIFT;
2211         grinder->wrr_tokens[2] =
2212                 ((uint16_t) pipe->wrr_tokens[2]) << RTE_SCHED_WRR_SHIFT;
2213         grinder->wrr_tokens[3] =
2214                 ((uint16_t) pipe->wrr_tokens[3]) << RTE_SCHED_WRR_SHIFT;
2215
2216         grinder->wrr_mask[0] = (qmask & 0x1) * 0xFFFF;
2217         grinder->wrr_mask[1] = ((qmask >> 1) & 0x1) * 0xFFFF;
2218         grinder->wrr_mask[2] = ((qmask >> 2) & 0x1) * 0xFFFF;
2219         grinder->wrr_mask[3] = ((qmask >> 3) & 0x1) * 0xFFFF;
2220
2221         grinder->wrr_cost[0] = pipe_params->wrr_cost[0];
2222         grinder->wrr_cost[1] = pipe_params->wrr_cost[1];
2223         grinder->wrr_cost[2] = pipe_params->wrr_cost[2];
2224         grinder->wrr_cost[3] = pipe_params->wrr_cost[3];
2225 }
2226
2227 static inline void
2228 grinder_wrr_store(struct rte_sched_port *port, uint32_t pos)
2229 {
2230         struct rte_sched_grinder *grinder = port->grinder + pos;
2231         struct rte_sched_pipe *pipe = grinder->pipe;
2232
2233         pipe->wrr_tokens[0] =
2234                         (grinder->wrr_tokens[0] & grinder->wrr_mask[0]) >>
2235                                 RTE_SCHED_WRR_SHIFT;
2236         pipe->wrr_tokens[1] =
2237                         (grinder->wrr_tokens[1] & grinder->wrr_mask[1]) >>
2238                                 RTE_SCHED_WRR_SHIFT;
2239         pipe->wrr_tokens[2] =
2240                         (grinder->wrr_tokens[2] & grinder->wrr_mask[2]) >>
2241                                 RTE_SCHED_WRR_SHIFT;
2242         pipe->wrr_tokens[3] =
2243                         (grinder->wrr_tokens[3] & grinder->wrr_mask[3]) >>
2244                                 RTE_SCHED_WRR_SHIFT;
2245 }
2246
2247 static inline void
2248 grinder_wrr(struct rte_sched_port *port, uint32_t pos)
2249 {
2250         struct rte_sched_grinder *grinder = port->grinder + pos;
2251         uint16_t wrr_tokens_min;
2252
2253         grinder->wrr_tokens[0] |= ~grinder->wrr_mask[0];
2254         grinder->wrr_tokens[1] |= ~grinder->wrr_mask[1];
2255         grinder->wrr_tokens[2] |= ~grinder->wrr_mask[2];
2256         grinder->wrr_tokens[3] |= ~grinder->wrr_mask[3];
2257
2258         grinder->qpos = rte_min_pos_4_u16(grinder->wrr_tokens);
2259         wrr_tokens_min = grinder->wrr_tokens[grinder->qpos];
2260
2261         grinder->wrr_tokens[0] -= wrr_tokens_min;
2262         grinder->wrr_tokens[1] -= wrr_tokens_min;
2263         grinder->wrr_tokens[2] -= wrr_tokens_min;
2264         grinder->wrr_tokens[3] -= wrr_tokens_min;
2265 }
2266
2267
2268 #define grinder_evict(port, pos)
2269
2270 static inline void
2271 grinder_prefetch_pipe(struct rte_sched_port *port, uint32_t pos)
2272 {
2273         struct rte_sched_grinder *grinder = port->grinder + pos;
2274
2275         rte_prefetch0(grinder->pipe);
2276         rte_prefetch0(grinder->queue[0]);
2277 }
2278
2279 static inline void
2280 grinder_prefetch_tc_queue_arrays(struct rte_sched_port *port, uint32_t pos)
2281 {
2282         struct rte_sched_grinder *grinder = port->grinder + pos;
2283         uint16_t qsize, qr[RTE_SCHED_MAX_QUEUES_PER_TC];
2284
2285         qsize = grinder->qsize;
2286         grinder->qpos = 0;
2287
2288         if (grinder->tc_index < RTE_SCHED_TRAFFIC_CLASS_BE) {
2289                 qr[0] = grinder->queue[0]->qr & (qsize - 1);
2290
2291                 rte_prefetch0(grinder->qbase[0] + qr[0]);
2292                 return;
2293         }
2294
2295         qr[0] = grinder->queue[0]->qr & (qsize - 1);
2296         qr[1] = grinder->queue[1]->qr & (qsize - 1);
2297         qr[2] = grinder->queue[2]->qr & (qsize - 1);
2298         qr[3] = grinder->queue[3]->qr & (qsize - 1);
2299
2300         rte_prefetch0(grinder->qbase[0] + qr[0]);
2301         rte_prefetch0(grinder->qbase[1] + qr[1]);
2302
2303         grinder_wrr_load(port, pos);
2304         grinder_wrr(port, pos);
2305
2306         rte_prefetch0(grinder->qbase[2] + qr[2]);
2307         rte_prefetch0(grinder->qbase[3] + qr[3]);
2308 }
2309
2310 static inline void
2311 grinder_prefetch_mbuf(struct rte_sched_port *port, uint32_t pos)
2312 {
2313         struct rte_sched_grinder *grinder = port->grinder + pos;
2314         uint32_t qpos = grinder->qpos;
2315         struct rte_mbuf **qbase = grinder->qbase[qpos];
2316         uint16_t qsize = grinder->qsize;
2317         uint16_t qr = grinder->queue[qpos]->qr & (qsize - 1);
2318
2319         grinder->pkt = qbase[qr];
2320         rte_prefetch0(grinder->pkt);
2321
2322         if (unlikely((qr & 0x7) == 7)) {
2323                 uint16_t qr_next = (grinder->queue[qpos]->qr + 1) & (qsize - 1);
2324
2325                 rte_prefetch0(qbase + qr_next);
2326         }
2327 }
2328
2329 static inline uint32_t
2330 grinder_handle(struct rte_sched_port *port, uint32_t pos)
2331 {
2332         struct rte_sched_grinder *grinder = port->grinder + pos;
2333
2334         switch (grinder->state) {
2335         case e_GRINDER_PREFETCH_PIPE:
2336         {
2337                 if (grinder_next_pipe(port, pos)) {
2338                         grinder_prefetch_pipe(port, pos);
2339                         port->busy_grinders++;
2340
2341                         grinder->state = e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS;
2342                         return 0;
2343                 }
2344
2345                 return 0;
2346         }
2347
2348         case e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS:
2349         {
2350                 struct rte_sched_pipe *pipe = grinder->pipe;
2351
2352                 grinder->pipe_params = port->pipe_profiles + pipe->profile;
2353                 grinder_prefetch_tc_queue_arrays(port, pos);
2354                 grinder_credits_update(port, pos);
2355
2356                 grinder->state = e_GRINDER_PREFETCH_MBUF;
2357                 return 0;
2358         }
2359
2360         case e_GRINDER_PREFETCH_MBUF:
2361         {
2362                 grinder_prefetch_mbuf(port, pos);
2363
2364                 grinder->state = e_GRINDER_READ_MBUF;
2365                 return 0;
2366         }
2367
2368         case e_GRINDER_READ_MBUF:
2369         {
2370                 uint32_t wrr_active, result = 0;
2371
2372                 result = grinder_schedule(port, pos);
2373
2374                 wrr_active = (grinder->tc_index == RTE_SCHED_TRAFFIC_CLASS_BE);
2375
2376                 /* Look for next packet within the same TC */
2377                 if (result && grinder->qmask) {
2378                         if (wrr_active)
2379                                 grinder_wrr(port, pos);
2380
2381                         grinder_prefetch_mbuf(port, pos);
2382
2383                         return 1;
2384                 }
2385
2386                 if (wrr_active)
2387                         grinder_wrr_store(port, pos);
2388
2389                 /* Look for another active TC within same pipe */
2390                 if (grinder_next_tc(port, pos)) {
2391                         grinder_prefetch_tc_queue_arrays(port, pos);
2392
2393                         grinder->state = e_GRINDER_PREFETCH_MBUF;
2394                         return result;
2395                 }
2396
2397                 if (grinder->productive == 0 &&
2398                     port->pipe_loop == RTE_SCHED_PIPE_INVALID)
2399                         port->pipe_loop = grinder->pindex;
2400
2401                 grinder_evict(port, pos);
2402
2403                 /* Look for another active pipe */
2404                 if (grinder_next_pipe(port, pos)) {
2405                         grinder_prefetch_pipe(port, pos);
2406
2407                         grinder->state = e_GRINDER_PREFETCH_TC_QUEUE_ARRAYS;
2408                         return result;
2409                 }
2410
2411                 /* No active pipe found */
2412                 port->busy_grinders--;
2413
2414                 grinder->state = e_GRINDER_PREFETCH_PIPE;
2415                 return result;
2416         }
2417
2418         default:
2419                 rte_panic("Algorithmic error (invalid state)\n");
2420                 return 0;
2421         }
2422 }
2423
2424 static inline void
2425 rte_sched_port_time_resync(struct rte_sched_port *port)
2426 {
2427         uint64_t cycles = rte_get_tsc_cycles();
2428         uint64_t cycles_diff = cycles - port->time_cpu_cycles;
2429         uint64_t bytes_diff;
2430
2431         /* Compute elapsed time in bytes */
2432         bytes_diff = rte_reciprocal_divide(cycles_diff << RTE_SCHED_TIME_SHIFT,
2433                                            port->inv_cycles_per_byte);
2434
2435         /* Advance port time */
2436         port->time_cpu_cycles = cycles;
2437         port->time_cpu_bytes += bytes_diff;
2438         if (port->time < port->time_cpu_bytes)
2439                 port->time = port->time_cpu_bytes;
2440
2441         /* Reset pipe loop detection */
2442         port->pipe_loop = RTE_SCHED_PIPE_INVALID;
2443 }
2444
2445 static inline int
2446 rte_sched_port_exceptions(struct rte_sched_port *port, int second_pass)
2447 {
2448         int exceptions;
2449
2450         /* Check if any exception flag is set */
2451         exceptions = (second_pass && port->busy_grinders == 0) ||
2452                 (port->pipe_exhaustion == 1);
2453
2454         /* Clear exception flags */
2455         port->pipe_exhaustion = 0;
2456
2457         return exceptions;
2458 }
2459
2460 int
2461 rte_sched_port_dequeue(struct rte_sched_port *port, struct rte_mbuf **pkts, uint32_t n_pkts)
2462 {
2463         uint32_t i, count;
2464
2465         port->pkts_out = pkts;
2466         port->n_pkts_out = 0;
2467
2468         rte_sched_port_time_resync(port);
2469
2470         /* Take each queue in the grinder one step further */
2471         for (i = 0, count = 0; ; i++)  {
2472                 count += grinder_handle(port, i & (RTE_SCHED_PORT_N_GRINDERS - 1));
2473                 if ((count == n_pkts) ||
2474                     rte_sched_port_exceptions(port, i >= RTE_SCHED_PORT_N_GRINDERS)) {
2475                         break;
2476                 }
2477         }
2478
2479         return count;
2480 }