#pragma warning(disable:2259) /* conversion may lose significant bits */
#endif
-#ifndef RTE_SCHED_DEBUG
-#define RTE_SCHED_DEBUG 0
-#endif
-
#ifndef RTE_SCHED_OPTIMIZATIONS
#define RTE_SCHED_OPTIMIZATIONS 0
#endif
#include <immintrin.h>
#endif
-#define RTE_SCHED_ENQUEUE 1
-
-#define RTE_SCHED_TS 1
-
-#if RTE_SCHED_TS == 0 /* Infinite credits. Traffic shaping disabled. */
-#define RTE_SCHED_TS_CREDITS_UPDATE 0
-#define RTE_SCHED_TS_CREDITS_CHECK 0
-#else /* Real Credits. Full traffic shaping implemented. */
-#define RTE_SCHED_TS_CREDITS_UPDATE 1
-#define RTE_SCHED_TS_CREDITS_CHECK 1
-#endif
-
#ifndef RTE_SCHED_TB_RATE_CONFIG_ERR
#define RTE_SCHED_TB_RATE_CONFIG_ERR (1e-7)
#endif
-#define RTE_SCHED_WRR 1
-
#ifndef RTE_SCHED_WRR_SHIFT
#define RTE_SCHED_WRR_SHIFT 3
#endif
e_GRINDER_READ_MBUF
};
+/*
+ * Path through the scheduler hierarchy used by the scheduler enqueue
+ * operation to identify the destination queue for the current
+ * packet. Stored in the field pkt.hash.sched of struct rte_mbuf of
+ * each packet, typically written by the classification stage and read
+ * by scheduler enqueue.
+ */
+struct rte_sched_port_hierarchy {
+ uint32_t queue:2; /**< Queue ID (0 .. 3) */
+ uint32_t traffic_class:2; /**< Traffic class ID (0 .. 3)*/
+ uint32_t pipe:20; /**< Pipe ID */
+ uint32_t subport:6; /**< Subport ID */
+ uint32_t color:2; /**< Color */
+};
+
struct rte_sched_grinder {
/* Pipe cache */
uint16_t pcache_qmask[RTE_SCHED_GRINDER_PCACHE_SIZE];
return -7;
}
- /* qsize: non-zero, power of 2, no bigger than 32K (due to 16-bit read/write pointers) */
+ /* qsize: non-zero, power of 2,
+ * no bigger than 32K (due to 16-bit read/write pointers) */
for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i ++) {
uint16_t qsize = params->qsize[i];
base = 0;
if (array == e_RTE_SCHED_PORT_ARRAY_SUBPORT) return base;
- base += CACHE_LINE_ROUNDUP(size_subport);
+ base += RTE_CACHE_LINE_ROUNDUP(size_subport);
if (array == e_RTE_SCHED_PORT_ARRAY_PIPE) return base;
- base += CACHE_LINE_ROUNDUP(size_pipe);
+ base += RTE_CACHE_LINE_ROUNDUP(size_pipe);
if (array == e_RTE_SCHED_PORT_ARRAY_QUEUE) return base;
- base += CACHE_LINE_ROUNDUP(size_queue);
+ base += RTE_CACHE_LINE_ROUNDUP(size_queue);
if (array == e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA) return base;
- base += CACHE_LINE_ROUNDUP(size_queue_extra);
+ base += RTE_CACHE_LINE_ROUNDUP(size_queue_extra);
if (array == e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES) return base;
- base += CACHE_LINE_ROUNDUP(size_pipe_profiles);
+ base += RTE_CACHE_LINE_ROUNDUP(size_pipe_profiles);
if (array == e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY) return base;
- base += CACHE_LINE_ROUNDUP(size_bmp_array);
+ base += RTE_CACHE_LINE_ROUNDUP(size_bmp_array);
if (array == e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY) return base;
- base += CACHE_LINE_ROUNDUP(size_queue_array);
+ base += RTE_CACHE_LINE_ROUNDUP(size_queue_array);
return base;
}
status = rte_sched_port_check_params(params);
if (status != 0) {
- RTE_LOG(INFO, SCHED, "Port scheduler params check failed (%d)\n", status);
+ RTE_LOG(NOTICE, SCHED,
+ "Port scheduler params check failed (%d)\n", status);
return 0;
}
{
struct rte_sched_pipe_profile *p = port->pipe_profiles + i;
- RTE_LOG(INFO, SCHED, "Low level config for pipe profile %u:\n"
- "\tToken bucket: period = %u, credits per period = %u, size = %u\n"
- "\tTraffic classes: period = %u, credits per period = [%u, %u, %u, %u]\n"
- "\tTraffic class 3 oversubscription: weight = %hhu\n"
- "\tWRR cost: [%hhu, %hhu, %hhu, %hhu], [%hhu, %hhu, %hhu, %hhu], [%hhu, %hhu, %hhu, %hhu], [%hhu, %hhu, %hhu, %hhu]\n",
+ RTE_LOG(DEBUG, SCHED, "Low level config for pipe profile %u:\n"
+ " Token bucket: period = %u, credits per period = %u, size = %u\n"
+ " Traffic classes: period = %u, credits per period = [%u, %u, %u, %u]\n"
+ " Traffic class 3 oversubscription: weight = %hhu\n"
+ " WRR cost: [%hhu, %hhu, %hhu, %hhu], [%hhu, %hhu, %hhu, %hhu], [%hhu, %hhu, %hhu, %hhu], [%hhu, %hhu, %hhu, %hhu]\n",
i,
/* Token bucket */
}
/* Allocate memory to store the data structures */
- port = rte_zmalloc("qos_params", mem_size, CACHE_LINE_SIZE);
+ port = rte_zmalloc("qos_params", mem_size, RTE_CACHE_LINE_SIZE);
if (port == NULL) {
return NULL;
}
uint32_t j;
for (j = 0; j < e_RTE_METER_COLORS; j++) {
+ /* if min/max are both zero, then RED is disabled */
+ if ((params->red_params[i][j].min_th |
+ params->red_params[i][j].max_th) == 0) {
+ continue;
+ }
+
if (rte_red_config_init(&port->red_config[i][j],
params->red_params[i][j].wq_log2,
params->red_params[i][j].min_th,
bmp_mem_size = rte_bitmap_get_memory_footprint(n_queues_per_port);
port->bmp = rte_bitmap_init(n_queues_per_port, port->bmp_array, bmp_mem_size);
if (port->bmp == NULL) {
- RTE_LOG(INFO, SCHED, "Bitmap init error\n");
+ RTE_LOG(ERR, SCHED, "Bitmap init error\n");
return NULL;
}
for (i = 0; i < RTE_SCHED_PORT_N_GRINDERS; i ++) {
{
struct rte_sched_subport *s = port->subport + i;
- RTE_LOG(INFO, SCHED, "Low level config for subport %u:\n"
- "\tToken bucket: period = %u, credits per period = %u, size = %u\n"
- "\tTraffic classes: period = %u, credits per period = [%u, %u, %u, %u]\n"
- "\tTraffic class 3 oversubscription: wm min = %u, wm max = %u\n",
+ RTE_LOG(DEBUG, SCHED, "Low level config for subport %u:\n"
+ " Token bucket: period = %u, credits per period = %u, size = %u\n"
+ " Traffic classes: period = %u, credits per period = [%u, %u, %u, %u]\n"
+ " Traffic class 3 oversubscription: wm min = %u, wm max = %u\n",
i,
/* Token bucket */
s->tc_ov = s->tc_ov_rate > subport_tc3_rate;
if (s->tc_ov != tc3_ov) {
- RTE_LOG(INFO, SCHED, "Subport %u TC3 oversubscription is OFF (%.4lf >= %.4lf)\n",
+ RTE_LOG(DEBUG, SCHED,
+ "Subport %u TC3 oversubscription is OFF (%.4lf >= %.4lf)\n",
subport_id, subport_tc3_rate, s->tc_ov_rate);
}
#endif
s->tc_ov = s->tc_ov_rate > subport_tc3_rate;
if (s->tc_ov != tc3_ov) {
- RTE_LOG(INFO, SCHED, "Subport %u TC3 oversubscription is ON (%.4lf < %.4lf)\n",
+ RTE_LOG(DEBUG, SCHED,
+ "Subport %u TC3 oversubscription is ON (%.4lf < %.4lf)\n",
subport_id, subport_tc3_rate, s->tc_ov_rate);
}
p->tc_ov_period_id = s->tc_ov_period_id;
return 0;
}
+void
+rte_sched_port_pkt_write(struct rte_mbuf *pkt,
+ uint32_t subport, uint32_t pipe, uint32_t traffic_class,
+ uint32_t queue, enum rte_meter_color color)
+{
+ struct rte_sched_port_hierarchy *sched
+ = (struct rte_sched_port_hierarchy *) &pkt->hash.sched;
+
+ sched->color = (uint32_t) color;
+ sched->subport = subport;
+ sched->pipe = pipe;
+ sched->traffic_class = traffic_class;
+ sched->queue = queue;
+}
+
+void
+rte_sched_port_pkt_read_tree_path(const struct rte_mbuf *pkt,
+ uint32_t *subport, uint32_t *pipe,
+ uint32_t *traffic_class, uint32_t *queue)
+{
+ const struct rte_sched_port_hierarchy *sched
+ = (const struct rte_sched_port_hierarchy *) &pkt->hash.sched;
+
+ *subport = sched->subport;
+ *pipe = sched->pipe;
+ *traffic_class = sched->traffic_class;
+ *queue = sched->queue;
+}
+
+
+enum rte_meter_color
+rte_sched_port_pkt_read_color(const struct rte_mbuf *pkt)
+{
+ const struct rte_sched_port_hierarchy *sched
+ = (const struct rte_sched_port_hierarchy *) &pkt->hash.sched;
+
+ return (enum rte_meter_color) sched->color;
+}
+
int
rte_sched_subport_read_stats(struct rte_sched_port *port,
uint32_t subport_id,
return port->qsize[tc];
}
-#if RTE_SCHED_DEBUG
+#ifdef RTE_SCHED_DEBUG
static inline int
rte_sched_port_queue_is_empty(struct rte_sched_port *port, uint32_t qindex)
{
struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port));
uint32_t tc_index = (qindex >> 2) & 0x3;
- uint32_t pkt_len = pkt->pkt.pkt_len;
+ uint32_t pkt_len = pkt->pkt_len;
s->stats.n_pkts_tc[tc_index] += 1;
s->stats.n_bytes_tc[tc_index] += pkt_len;
{
struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port));
uint32_t tc_index = (qindex >> 2) & 0x3;
- uint32_t pkt_len = pkt->pkt.pkt_len;
+ uint32_t pkt_len = pkt->pkt_len;
s->stats.n_pkts_tc_dropped[tc_index] += 1;
s->stats.n_bytes_tc_dropped[tc_index] += pkt_len;
rte_sched_port_update_queue_stats(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
{
struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
- uint32_t pkt_len = pkt->pkt.pkt_len;
+ uint32_t pkt_len = pkt->pkt_len;
qe->stats.n_pkts += 1;
qe->stats.n_bytes += pkt_len;
rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
{
struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
- uint32_t pkt_len = pkt->pkt.pkt_len;
+ uint32_t pkt_len = pkt->pkt_len;
qe->stats.n_pkts_dropped += 1;
qe->stats.n_bytes_dropped += pkt_len;
color = rte_sched_port_pkt_read_color(pkt);
red_cfg = &port->red_config[tc_index][color];
+ if ((red_cfg->min_th | red_cfg->max_th) == 0)
+ return 0;
+
qe = port->queue_extra + qindex;
red = &qe->red;
#endif /* RTE_SCHED_RED */
-#if RTE_SCHED_DEBUG
+#ifdef RTE_SCHED_DEBUG
static inline int
debug_pipe_is_empty(struct rte_sched_port *port, uint32_t pindex)
return 1;
}
-#if RTE_SCHED_ENQUEUE == 0
-
-int
-rte_sched_port_enqueue(struct rte_sched_port *port, struct rte_mbuf **pkts, uint32_t n_pkts)
-{
- uint32_t result, i;
- result = 0;
-
- for (i = 0; i < n_pkts; i ++) {
- struct rte_mbuf *pkt;
- struct rte_mbuf **q_base;
- uint32_t subport, pipe, traffic_class, queue, qindex;
-
- pkt = pkts[i];
-
- rte_sched_port_pkt_read_tree_path(pkt, &subport, &pipe, &traffic_class, &queue);
-
- qindex = rte_sched_port_qindex(port, subport, pipe, traffic_class, queue);
-
- q_base = rte_sched_port_qbase(port, qindex);
-
- result += rte_sched_port_enqueue_qwa(port, qindex, q_base, pkt);
- }
-
- return result;
-}
-
-#else
-
-/* The enqueue function implements a 4-level pipeline with each stage processing
+/*
+ * The enqueue function implements a 4-level pipeline with each stage processing
* two different packets. The purpose of using a pipeline is to hide the latency
* of prefetching the data structures. The naming convention is presented in the
* diagram below:
* ----->|_______|----->|_______|----->|_______|----->|_______|----->
* p01 p11 p21 p31
*
- ***/
+ */
int
rte_sched_port_enqueue(struct rte_sched_port *port, struct rte_mbuf **pkts, uint32_t n_pkts)
{
return result;
}
-#endif /* RTE_SCHED_ENQUEUE */
-
-#if RTE_SCHED_TS_CREDITS_UPDATE == 0
-
-#define grinder_credits_update(port, pos)
-
-#elif !defined(RTE_SCHED_SUBPORT_TC_OV)
+#ifndef RTE_SCHED_SUBPORT_TC_OV
static inline void
grinder_credits_update(struct rte_sched_port *port, uint32_t pos)
#endif /* RTE_SCHED_TS_CREDITS_UPDATE, RTE_SCHED_SUBPORT_TC_OV */
-#if RTE_SCHED_TS_CREDITS_CHECK
#ifndef RTE_SCHED_SUBPORT_TC_OV
struct rte_sched_pipe *pipe = grinder->pipe;
struct rte_mbuf *pkt = grinder->pkt;
uint32_t tc_index = grinder->tc_index;
- uint32_t pkt_len = pkt->pkt.pkt_len + port->frame_overhead;
+ uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
uint32_t subport_tb_credits = subport->tb_credits;
uint32_t subport_tc_credits = subport->tc_credits[tc_index];
uint32_t pipe_tb_credits = pipe->tb_credits;
struct rte_sched_pipe *pipe = grinder->pipe;
struct rte_mbuf *pkt = grinder->pkt;
uint32_t tc_index = grinder->tc_index;
- uint32_t pkt_len = pkt->pkt.pkt_len + port->frame_overhead;
+ uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
uint32_t subport_tb_credits = subport->tb_credits;
uint32_t subport_tc_credits = subport->tc_credits[tc_index];
uint32_t pipe_tb_credits = pipe->tb_credits;
#endif /* RTE_SCHED_SUBPORT_TC_OV */
-#endif /* RTE_SCHED_TS_CREDITS_CHECK */
static inline int
grinder_schedule(struct rte_sched_port *port, uint32_t pos)
struct rte_sched_grinder *grinder = port->grinder + pos;
struct rte_sched_queue *queue = grinder->queue[grinder->qpos];
struct rte_mbuf *pkt = grinder->pkt;
- uint32_t pkt_len = pkt->pkt.pkt_len + port->frame_overhead;
+ uint32_t pkt_len = pkt->pkt_len + port->frame_overhead;
-#if RTE_SCHED_TS_CREDITS_CHECK
if (!grinder_credits_check(port, pos)) {
return 0;
}
-#endif
/* Advance port time */
port->time += pkt_len;
return 0;
}
-#if RTE_SCHED_DEBUG
+#ifdef RTE_SCHED_DEBUG
debug_check_queue_slab(port, bmp_pos, bmp_slab);
#endif
return 1;
}
-#if RTE_SCHED_WRR == 0
-
-#define grinder_wrr_load(a,b)
-
-#define grinder_wrr_store(a,b)
-
-static inline void
-grinder_wrr(struct rte_sched_port *port, uint32_t pos)
-{
- struct rte_sched_grinder *grinder = port->grinder + pos;
- uint64_t slab = grinder->qmask;
-
- if (rte_bsf64(slab, &grinder->qpos) == 0) {
- rte_panic("grinder wrr\n");
- }
-}
-
-#elif RTE_SCHED_WRR == 1
static inline void
grinder_wrr_load(struct rte_sched_port *port, uint32_t pos)
grinder->wrr_tokens[3] -= wrr_tokens_min;
}
-#else
-
-#error Invalid value for RTE_SCHED_WRR
-
-#endif /* RTE_SCHED_WRR */
#define grinder_evict(port, pos)
git.droids-corp.org / dpdk.git / blobdiff