X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_sched%2Frte_sched.c;h=614705d81ed8cfbac7cf25484da86bd483d61601;hb=f28ede500c2ed5f14cb5aa549ea1116c4e04c5f7;hp=ff471982143eec33c42567ad6b758bc018ead95a;hpb=51fa9d616dff232f42f3905a4ff16f94f25838b0;p=dpdk.git

diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c
index ff47198214..614705d81e 100644
--- a/lib/librte_sched/rte_sched.c
+++ b/lib/librte_sched/rte_sched.c
@@ -47,13 +47,19 @@
 #include "rte_bitmap.h"
 #include "rte_sched_common.h"
 #include "rte_approx.h"
+#include "rte_reciprocal.h"
 
 #ifdef __INTEL_COMPILER
 #pragma warning(disable:2259) /* conversion may lose significant bits */
 #endif
 
 #ifdef RTE_SCHED_VECTOR
-#include <immintrin.h>
+#include <rte_vect.h>
+
+#if defined(__SSE4__)
+#define SCHED_VECTOR_SSE4
+#endif
+
 #endif
 
 #define RTE_SCHED_TB_RATE_CONFIG_ERR          (1e-7)
@@ -62,6 +68,11 @@
 #define RTE_SCHED_PIPE_INVALID                UINT32_MAX
 #define RTE_SCHED_BMP_POS_INVALID             UINT32_MAX
 
+/* Scaling for cycles_per_byte calculation
+ * Chosen so that minimum rate is 480 bit/sec
+ */
+#define RTE_SCHED_TIME_SHIFT		      8
+
 struct rte_sched_subport {
 	/* Token bucket (TB) */
 	uint64_t tb_time; /* time of last update */
@@ -152,11 +163,12 @@ enum grinder_state {
  * by scheduler enqueue.
  */
 struct rte_sched_port_hierarchy {
-	uint32_t queue:2;                /**< Queue ID (0 .. 3) */
-	uint32_t traffic_class:2;        /**< Traffic class ID (0 .. 3)*/
-	uint32_t pipe:20;                /**< Pipe ID */
-	uint32_t subport:6;              /**< Subport ID */
+	uint16_t queue:2;                /**< Queue ID (0 .. 3) */
+	uint16_t traffic_class:2;        /**< Traffic class ID (0 .. 3)*/
 	uint32_t color:2;                /**< Color */
+	uint16_t unused:10;
+	uint16_t subport;                /**< Subport ID */
+	uint32_t pipe;		         /**< Pipe ID */
 };
 
 struct rte_sched_grinder {
@@ -214,7 +226,7 @@ struct rte_sched_port {
 	uint64_t time_cpu_cycles;     /* Current CPU time measured in CPU cyles */
 	uint64_t time_cpu_bytes;      /* Current CPU time measured in bytes */
 	uint64_t time;                /* Current NIC TX time measured in bytes */
-	double cycles_per_byte;       /* CPU cycles per byte */
+	struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */
 
 	/* Scheduling loop detection */
 	uint32_t pipe_loop;
@@ -272,6 +284,24 @@ rte_sched_port_queues_per_port(struct rte_sched_port *port)
 	return RTE_SCHED_QUEUES_PER_PIPE * port->n_pipes_per_subport * port->n_subports_per_port;
 }
 
+static inline struct rte_mbuf **
+rte_sched_port_qbase(struct rte_sched_port *port, uint32_t qindex)
+{
+	uint32_t pindex = qindex >> 4;
+	uint32_t qpos = qindex & 0xF;
+
+	return (port->queue_array + pindex *
+		port->qsize_sum + port->qsize_add[qpos]);
+}
+
+static inline uint16_t
+rte_sched_port_qsize(struct rte_sched_port *port, uint32_t qindex)
+{
+	uint32_t tc = (qindex >> 2) & 0x3;
+
+	return port->qsize[tc];
+}
+
 static int
 rte_sched_port_check_params(struct rte_sched_port_params *params)
 {
@@ -292,8 +322,9 @@ rte_sched_port_check_params(struct rte_sched_port_params *params)
 	if (params->mtu == 0)
 		return -5;
 
-	/* n_subports_per_port: non-zero, power of 2 */
+	/* n_subports_per_port: non-zero, limited to 16 bits, power of 2 */
 	if (params->n_subports_per_port == 0 ||
+	    params->n_subports_per_port > 1u << 16 ||
 	    !rte_is_power_of_2(params->n_subports_per_port))
 		return -6;
 
@@ -432,7 +463,7 @@ rte_sched_port_get_memory_footprint(struct rte_sched_port_params *params)
 	size0 = sizeof(struct rte_sched_port);
 	size1 = rte_sched_port_get_array_base(params, e_RTE_SCHED_PORT_ARRAY_TOTAL);
 
-	return (size0 + size1);
+	return size0 + size1;
 }
 
 static void
@@ -590,7 +621,7 @@ struct rte_sched_port *
 rte_sched_port_config(struct rte_sched_port_params *params)
 {
 	struct rte_sched_port *port = NULL;
-	uint32_t mem_size, bmp_mem_size, n_queues_per_port, i;
+	uint32_t mem_size, bmp_mem_size, n_queues_per_port, i, cycles_per_byte;
 
 	/* Check user parameters. Determine the amount of memory to allocate */
 	mem_size = rte_sched_port_get_memory_footprint(params);
@@ -641,7 +672,10 @@ rte_sched_port_config(struct rte_sched_port_params *params)
 	port->time_cpu_cycles = rte_get_tsc_cycles();
 	port->time_cpu_bytes = 0;
 	port->time = 0;
-	port->cycles_per_byte = ((double) rte_get_tsc_hz()) / ((double) params->rate);
+
+	cycles_per_byte = (rte_get_tsc_hz() << RTE_SCHED_TIME_SHIFT)
+		/ params->rate;
+	port->inv_cycles_per_byte = rte_reciprocal_value(cycles_per_byte);
 
 	/* Scheduling loop detection */
 	port->pipe_loop = RTE_SCHED_PIPE_INVALID;
@@ -700,10 +734,27 @@ rte_sched_port_config(struct rte_sched_port_params *params)
 void
 rte_sched_port_free(struct rte_sched_port *port)
 {
+	uint32_t qindex;
+	uint32_t n_queues_per_port;
+
 	/* Check user parameters */
 	if (port == NULL)
 		return;
 
+	n_queues_per_port = rte_sched_port_queues_per_port(port);
+
+	/* Free enqueued mbufs */
+	for (qindex = 0; qindex < n_queues_per_port; qindex++) {
+		struct rte_mbuf **mbufs = rte_sched_port_qbase(port, qindex);
+		uint16_t qsize = rte_sched_port_qsize(port, qindex);
+		struct rte_sched_queue *queue = port->queue + qindex;
+		uint16_t qr = queue->qr & (qsize - 1);
+		uint16_t qw = queue->qw & (qsize - 1);
+
+		for (; qr != qw; qr = (qr + 1) & (qsize - 1))
+			rte_pktmbuf_free(mbufs[qr]);
+	}
+
 	rte_bitmap_free(port->bmp);
 	rte_free(port);
 }
@@ -916,6 +967,8 @@ rte_sched_port_pkt_write(struct rte_mbuf *pkt,
 	struct rte_sched_port_hierarchy *sched
 		= (struct rte_sched_port_hierarchy *) &pkt->hash.sched;
 
+	RTE_BUILD_BUG_ON(sizeof(*sched) > sizeof(pkt->hash.sched));
+
 	sched->color = (uint32_t) color;
 	sched->subport = subport;
 	sched->pipe = pipe;
@@ -1012,23 +1065,6 @@ rte_sched_port_qindex(struct rte_sched_port *port, uint32_t subport, uint32_t pi
 	return result;
 }
 
-static inline struct rte_mbuf **
-rte_sched_port_qbase(struct rte_sched_port *port, uint32_t qindex)
-{
-	uint32_t pindex = qindex >> 4;
-	uint32_t qpos = qindex & 0xF;
-
-	return (port->queue_array + pindex * port->qsize_sum + port->qsize_add[qpos]);
-}
-
-static inline uint16_t
-rte_sched_port_qsize(struct rte_sched_port *port, uint32_t qindex)
-{
-	uint32_t tc = (qindex >> 2) & 0x3;
-
-	return port->qsize[tc];
-}
-
 #ifdef RTE_SCHED_DEBUG
 
 static inline int
@@ -1036,17 +1072,7 @@ rte_sched_port_queue_is_empty(struct rte_sched_port *port, uint32_t qindex)
 {
 	struct rte_sched_queue *queue = port->queue + qindex;
 
-	return (queue->qr == queue->qw);
-}
-
-static inline int
-rte_sched_port_queue_is_full(struct rte_sched_port *port, uint32_t qindex)
-{
-	struct rte_sched_queue *queue = port->queue + qindex;
-	uint16_t qsize = rte_sched_port_qsize(port, qindex);
-	uint16_t qlen = queue->qw - queue->qr;
-
-	return (qlen >= qsize);
+	return queue->qr == queue->qw;
 }
 
 #endif /* RTE_SCHED_DEBUG */
@@ -1064,8 +1090,17 @@ rte_sched_port_update_subport_stats(struct rte_sched_port *port, uint32_t qindex
 	s->stats.n_bytes_tc[tc_index] += pkt_len;
 }
 
+#ifdef RTE_SCHED_RED
+static inline void
+rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port,
+						uint32_t qindex,
+						struct rte_mbuf *pkt, uint32_t red)
+#else
 static inline void
-rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
+rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port,
+						uint32_t qindex,
+						struct rte_mbuf *pkt, __rte_unused uint32_t red)
+#endif
 {
 	struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port));
 	uint32_t tc_index = (qindex >> 2) & 0x3;
@@ -1073,6 +1108,9 @@ rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port, uint32_
 
 	s->stats.n_pkts_tc_dropped[tc_index] += 1;
 	s->stats.n_bytes_tc_dropped[tc_index] += pkt_len;
+#ifdef RTE_SCHED_RED
+	s->stats.n_pkts_red_dropped[tc_index] += red;
+#endif
 }
 
 static inline void
@@ -1085,14 +1123,26 @@ rte_sched_port_update_queue_stats(struct rte_sched_port *port, uint32_t qindex,
 	qe->stats.n_bytes += pkt_len;
 }
 
+#ifdef RTE_SCHED_RED
 static inline void
-rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
+rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port,
+						uint32_t qindex,
+						struct rte_mbuf *pkt, uint32_t red)
+#else
+static inline void
+rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port,
+						uint32_t qindex,
+						struct rte_mbuf *pkt, __rte_unused uint32_t red)
+#endif
 {
 	struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
 	uint32_t pkt_len = pkt->pkt_len;
 
 	qe->stats.n_pkts_dropped += 1;
 	qe->stats.n_bytes_dropped += pkt_len;
+#ifdef RTE_SCHED_RED
+	qe->stats.n_pkts_red_dropped += red;
+#endif
 }
 
 #endif /* RTE_SCHED_COLLECT_STATS */
@@ -1140,27 +1190,6 @@ rte_sched_port_set_queue_empty_timestamp(struct rte_sched_port *port, uint32_t q
 
 #ifdef RTE_SCHED_DEBUG
 
-static inline int
-debug_pipe_is_empty(struct rte_sched_port *port, uint32_t pindex)
-{
-	uint32_t qindex, i;
-
-	qindex = pindex << 4;
-
-	for (i = 0; i < 16; i++) {
-		uint32_t queue_empty = rte_sched_port_queue_is_empty(port, qindex + i);
-		uint32_t bmp_bit_clear = (rte_bitmap_get(port->bmp, qindex + i) == 0);
-
-		if (queue_empty != bmp_bit_clear)
-			rte_panic("Queue status mismatch for queue %u of pipe %u\n", i, pindex);
-
-		if (!queue_empty)
-			return 0;
-	}
-
-	return 1;
-}
-
 static inline void
 debug_check_queue_slab(struct rte_sched_port *port, uint32_t bmp_pos,
 		       uint64_t bmp_slab)
@@ -1244,8 +1273,10 @@ rte_sched_port_enqueue_qwa(struct rte_sched_port *port, uint32_t qindex,
 		     (qlen >= qsize))) {
 		rte_pktmbuf_free(pkt);
 #ifdef RTE_SCHED_COLLECT_STATS
-		rte_sched_port_update_subport_stats_on_drop(port, qindex, pkt);
-		rte_sched_port_update_queue_stats_on_drop(port, qindex, pkt);
+		rte_sched_port_update_subport_stats_on_drop(port, qindex, pkt,
+							    qlen < qsize);
+		rte_sched_port_update_queue_stats_on_drop(port, qindex, pkt,
+							  qlen < qsize);
 #endif
 		return 0;
 	}
@@ -1682,7 +1713,7 @@ grinder_schedule(struct rte_sched_port *port, uint32_t pos)
 	return 1;
 }
 
-#ifdef RTE_SCHED_VECTOR
+#ifdef SCHED_VECTOR_SSE4
 
 static inline int
 grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe)
@@ -2091,11 +2122,15 @@ rte_sched_port_time_resync(struct rte_sched_port *port)
 {
 	uint64_t cycles = rte_get_tsc_cycles();
 	uint64_t cycles_diff = cycles - port->time_cpu_cycles;
-	double bytes_diff = ((double) cycles_diff) / port->cycles_per_byte;
+	uint64_t bytes_diff;
+
+	/* Compute elapsed time in bytes */
+	bytes_diff = rte_reciprocal_divide(cycles_diff << RTE_SCHED_TIME_SHIFT,
+					   port->inv_cycles_per_byte);
 
 	/* Advance port time */
 	port->time_cpu_cycles = cycles;
-	port->time_cpu_bytes += (uint64_t) bytes_diff;
+	port->time_cpu_bytes += bytes_diff;
 	if (port->time < port->time_cpu_bytes)
 		port->time = port->time_cpu_bytes;