X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_sched%2Frte_sched.c;h=7252f850d58d8f36e7ebcaa80a5804067d09dce1;hb=e32cb57973fc311b4b5f60ae5dac37d99e48c94d;hp=89007ad11743a6ec63f847ba6c68d66ce089bbf6;hpb=4d51afb5cdb668d8fa9737f3a3665c14b868a5d0;p=dpdk.git diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c index 89007ad117..7252f850d5 100644 --- a/lib/librte_sched/rte_sched.c +++ b/lib/librte_sched/rte_sched.c @@ -42,11 +42,12 @@ #include #include #include +#include #include "rte_sched.h" -#include "rte_bitmap.h" #include "rte_sched_common.h" #include "rte_approx.h" +#include "rte_reciprocal.h" #ifdef __INTEL_COMPILER #pragma warning(disable:2259) /* conversion may lose significant bits */ @@ -55,8 +56,10 @@ #ifdef RTE_SCHED_VECTOR #include -#if defined(__SSE4__) +#ifdef RTE_ARCH_X86 #define SCHED_VECTOR_SSE4 +#elif defined(RTE_MACHINE_CPUFLAG_NEON) +#define SCHED_VECTOR_NEON #endif #endif @@ -67,6 +70,11 @@ #define RTE_SCHED_PIPE_INVALID UINT32_MAX #define RTE_SCHED_BMP_POS_INVALID UINT32_MAX +/* Scaling for cycles_per_byte calculation + * Chosen so that minimum rate is 480 bit/sec + */ +#define RTE_SCHED_TIME_SHIFT 8 + struct rte_sched_subport { /* Token bucket (TB) */ uint64_t tb_time; /* time of last update */ @@ -220,7 +228,7 @@ struct rte_sched_port { uint64_t time_cpu_cycles; /* Current CPU time measured in CPU cyles */ uint64_t time_cpu_bytes; /* Current CPU time measured in bytes */ uint64_t time; /* Current NIC TX time measured in bytes */ - double cycles_per_byte; /* CPU cycles per byte */ + struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */ /* Scheduling loop detection */ uint32_t pipe_loop; @@ -615,7 +623,7 @@ struct rte_sched_port * rte_sched_port_config(struct rte_sched_port_params *params) { struct rte_sched_port *port = NULL; - uint32_t mem_size, bmp_mem_size, n_queues_per_port, i; + uint32_t mem_size, bmp_mem_size, n_queues_per_port, i, cycles_per_byte; /* Check user parameters. Determine the amount of memory to allocate */ mem_size = rte_sched_port_get_memory_footprint(params); @@ -666,7 +674,10 @@ rte_sched_port_config(struct rte_sched_port_params *params) port->time_cpu_cycles = rte_get_tsc_cycles(); port->time_cpu_bytes = 0; port->time = 0; - port->cycles_per_byte = ((double) rte_get_tsc_hz()) / ((double) params->rate); + + cycles_per_byte = (rte_get_tsc_hz() << RTE_SCHED_TIME_SHIFT) + / params->rate; + port->inv_cycles_per_byte = rte_reciprocal_value(cycles_per_byte); /* Scheduling loop detection */ port->pipe_loop = RTE_SCHED_PIPE_INVALID; @@ -725,19 +736,25 @@ rte_sched_port_config(struct rte_sched_port_params *params) void rte_sched_port_free(struct rte_sched_port *port) { - unsigned int queue; + uint32_t qindex; + uint32_t n_queues_per_port; /* Check user parameters */ if (port == NULL) return; - /* Free enqueued mbufs */ - for (queue = 0; queue < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; queue++) { - struct rte_mbuf **mbufs = rte_sched_port_qbase(port, queue); - unsigned int i; + n_queues_per_port = rte_sched_port_queues_per_port(port); - for (i = 0; i < rte_sched_port_qsize(port, queue); i++) - rte_pktmbuf_free(mbufs[i]); + /* Free enqueued mbufs */ + for (qindex = 0; qindex < n_queues_per_port; qindex++) { + struct rte_mbuf **mbufs = rte_sched_port_qbase(port, qindex); + uint16_t qsize = rte_sched_port_qsize(port, qindex); + struct rte_sched_queue *queue = port->queue + qindex; + uint16_t qr = queue->qr & (qsize - 1); + uint16_t qw = queue->qw & (qsize - 1); + + for (; qr != qw; qr = (qr + 1) & (qsize - 1)) + rte_pktmbuf_free(mbufs[qr]); } rte_bitmap_free(port->bmp); @@ -1003,7 +1020,7 @@ rte_sched_subport_read_stats(struct rte_sched_port *port, memcpy(stats, &s->stats, sizeof(struct rte_sched_subport_stats)); memset(&s->stats, 0, sizeof(struct rte_sched_subport_stats)); - /* Subport TC ovesubscription status */ + /* Subport TC oversubscription status */ *tc_ov = s->tc_ov; return 0; @@ -1075,10 +1092,17 @@ rte_sched_port_update_subport_stats(struct rte_sched_port *port, uint32_t qindex s->stats.n_bytes_tc[tc_index] += pkt_len; } +#ifdef RTE_SCHED_RED +static inline void +rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port, + uint32_t qindex, + struct rte_mbuf *pkt, uint32_t red) +#else static inline void rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port, - uint32_t qindex, - struct rte_mbuf *pkt, uint32_t red) + uint32_t qindex, + struct rte_mbuf *pkt, __rte_unused uint32_t red) +#endif { struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port)); uint32_t tc_index = (qindex >> 2) & 0x3; @@ -1101,10 +1125,17 @@ rte_sched_port_update_queue_stats(struct rte_sched_port *port, uint32_t qindex, qe->stats.n_bytes += pkt_len; } +#ifdef RTE_SCHED_RED +static inline void +rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port, + uint32_t qindex, + struct rte_mbuf *pkt, uint32_t red) +#else static inline void rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port, - uint32_t qindex, - struct rte_mbuf *pkt, uint32_t red) + uint32_t qindex, + struct rte_mbuf *pkt, __rte_unused uint32_t red) +#endif { struct rte_sched_queue_extra *qe = port->queue_extra + qindex; uint32_t pkt_len = pkt->pkt_len; @@ -1703,6 +1734,26 @@ grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe) return 1; } +#elif defined(SCHED_VECTOR_NEON) + +static inline int +grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe) +{ + uint32x4_t index, pipes; + uint32_t *pos = (uint32_t *)port->grinder_base_bmp_pos; + + index = vmovq_n_u32(base_pipe); + pipes = vld1q_u32(pos); + if (!vminvq_u32(veorq_u32(pipes, index))) + return 1; + + pipes = vld1q_u32(pos + 4); + if (!vminvq_u32(veorq_u32(pipes, index))) + return 1; + + return 0; +} + #else static inline int @@ -2093,11 +2144,15 @@ rte_sched_port_time_resync(struct rte_sched_port *port) { uint64_t cycles = rte_get_tsc_cycles(); uint64_t cycles_diff = cycles - port->time_cpu_cycles; - double bytes_diff = ((double) cycles_diff) / port->cycles_per_byte; + uint64_t bytes_diff; + + /* Compute elapsed time in bytes */ + bytes_diff = rte_reciprocal_divide(cycles_diff << RTE_SCHED_TIME_SHIFT, + port->inv_cycles_per_byte); /* Advance port time */ port->time_cpu_cycles = cycles; - port->time_cpu_bytes += (uint64_t) bytes_diff; + port->time_cpu_bytes += bytes_diff; if (port->time < port->time_cpu_bytes) port->time = port->time_cpu_bytes;