X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_sched%2Frte_sched.c;h=b7cba110f6a2edb810f6918fb09d6361b10a9436;hb=ff5d5b01f8f21a87387042843b8494bc2d82b1cd;hp=15cd297bbdb7beb019a73ee5062e0566a08699be;hpb=d9d15a2eab4bfcbbace9f23ba7ab72e453a1b21f;p=dpdk.git diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c index 15cd297bbd..b7cba110f6 100644 --- a/lib/librte_sched/rte_sched.c +++ b/lib/librte_sched/rte_sched.c @@ -47,13 +47,21 @@ #include "rte_bitmap.h" #include "rte_sched_common.h" #include "rte_approx.h" +#include "rte_reciprocal.h" #ifdef __INTEL_COMPILER #pragma warning(disable:2259) /* conversion may lose significant bits */ #endif #ifdef RTE_SCHED_VECTOR -#include +#include + +#ifdef RTE_ARCH_X86 +#define SCHED_VECTOR_SSE4 +#elif defined(RTE_MACHINE_CPUFLAG_NEON) +#define SCHED_VECTOR_NEON +#endif + #endif #define RTE_SCHED_TB_RATE_CONFIG_ERR (1e-7) @@ -62,6 +70,11 @@ #define RTE_SCHED_PIPE_INVALID UINT32_MAX #define RTE_SCHED_BMP_POS_INVALID UINT32_MAX +/* Scaling for cycles_per_byte calculation + * Chosen so that minimum rate is 480 bit/sec + */ +#define RTE_SCHED_TIME_SHIFT 8 + struct rte_sched_subport { /* Token bucket (TB) */ uint64_t tb_time; /* time of last update */ @@ -215,7 +228,7 @@ struct rte_sched_port { uint64_t time_cpu_cycles; /* Current CPU time measured in CPU cyles */ uint64_t time_cpu_bytes; /* Current CPU time measured in bytes */ uint64_t time; /* Current NIC TX time measured in bytes */ - double cycles_per_byte; /* CPU cycles per byte */ + struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */ /* Scheduling loop detection */ uint32_t pipe_loop; @@ -273,6 +286,24 @@ rte_sched_port_queues_per_port(struct rte_sched_port *port) return RTE_SCHED_QUEUES_PER_PIPE * port->n_pipes_per_subport * port->n_subports_per_port; } +static inline struct rte_mbuf ** +rte_sched_port_qbase(struct rte_sched_port *port, uint32_t qindex) +{ + uint32_t pindex = qindex >> 4; + uint32_t qpos = qindex & 0xF; + + return (port->queue_array + pindex * + port->qsize_sum + port->qsize_add[qpos]); +} + +static inline uint16_t +rte_sched_port_qsize(struct rte_sched_port *port, uint32_t qindex) +{ + uint32_t tc = (qindex >> 2) & 0x3; + + return port->qsize[tc]; +} + static int rte_sched_port_check_params(struct rte_sched_port_params *params) { @@ -434,7 +465,7 @@ rte_sched_port_get_memory_footprint(struct rte_sched_port_params *params) size0 = sizeof(struct rte_sched_port); size1 = rte_sched_port_get_array_base(params, e_RTE_SCHED_PORT_ARRAY_TOTAL); - return (size0 + size1); + return size0 + size1; } static void @@ -592,7 +623,7 @@ struct rte_sched_port * rte_sched_port_config(struct rte_sched_port_params *params) { struct rte_sched_port *port = NULL; - uint32_t mem_size, bmp_mem_size, n_queues_per_port, i; + uint32_t mem_size, bmp_mem_size, n_queues_per_port, i, cycles_per_byte; /* Check user parameters. Determine the amount of memory to allocate */ mem_size = rte_sched_port_get_memory_footprint(params); @@ -643,7 +674,10 @@ rte_sched_port_config(struct rte_sched_port_params *params) port->time_cpu_cycles = rte_get_tsc_cycles(); port->time_cpu_bytes = 0; port->time = 0; - port->cycles_per_byte = ((double) rte_get_tsc_hz()) / ((double) params->rate); + + cycles_per_byte = (rte_get_tsc_hz() << RTE_SCHED_TIME_SHIFT) + / params->rate; + port->inv_cycles_per_byte = rte_reciprocal_value(cycles_per_byte); /* Scheduling loop detection */ port->pipe_loop = RTE_SCHED_PIPE_INVALID; @@ -702,10 +736,27 @@ rte_sched_port_config(struct rte_sched_port_params *params) void rte_sched_port_free(struct rte_sched_port *port) { + uint32_t qindex; + uint32_t n_queues_per_port; + /* Check user parameters */ if (port == NULL) return; + n_queues_per_port = rte_sched_port_queues_per_port(port); + + /* Free enqueued mbufs */ + for (qindex = 0; qindex < n_queues_per_port; qindex++) { + struct rte_mbuf **mbufs = rte_sched_port_qbase(port, qindex); + uint16_t qsize = rte_sched_port_qsize(port, qindex); + struct rte_sched_queue *queue = port->queue + qindex; + uint16_t qr = queue->qr & (qsize - 1); + uint16_t qw = queue->qw & (qsize - 1); + + for (; qr != qw; qr = (qr + 1) & (qsize - 1)) + rte_pktmbuf_free(mbufs[qr]); + } + rte_bitmap_free(port->bmp); rte_free(port); } @@ -1016,23 +1067,6 @@ rte_sched_port_qindex(struct rte_sched_port *port, uint32_t subport, uint32_t pi return result; } -static inline struct rte_mbuf ** -rte_sched_port_qbase(struct rte_sched_port *port, uint32_t qindex) -{ - uint32_t pindex = qindex >> 4; - uint32_t qpos = qindex & 0xF; - - return (port->queue_array + pindex * port->qsize_sum + port->qsize_add[qpos]); -} - -static inline uint16_t -rte_sched_port_qsize(struct rte_sched_port *port, uint32_t qindex) -{ - uint32_t tc = (qindex >> 2) & 0x3; - - return port->qsize[tc]; -} - #ifdef RTE_SCHED_DEBUG static inline int @@ -1040,17 +1074,7 @@ rte_sched_port_queue_is_empty(struct rte_sched_port *port, uint32_t qindex) { struct rte_sched_queue *queue = port->queue + qindex; - return (queue->qr == queue->qw); -} - -static inline int -rte_sched_port_queue_is_full(struct rte_sched_port *port, uint32_t qindex) -{ - struct rte_sched_queue *queue = port->queue + qindex; - uint16_t qsize = rte_sched_port_qsize(port, qindex); - uint16_t qlen = queue->qw - queue->qr; - - return (qlen >= qsize); + return queue->qr == queue->qw; } #endif /* RTE_SCHED_DEBUG */ @@ -1068,8 +1092,17 @@ rte_sched_port_update_subport_stats(struct rte_sched_port *port, uint32_t qindex s->stats.n_bytes_tc[tc_index] += pkt_len; } +#ifdef RTE_SCHED_RED static inline void -rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt) +rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port, + uint32_t qindex, + struct rte_mbuf *pkt, uint32_t red) +#else +static inline void +rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port, + uint32_t qindex, + struct rte_mbuf *pkt, __rte_unused uint32_t red) +#endif { struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port)); uint32_t tc_index = (qindex >> 2) & 0x3; @@ -1077,6 +1110,9 @@ rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port, uint32_ s->stats.n_pkts_tc_dropped[tc_index] += 1; s->stats.n_bytes_tc_dropped[tc_index] += pkt_len; +#ifdef RTE_SCHED_RED + s->stats.n_pkts_red_dropped[tc_index] += red; +#endif } static inline void @@ -1089,14 +1125,26 @@ rte_sched_port_update_queue_stats(struct rte_sched_port *port, uint32_t qindex, qe->stats.n_bytes += pkt_len; } +#ifdef RTE_SCHED_RED static inline void -rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt) +rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port, + uint32_t qindex, + struct rte_mbuf *pkt, uint32_t red) +#else +static inline void +rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port, + uint32_t qindex, + struct rte_mbuf *pkt, __rte_unused uint32_t red) +#endif { struct rte_sched_queue_extra *qe = port->queue_extra + qindex; uint32_t pkt_len = pkt->pkt_len; qe->stats.n_pkts_dropped += 1; qe->stats.n_bytes_dropped += pkt_len; +#ifdef RTE_SCHED_RED + qe->stats.n_pkts_red_dropped += red; +#endif } #endif /* RTE_SCHED_COLLECT_STATS */ @@ -1144,27 +1192,6 @@ rte_sched_port_set_queue_empty_timestamp(struct rte_sched_port *port, uint32_t q #ifdef RTE_SCHED_DEBUG -static inline int -debug_pipe_is_empty(struct rte_sched_port *port, uint32_t pindex) -{ - uint32_t qindex, i; - - qindex = pindex << 4; - - for (i = 0; i < 16; i++) { - uint32_t queue_empty = rte_sched_port_queue_is_empty(port, qindex + i); - uint32_t bmp_bit_clear = (rte_bitmap_get(port->bmp, qindex + i) == 0); - - if (queue_empty != bmp_bit_clear) - rte_panic("Queue status mismatch for queue %u of pipe %u\n", i, pindex); - - if (!queue_empty) - return 0; - } - - return 1; -} - static inline void debug_check_queue_slab(struct rte_sched_port *port, uint32_t bmp_pos, uint64_t bmp_slab) @@ -1248,8 +1275,10 @@ rte_sched_port_enqueue_qwa(struct rte_sched_port *port, uint32_t qindex, (qlen >= qsize))) { rte_pktmbuf_free(pkt); #ifdef RTE_SCHED_COLLECT_STATS - rte_sched_port_update_subport_stats_on_drop(port, qindex, pkt); - rte_sched_port_update_queue_stats_on_drop(port, qindex, pkt); + rte_sched_port_update_subport_stats_on_drop(port, qindex, pkt, + qlen < qsize); + rte_sched_port_update_queue_stats_on_drop(port, qindex, pkt, + qlen < qsize); #endif return 0; } @@ -1686,7 +1715,7 @@ grinder_schedule(struct rte_sched_port *port, uint32_t pos) return 1; } -#ifdef RTE_SCHED_VECTOR +#ifdef SCHED_VECTOR_SSE4 static inline int grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe) @@ -1705,6 +1734,26 @@ grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe) return 1; } +#elif defined(SCHED_VECTOR_NEON) + +static inline int +grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe) +{ + uint32x4_t index, pipes; + uint32_t *pos = (uint32_t *)port->grinder_base_bmp_pos; + + index = vmovq_n_u32(base_pipe); + pipes = vld1q_u32(pos); + if (!vminvq_u32(veorq_u32(pipes, index))) + return 1; + + pipes = vld1q_u32(pos + 4); + if (!vminvq_u32(veorq_u32(pipes, index))) + return 1; + + return 0; +} + #else static inline int @@ -2095,11 +2144,15 @@ rte_sched_port_time_resync(struct rte_sched_port *port) { uint64_t cycles = rte_get_tsc_cycles(); uint64_t cycles_diff = cycles - port->time_cpu_cycles; - double bytes_diff = ((double) cycles_diff) / port->cycles_per_byte; + uint64_t bytes_diff; + + /* Compute elapsed time in bytes */ + bytes_diff = rte_reciprocal_divide(cycles_diff << RTE_SCHED_TIME_SHIFT, + port->inv_cycles_per_byte); /* Advance port time */ port->time_cpu_cycles = cycles; - port->time_cpu_bytes += (uint64_t) bytes_diff; + port->time_cpu_bytes += bytes_diff; if (port->time < port->time_cpu_bytes) port->time = port->time_cpu_bytes;