X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_sched%2Frte_sched.c;h=c0983ddda4ae6b02e7ad9ca7573411b30ff4e7a9;hb=be848992298b7a64a161487600a4070e52b9d26d;hp=6eba69d9906b0fe7404ce5c96df0d60687e5bf95;hpb=4d2ad6e34b769dcaeac6b93e5dae7951337f0103;p=dpdk.git diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c index 6eba69d990..c0983ddda4 100644 --- a/lib/librte_sched/rte_sched.c +++ b/lib/librte_sched/rte_sched.c @@ -49,13 +49,13 @@ struct rte_sched_pipe_profile { /* Token bucket (TB) */ - uint32_t tb_period; - uint32_t tb_credits_per_period; - uint32_t tb_size; + uint64_t tb_period; + uint64_t tb_credits_per_period; + uint64_t tb_size; /* Pipe traffic classes */ - uint32_t tc_period; - uint32_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + uint64_t tc_period; + uint64_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; uint8_t tc_ov_weight; /* Pipe best-effort traffic class queues */ @@ -65,20 +65,20 @@ struct rte_sched_pipe_profile { struct rte_sched_pipe { /* Token bucket (TB) */ uint64_t tb_time; /* time of last update */ - uint32_t tb_credits; + uint64_t tb_credits; /* Pipe profile and flags */ uint32_t profile; /* Traffic classes (TCs) */ uint64_t tc_time; /* time of next update */ - uint32_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + uint64_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; /* Weighted Round Robin (WRR) */ uint8_t wrr_tokens[RTE_SCHED_BE_QUEUES_PER_PIPE]; /* TC oversubscription */ - uint32_t tc_ov_credits; + uint64_t tc_ov_credits; uint8_t tc_ov_period_id; } __rte_cache_aligned; @@ -141,28 +141,28 @@ struct rte_sched_grinder { struct rte_sched_subport { /* Token bucket (TB) */ uint64_t tb_time; /* time of last update */ - uint32_t tb_period; - uint32_t tb_credits_per_period; - uint32_t tb_size; - uint32_t tb_credits; + uint64_t tb_period; + uint64_t tb_credits_per_period; + uint64_t tb_size; + uint64_t tb_credits; /* Traffic classes (TCs) */ uint64_t tc_time; /* time of next update */ - uint32_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; - uint32_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; - uint32_t tc_period; + uint64_t tc_credits_per_period[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + uint64_t tc_credits[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + uint64_t tc_period; /* TC oversubscription */ - uint32_t tc_ov_wm; - uint32_t tc_ov_wm_min; - uint32_t tc_ov_wm_max; + uint64_t tc_ov_wm; + uint64_t tc_ov_wm_min; + uint64_t tc_ov_wm_max; uint8_t tc_ov_period_id; uint8_t tc_ov; uint32_t tc_ov_n; double tc_ov_rate; /* Statistics */ - struct rte_sched_subport_stats stats; + struct rte_sched_subport_stats stats __rte_cache_aligned; /* Subport pipes */ uint32_t n_pipes_per_subport_enabled; @@ -170,7 +170,7 @@ struct rte_sched_subport { uint32_t n_max_pipe_profiles; /* Pipe best-effort TC rate */ - uint32_t pipe_tc_be_rate_max; + uint64_t pipe_tc_be_rate_max; /* Pipe queues size */ uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; @@ -212,17 +212,10 @@ struct rte_sched_port { uint16_t pipe_queue[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; uint8_t pipe_tc[RTE_SCHED_QUEUES_PER_PIPE]; uint8_t tc_queue[RTE_SCHED_QUEUES_PER_PIPE]; - uint32_t rate; + uint64_t rate; uint32_t mtu; uint32_t frame_overhead; int socket; - uint16_t qsize[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; - uint32_t n_pipe_profiles; - uint32_t n_max_pipe_profiles; - uint32_t pipe_tc_be_rate_max; -#ifdef RTE_SCHED_RED - struct rte_red_config red_config[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE][RTE_COLORS]; -#endif /* Timing */ uint64_t time_cpu_cycles; /* Current CPU time measured in CPU cyles */ @@ -230,47 +223,15 @@ struct rte_sched_port { uint64_t time; /* Current NIC TX time measured in bytes */ struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */ - /* Scheduling loop detection */ - uint32_t pipe_loop; - uint32_t pipe_exhaustion; - - /* Bitmap */ - struct rte_bitmap *bmp; - uint32_t grinder_base_bmp_pos[RTE_SCHED_PORT_N_GRINDERS] __rte_aligned_16; - /* Grinders */ - struct rte_sched_grinder grinder[RTE_SCHED_PORT_N_GRINDERS]; - uint32_t busy_grinders; struct rte_mbuf **pkts_out; uint32_t n_pkts_out; - - /* Queue base calculation */ - uint32_t qsize_add[RTE_SCHED_QUEUES_PER_PIPE]; - uint32_t qsize_sum; + uint32_t subport_id; /* Large data structures */ - struct rte_sched_subport *subports[0]; - struct rte_sched_subport *subport; - struct rte_sched_pipe *pipe; - struct rte_sched_queue *queue; - struct rte_sched_queue_extra *queue_extra; - struct rte_sched_pipe_profile *pipe_profiles; - uint8_t *bmp_array; - struct rte_mbuf **queue_array; - uint8_t memory[0] __rte_cache_aligned; + struct rte_sched_subport *subports[0] __rte_cache_aligned; } __rte_cache_aligned; -enum rte_sched_port_array { - e_RTE_SCHED_PORT_ARRAY_SUBPORT = 0, - e_RTE_SCHED_PORT_ARRAY_PIPE, - e_RTE_SCHED_PORT_ARRAY_QUEUE, - e_RTE_SCHED_PORT_ARRAY_QUEUE_EXTRA, - e_RTE_SCHED_PORT_ARRAY_PIPE_PROFILES, - e_RTE_SCHED_PORT_ARRAY_BMP_ARRAY, - e_RTE_SCHED_PORT_ARRAY_QUEUE_ARRAY, - e_RTE_SCHED_PORT_ARRAY_TOTAL, -}; - enum rte_sched_subport_array { e_RTE_SCHED_SUBPORT_ARRAY_PIPE = 0, e_RTE_SCHED_SUBPORT_ARRAY_QUEUE, @@ -309,7 +270,12 @@ struct rte_sched_subport *subport, uint32_t qindex) static inline uint32_t rte_sched_port_queues_per_port(struct rte_sched_port *port) { - return RTE_SCHED_QUEUES_PER_PIPE * port->n_pipes_per_subport * port->n_subports_per_port; + uint32_t n_queues = 0, i; + + for (i = 0; i < port->n_subports_per_port; i++) + n_queues += rte_sched_subport_pipe_queues(port->subports[i]); + + return n_queues; } static inline uint16_t @@ -551,9 +517,11 @@ rte_sched_port_log_pipe_profile(struct rte_sched_subport *subport, uint32_t i) struct rte_sched_pipe_profile *p = subport->pipe_profiles + i; RTE_LOG(DEBUG, SCHED, "Low level config for pipe profile %u:\n" - " Token bucket: period = %u, credits per period = %u, size = %u\n" - " Traffic classes: period = %u,\n" - " credits per period = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u]\n" + " Token bucket: period = %"PRIu64", credits per period = %"PRIu64", size = %"PRIu64"\n" + " Traffic classes: period = %"PRIu64",\n" + " credits per period = [%"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64 + ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64 + ", %"PRIu64", %"PRIu64", %"PRIu64"]\n" " Best-effort traffic class oversubscription: weight = %hhu\n" " WRR cost: [%hhu, %hhu, %hhu, %hhu]\n", i, @@ -587,7 +555,7 @@ rte_sched_port_log_pipe_profile(struct rte_sched_subport *subport, uint32_t i) } static inline uint64_t -rte_sched_time_ms_to_bytes(uint32_t time_ms, uint32_t rate) +rte_sched_time_ms_to_bytes(uint64_t time_ms, uint64_t rate) { uint64_t time = time_ms; @@ -600,7 +568,7 @@ static void rte_sched_pipe_profile_convert(struct rte_sched_subport *subport, struct rte_sched_pipe_params *src, struct rte_sched_pipe_profile *dst, - uint32_t rate) + uint64_t rate) { uint32_t wrr_cost[RTE_SCHED_BE_QUEUES_PER_PIPE]; uint32_t lcd1, lcd2, lcd; @@ -615,8 +583,8 @@ rte_sched_pipe_profile_convert(struct rte_sched_subport *subport, / (double) rate; double d = RTE_SCHED_TB_RATE_CONFIG_ERR; - rte_approx(tb_rate, d, - &dst->tb_credits_per_period, &dst->tb_period); + rte_approx_64(tb_rate, d, &dst->tb_credits_per_period, + &dst->tb_period); } dst->tb_size = src->tb_size; @@ -671,7 +639,7 @@ rte_sched_subport_config_pipe_profile_table(struct rte_sched_subport *subport, subport->pipe_tc_be_rate_max = 0; for (i = 0; i < subport->n_pipe_profiles; i++) { struct rte_sched_pipe_params *src = params->pipe_profiles + i; - uint32_t pipe_tc_be_rate = src->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE]; + uint64_t pipe_tc_be_rate = src->tc_rate[RTE_SCHED_TRAFFIC_CLASS_BE]; if (subport->pipe_tc_be_rate_max < pipe_tc_be_rate) subport->pipe_tc_be_rate_max = pipe_tc_be_rate; @@ -681,7 +649,7 @@ rte_sched_subport_config_pipe_profile_table(struct rte_sched_subport *subport, static int rte_sched_subport_check_params(struct rte_sched_subport_params *params, uint32_t n_max_pipes_per_subport, - uint32_t rate) + uint64_t rate) { uint32_t i; @@ -718,7 +686,7 @@ rte_sched_subport_check_params(struct rte_sched_subport_params *params, } for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) { - uint32_t tc_rate = params->tc_rate[i]; + uint64_t tc_rate = params->tc_rate[i]; uint16_t qsize = params->qsize[i]; if ((qsize == 0 && tc_rate != 0) || @@ -888,6 +856,7 @@ rte_sched_port_config(struct rte_sched_port_params *params) /* Grinders */ port->pkts_out = NULL; port->n_pkts_out = 0; + port->subport_id = 0; return port; } @@ -943,10 +912,14 @@ rte_sched_port_log_subport_config(struct rte_sched_port *port, uint32_t i) struct rte_sched_subport *s = port->subports[i]; RTE_LOG(DEBUG, SCHED, "Low level config for subport %u:\n" - " Token bucket: period = %u, credits per period = %u, size = %u\n" - " Traffic classes: period = %u\n" - " credits per period = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u, %u]\n" - " Best effort traffic class oversubscription: wm min = %u, wm max = %u\n", + " Token bucket: period = %"PRIu64", credits per period = %"PRIu64 + ", size = %"PRIu64"\n" + " Traffic classes: period = %"PRIu64"\n" + " credits per period = [%"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64 + ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64 + ", %"PRIu64", %"PRIu64", %"PRIu64"]\n" + " Best effort traffic class oversubscription: wm min = %"PRIu64 + ", wm max = %"PRIu64"\n", i, /* Token bucket */ @@ -1056,7 +1029,7 @@ rte_sched_subport_config(struct rte_sched_port *port, double tb_rate = ((double) params->tb_rate) / ((double) port->rate); double d = RTE_SCHED_TB_RATE_CONFIG_ERR; - rte_approx(tb_rate, d, &s->tb_credits_per_period, &s->tb_period); + rte_approx_64(tb_rate, d, &s->tb_credits_per_period, &s->tb_period); } s->tb_size = params->tb_size; @@ -1462,8 +1435,10 @@ rte_sched_queue_read_stats(struct rte_sched_port *port, struct rte_sched_queue_stats *stats, uint16_t *qlen) { + struct rte_sched_subport *s; struct rte_sched_queue *q; struct rte_sched_queue_extra *qe; + uint32_t subport_id, subport_qmask, subport_qindex; /* Check user parameters */ if (port == NULL) { @@ -1489,8 +1464,13 @@ rte_sched_queue_read_stats(struct rte_sched_port *port, "%s: Incorrect value for parameter qlen\n", __func__); return -EINVAL; } - q = port->queue + queue_id; - qe = port->queue_extra + queue_id; + subport_qmask = port->n_pipes_per_subport_log2 + 4; + subport_id = (queue_id >> subport_qmask) & (port->n_subports_per_port - 1); + + s = port->subports[subport_id]; + subport_qindex = ((1 << subport_qmask) - 1) & queue_id; + q = s->queue + subport_qindex; + qe = s->queue_extra + subport_qindex; /* Copy queue stats and clear */ memcpy(stats, &qe->stats, sizeof(struct rte_sched_queue_stats)); @@ -1996,13 +1976,13 @@ grinder_credits_update(struct rte_sched_port *port, /* Subport TB */ n_periods = (port->time - subport->tb_time) / subport->tb_period; subport->tb_credits += n_periods * subport->tb_credits_per_period; - subport->tb_credits = rte_sched_min_val_2_u32(subport->tb_credits, subport->tb_size); + subport->tb_credits = RTE_MIN(subport->tb_credits, subport->tb_size); subport->tb_time += n_periods * subport->tb_period; /* Pipe TB */ n_periods = (port->time - pipe->tb_time) / params->tb_period; pipe->tb_credits += n_periods * params->tb_credits_per_period; - pipe->tb_credits = rte_sched_min_val_2_u32(pipe->tb_credits, params->tb_size); + pipe->tb_credits = RTE_MIN(pipe->tb_credits, params->tb_size); pipe->tb_time += n_periods * params->tb_period; /* Subport TCs */ @@ -2024,13 +2004,13 @@ grinder_credits_update(struct rte_sched_port *port, #else -static inline uint32_t +static inline uint64_t grinder_tc_ov_credits_update(struct rte_sched_port *port, struct rte_sched_subport *subport) { - uint32_t tc_ov_consumption[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; - uint32_t tc_consumption = 0, tc_ov_consumption_max; - uint32_t tc_ov_wm = subport->tc_ov_wm; + uint64_t tc_ov_consumption[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + uint64_t tc_consumption = 0, tc_ov_consumption_max; + uint64_t tc_ov_wm = subport->tc_ov_wm; uint32_t i; if (subport->tc_ov == 0) @@ -2079,13 +2059,13 @@ grinder_credits_update(struct rte_sched_port *port, /* Subport TB */ n_periods = (port->time - subport->tb_time) / subport->tb_period; subport->tb_credits += n_periods * subport->tb_credits_per_period; - subport->tb_credits = rte_sched_min_val_2_u32(subport->tb_credits, subport->tb_size); + subport->tb_credits = RTE_MIN(subport->tb_credits, subport->tb_size); subport->tb_time += n_periods * subport->tb_period; /* Pipe TB */ n_periods = (port->time - pipe->tb_time) / params->tb_period; pipe->tb_credits += n_periods * params->tb_credits_per_period; - pipe->tb_credits = rte_sched_min_val_2_u32(pipe->tb_credits, params->tb_size); + pipe->tb_credits = RTE_MIN(pipe->tb_credits, params->tb_size); pipe->tb_time += n_periods * params->tb_period; /* Subport TCs */ @@ -2127,11 +2107,11 @@ grinder_credits_check(struct rte_sched_port *port, struct rte_sched_pipe *pipe = grinder->pipe; struct rte_mbuf *pkt = grinder->pkt; uint32_t tc_index = grinder->tc_index; - uint32_t pkt_len = pkt->pkt_len + port->frame_overhead; - uint32_t subport_tb_credits = subport->tb_credits; - uint32_t subport_tc_credits = subport->tc_credits[tc_index]; - uint32_t pipe_tb_credits = pipe->tb_credits; - uint32_t pipe_tc_credits = pipe->tc_credits[tc_index]; + uint64_t pkt_len = pkt->pkt_len + port->frame_overhead; + uint64_t subport_tb_credits = subport->tb_credits; + uint64_t subport_tc_credits = subport->tc_credits[tc_index]; + uint64_t pipe_tb_credits = pipe->tb_credits; + uint64_t pipe_tc_credits = pipe->tc_credits[tc_index]; int enough_credits; /* Check queue credits */ @@ -2162,21 +2142,22 @@ grinder_credits_check(struct rte_sched_port *port, struct rte_sched_pipe *pipe = grinder->pipe; struct rte_mbuf *pkt = grinder->pkt; uint32_t tc_index = grinder->tc_index; - uint32_t pkt_len = pkt->pkt_len + port->frame_overhead; - uint32_t subport_tb_credits = subport->tb_credits; - uint32_t subport_tc_credits = subport->tc_credits[tc_index]; - uint32_t pipe_tb_credits = pipe->tb_credits; - uint32_t pipe_tc_credits = pipe->tc_credits[tc_index]; - uint32_t pipe_tc_ov_mask1[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; - uint32_t pipe_tc_ov_mask2[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE] = {0}; - uint32_t pipe_tc_ov_credits, i; + uint64_t pkt_len = pkt->pkt_len + port->frame_overhead; + uint64_t subport_tb_credits = subport->tb_credits; + uint64_t subport_tc_credits = subport->tc_credits[tc_index]; + uint64_t pipe_tb_credits = pipe->tb_credits; + uint64_t pipe_tc_credits = pipe->tc_credits[tc_index]; + uint64_t pipe_tc_ov_mask1[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE]; + uint64_t pipe_tc_ov_mask2[RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE] = {0}; + uint64_t pipe_tc_ov_credits; + uint32_t i; int enough_credits; for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++) - pipe_tc_ov_mask1[i] = UINT32_MAX; + pipe_tc_ov_mask1[i] = ~0LLU; pipe_tc_ov_mask1[RTE_SCHED_TRAFFIC_CLASS_BE] = pipe->tc_ov_credits; - pipe_tc_ov_mask2[RTE_SCHED_TRAFFIC_CLASS_BE] = UINT32_MAX; + pipe_tc_ov_mask2[RTE_SCHED_TRAFFIC_CLASS_BE] = ~0LLU; pipe_tc_ov_credits = pipe_tc_ov_mask1[tc_index]; /* Check pipe and subport credits */ @@ -2593,9 +2574,9 @@ grinder_prefetch_mbuf(struct rte_sched_subport *subport, uint32_t pos) } static inline uint32_t -grinder_handle(struct rte_sched_port *port, uint32_t pos) +grinder_handle(struct rte_sched_port *port, + struct rte_sched_subport *subport, uint32_t pos) { - struct rte_sched_subport *subport = port->subport; struct rte_sched_grinder *grinder = subport->grinder + pos; switch (grinder->state) { @@ -2694,6 +2675,7 @@ rte_sched_port_time_resync(struct rte_sched_port *port) uint64_t cycles = rte_get_tsc_cycles(); uint64_t cycles_diff = cycles - port->time_cpu_cycles; uint64_t bytes_diff; + uint32_t i; /* Compute elapsed time in bytes */ bytes_diff = rte_reciprocal_divide(cycles_diff << RTE_SCHED_TIME_SHIFT, @@ -2706,20 +2688,21 @@ rte_sched_port_time_resync(struct rte_sched_port *port) port->time = port->time_cpu_bytes; /* Reset pipe loop detection */ - port->pipe_loop = RTE_SCHED_PIPE_INVALID; + for (i = 0; i < port->n_subports_per_port; i++) + port->subports[i]->pipe_loop = RTE_SCHED_PIPE_INVALID; } static inline int -rte_sched_port_exceptions(struct rte_sched_port *port, int second_pass) +rte_sched_port_exceptions(struct rte_sched_subport *subport, int second_pass) { int exceptions; /* Check if any exception flag is set */ - exceptions = (second_pass && port->busy_grinders == 0) || - (port->pipe_exhaustion == 1); + exceptions = (second_pass && subport->busy_grinders == 0) || + (subport->pipe_exhaustion == 1); /* Clear exception flags */ - port->pipe_exhaustion = 0; + subport->pipe_exhaustion = 0; return exceptions; } @@ -2727,7 +2710,9 @@ rte_sched_port_exceptions(struct rte_sched_port *port, int second_pass) int rte_sched_port_dequeue(struct rte_sched_port *port, struct rte_mbuf **pkts, uint32_t n_pkts) { - uint32_t i, count; + struct rte_sched_subport *subport; + uint32_t subport_id = port->subport_id; + uint32_t i, n_subports = 0, count; port->pkts_out = pkts; port->n_pkts_out = 0; @@ -2736,9 +2721,32 @@ rte_sched_port_dequeue(struct rte_sched_port *port, struct rte_mbuf **pkts, uint /* Take each queue in the grinder one step further */ for (i = 0, count = 0; ; i++) { - count += grinder_handle(port, i & (RTE_SCHED_PORT_N_GRINDERS - 1)); - if ((count == n_pkts) || - rte_sched_port_exceptions(port, i >= RTE_SCHED_PORT_N_GRINDERS)) { + subport = port->subports[subport_id]; + + count += grinder_handle(port, subport, + i & (RTE_SCHED_PORT_N_GRINDERS - 1)); + + if (count == n_pkts) { + subport_id++; + + if (subport_id == port->n_subports_per_port) + subport_id = 0; + + port->subport_id = subport_id; + break; + } + + if (rte_sched_port_exceptions(subport, i >= RTE_SCHED_PORT_N_GRINDERS)) { + i = 0; + subport_id++; + n_subports++; + } + + if (subport_id == port->n_subports_per_port) + subport_id = 0; + + if (n_subports == port->n_subports_per_port) { + port->subport_id = subport_id; break; } }