#include <rte_common.h>
#include <rte_log.h>
-#include <rte_memory.h>
#include <rte_malloc.h>
#include <rte_cycles.h>
#include <rte_prefetch.h>
#pragma warning(disable:2259) /* conversion may lose significant bits */
#endif
-#ifdef RTE_SCHED_VECTOR
-#include <rte_vect.h>
-
-#ifdef RTE_ARCH_X86
-#define SCHED_VECTOR_SSE4
-#elif defined(__ARM_NEON)
-#define SCHED_VECTOR_NEON
-#endif
-
+#ifndef RTE_SCHED_PORT_N_GRINDERS
+#define RTE_SCHED_PORT_N_GRINDERS 8
#endif
#define RTE_SCHED_TB_RATE_CONFIG_ERR (1e-7)
int socket;
/* Timing */
- uint64_t time_cpu_cycles; /* Current CPU time measured in CPU cyles */
+ uint64_t time_cpu_cycles; /* Current CPU time measured in CPU cycles */
uint64_t time_cpu_bytes; /* Current CPU time measured in bytes */
uint64_t time; /* Current NIC TX time measured in bytes */
struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */
subport->qsize_add[0] = 0;
- /* Strict prority traffic class */
+ /* Strict priority traffic class */
for (i = 1; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
subport->qsize_add[i] = subport->qsize_add[i-1] + subport->qsize[i-1];
for (i = 0; i < RTE_SCHED_PORT_N_GRINDERS; i++)
s->grinder_base_bmp_pos[i] = RTE_SCHED_PIPE_INVALID;
-#ifdef RTE_SCHED_SUBPORT_TC_OV
/* TC oversubscription */
s->tc_ov_wm_min = port->mtu;
s->tc_ov_period_id = 0;
s->tc_ov = 0;
s->tc_ov_n = 0;
s->tc_ov_rate = 0;
-#endif
}
{
else
profile->tc_credits_per_period[i] = 0;
-#ifdef RTE_SCHED_SUBPORT_TC_OV
s->tc_ov_wm_max = rte_sched_time_ms_to_bytes(profile->tc_period,
s->pipe_tc_be_rate_max);
s->tc_ov_wm = s->tc_ov_wm_max;
-#endif
s->profile = subport_profile_id;
}
#endif /* RTE_SCHED_DEBUG */
-#ifdef RTE_SCHED_COLLECT_STATS
-
static inline void
rte_sched_port_update_subport_stats(struct rte_sched_port *port,
struct rte_sched_subport *subport,
#endif
}
-#endif /* RTE_SCHED_COLLECT_STATS */
-
#ifdef RTE_SCHED_CMAN
static inline int
struct rte_mbuf *pkt, uint32_t subport_qmask)
{
struct rte_sched_queue *q;
-#ifdef RTE_SCHED_COLLECT_STATS
struct rte_sched_queue_extra *qe;
-#endif
uint32_t qindex = rte_mbuf_sched_queue_get(pkt);
uint32_t subport_queue_id = subport_qmask & qindex;
q = subport->queue + subport_queue_id;
rte_prefetch0(q);
-#ifdef RTE_SCHED_COLLECT_STATS
qe = subport->queue_extra + subport_queue_id;
rte_prefetch0(qe);
-#endif
return subport_queue_id;
}
if (unlikely(rte_sched_port_cman_drop(port, subport, pkt, qindex, qlen) ||
(qlen >= qsize))) {
rte_pktmbuf_free(pkt);
-#ifdef RTE_SCHED_COLLECT_STATS
rte_sched_port_update_subport_stats_on_drop(port, subport,
qindex, pkt, qlen < qsize);
rte_sched_port_update_queue_stats_on_drop(subport, qindex, pkt,
qlen < qsize);
-#endif
return 0;
}
rte_bitmap_set(subport->bmp, qindex);
/* Statistics */
-#ifdef RTE_SCHED_COLLECT_STATS
rte_sched_port_update_subport_stats(port, subport, qindex, pkt);
rte_sched_port_update_queue_stats(subport, qindex, pkt);
-#endif
return 1;
}
return result;
}
-#ifndef RTE_SCHED_SUBPORT_TC_OV
-
-static inline void
-grinder_credits_update(struct rte_sched_port *port,
- struct rte_sched_subport *subport, uint32_t pos)
-{
- struct rte_sched_grinder *grinder = subport->grinder + pos;
- struct rte_sched_pipe *pipe = grinder->pipe;
- struct rte_sched_pipe_profile *params = grinder->pipe_params;
- struct rte_sched_subport_profile *sp = grinder->subport_params;
- uint64_t n_periods;
- uint32_t i;
-
- /* Subport TB */
- n_periods = (port->time - subport->tb_time) / sp->tb_period;
- subport->tb_credits += n_periods * sp->tb_credits_per_period;
- subport->tb_credits = RTE_MIN(subport->tb_credits, sp->tb_size);
- subport->tb_time += n_periods * sp->tb_period;
-
- /* Pipe TB */
- n_periods = (port->time - pipe->tb_time) / params->tb_period;
- pipe->tb_credits += n_periods * params->tb_credits_per_period;
- pipe->tb_credits = RTE_MIN(pipe->tb_credits, params->tb_size);
- pipe->tb_time += n_periods * params->tb_period;
-
- /* Subport TCs */
- if (unlikely(port->time >= subport->tc_time)) {
- for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
- subport->tc_credits[i] = sp->tc_credits_per_period[i];
-
- subport->tc_time = port->time + sp->tc_period;
- }
-
- /* Pipe TCs */
- if (unlikely(port->time >= pipe->tc_time)) {
- for (i = 0; i < RTE_SCHED_TRAFFIC_CLASSES_PER_PIPE; i++)
- pipe->tc_credits[i] = params->tc_credits_per_period[i];
-
- pipe->tc_time = port->time + params->tc_period;
- }
-}
-
-#else
-
static inline uint64_t
grinder_tc_ov_credits_update(struct rte_sched_port *port,
struct rte_sched_subport *subport, uint32_t pos)
}
}
-#endif /* RTE_SCHED_TS_CREDITS_UPDATE, RTE_SCHED_SUBPORT_TC_OV */
-
-
-#ifndef RTE_SCHED_SUBPORT_TC_OV
-
-static inline int
-grinder_credits_check(struct rte_sched_port *port,
- struct rte_sched_subport *subport, uint32_t pos)
-{
- struct rte_sched_grinder *grinder = subport->grinder + pos;
- struct rte_sched_pipe *pipe = grinder->pipe;
- struct rte_mbuf *pkt = grinder->pkt;
- uint32_t tc_index = grinder->tc_index;
- uint64_t pkt_len = pkt->pkt_len + port->frame_overhead;
- uint64_t subport_tb_credits = subport->tb_credits;
- uint64_t subport_tc_credits = subport->tc_credits[tc_index];
- uint64_t pipe_tb_credits = pipe->tb_credits;
- uint64_t pipe_tc_credits = pipe->tc_credits[tc_index];
- int enough_credits;
-
- /* Check queue credits */
- enough_credits = (pkt_len <= subport_tb_credits) &&
- (pkt_len <= subport_tc_credits) &&
- (pkt_len <= pipe_tb_credits) &&
- (pkt_len <= pipe_tc_credits);
-
- if (!enough_credits)
- return 0;
-
- /* Update port credits */
- subport->tb_credits -= pkt_len;
- subport->tc_credits[tc_index] -= pkt_len;
- pipe->tb_credits -= pkt_len;
- pipe->tc_credits[tc_index] -= pkt_len;
-
- return 1;
-}
-
-#else
-
static inline int
grinder_credits_check(struct rte_sched_port *port,
struct rte_sched_subport *subport, uint32_t pos)
return 1;
}
-#endif /* RTE_SCHED_SUBPORT_TC_OV */
-
static inline int
grinder_schedule(struct rte_sched_port *port,
return 1;
}
-#ifdef SCHED_VECTOR_SSE4
-
-static inline int
-grinder_pipe_exists(struct rte_sched_subport *subport, uint32_t base_pipe)
-{
- __m128i index = _mm_set1_epi32(base_pipe);
- __m128i pipes = _mm_load_si128((__m128i *)subport->grinder_base_bmp_pos);
- __m128i res = _mm_cmpeq_epi32(pipes, index);
-
- pipes = _mm_load_si128((__m128i *)(subport->grinder_base_bmp_pos + 4));
- pipes = _mm_cmpeq_epi32(pipes, index);
- res = _mm_or_si128(res, pipes);
-
- if (_mm_testz_si128(res, res))
- return 0;
-
- return 1;
-}
-
-#elif defined(SCHED_VECTOR_NEON)
-
-static inline int
-grinder_pipe_exists(struct rte_sched_subport *subport, uint32_t base_pipe)
-{
- uint32x4_t index, pipes;
- uint32_t *pos = (uint32_t *)subport->grinder_base_bmp_pos;
-
- index = vmovq_n_u32(base_pipe);
- pipes = vld1q_u32(pos);
- if (!vminvq_u32(veorq_u32(pipes, index)))
- return 1;
-
- pipes = vld1q_u32(pos + 4);
- if (!vminvq_u32(veorq_u32(pipes, index)))
- return 1;
-
- return 0;
-}
-
-#else
-
static inline int
grinder_pipe_exists(struct rte_sched_subport *subport, uint32_t base_pipe)
{
return 0;
}
-#endif /* RTE_SCHED_OPTIMIZATIONS */
-
static inline void
grinder_pcache_populate(struct rte_sched_subport *subport,
uint32_t pos, uint32_t bmp_pos, uint64_t bmp_slab)
git.droids-corp.org / dpdk.git / blobdiff