-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation
*/
#include <stdio.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>
#include <rte_mbuf.h>
+#include <rte_bitmap.h>
+#include <rte_reciprocal.h>
#include "rte_sched.h"
-#include "rte_bitmap.h"
#include "rte_sched_common.h"
#include "rte_approx.h"
#endif
#ifdef RTE_SCHED_VECTOR
-#include <immintrin.h>
+#include <rte_vect.h>
+
+#ifdef RTE_ARCH_X86
+#define SCHED_VECTOR_SSE4
+#elif defined(RTE_MACHINE_CPUFLAG_NEON)
+#define SCHED_VECTOR_NEON
+#endif
+
#endif
#define RTE_SCHED_TB_RATE_CONFIG_ERR (1e-7)
#define RTE_SCHED_PIPE_INVALID UINT32_MAX
#define RTE_SCHED_BMP_POS_INVALID UINT32_MAX
+/* Scaling for cycles_per_byte calculation
+ * Chosen so that minimum rate is 480 bit/sec
+ */
+#define RTE_SCHED_TIME_SHIFT 8
+
struct rte_sched_subport {
/* Token bucket (TB) */
uint64_t tb_time; /* time of last update */
* by scheduler enqueue.
*/
struct rte_sched_port_hierarchy {
- uint32_t queue:2; /**< Queue ID (0 .. 3) */
- uint32_t traffic_class:2; /**< Traffic class ID (0 .. 3)*/
- uint32_t pipe:20; /**< Pipe ID */
- uint32_t subport:6; /**< Subport ID */
+ uint16_t queue:2; /**< Queue ID (0 .. 3) */
+ uint16_t traffic_class:2; /**< Traffic class ID (0 .. 3)*/
uint32_t color:2; /**< Color */
+ uint16_t unused:10;
+ uint16_t subport; /**< Subport ID */
+ uint32_t pipe; /**< Pipe ID */
};
struct rte_sched_grinder {
uint64_t time_cpu_cycles; /* Current CPU time measured in CPU cyles */
uint64_t time_cpu_bytes; /* Current CPU time measured in bytes */
uint64_t time; /* Current NIC TX time measured in bytes */
- double cycles_per_byte; /* CPU cycles per byte */
+ struct rte_reciprocal inv_cycles_per_byte; /* CPU cycles per byte */
/* Scheduling loop detection */
uint32_t pipe_loop;
return RTE_SCHED_QUEUES_PER_PIPE * port->n_pipes_per_subport * port->n_subports_per_port;
}
+static inline struct rte_mbuf **
+rte_sched_port_qbase(struct rte_sched_port *port, uint32_t qindex)
+{
+ uint32_t pindex = qindex >> 4;
+ uint32_t qpos = qindex & 0xF;
+
+ return (port->queue_array + pindex *
+ port->qsize_sum + port->qsize_add[qpos]);
+}
+
+static inline uint16_t
+rte_sched_port_qsize(struct rte_sched_port *port, uint32_t qindex)
+{
+ uint32_t tc = (qindex >> 2) & 0x3;
+
+ return port->qsize[tc];
+}
+
static int
rte_sched_port_check_params(struct rte_sched_port_params *params)
{
if (params->mtu == 0)
return -5;
- /* n_subports_per_port: non-zero, power of 2 */
+ /* n_subports_per_port: non-zero, limited to 16 bits, power of 2 */
if (params->n_subports_per_port == 0 ||
+ params->n_subports_per_port > 1u << 16 ||
!rte_is_power_of_2(params->n_subports_per_port))
return -6;
size0 = sizeof(struct rte_sched_port);
size1 = rte_sched_port_get_array_base(params, e_RTE_SCHED_PORT_ARRAY_TOTAL);
- return (size0 + size1);
+ return size0 + size1;
}
static void
rte_sched_port_config(struct rte_sched_port_params *params)
{
struct rte_sched_port *port = NULL;
- uint32_t mem_size, bmp_mem_size, n_queues_per_port, i;
+ uint32_t mem_size, bmp_mem_size, n_queues_per_port, i, cycles_per_byte;
/* Check user parameters. Determine the amount of memory to allocate */
mem_size = rte_sched_port_get_memory_footprint(params);
port->time_cpu_cycles = rte_get_tsc_cycles();
port->time_cpu_bytes = 0;
port->time = 0;
- port->cycles_per_byte = ((double) rte_get_tsc_hz()) / ((double) params->rate);
+
+ cycles_per_byte = (rte_get_tsc_hz() << RTE_SCHED_TIME_SHIFT)
+ / params->rate;
+ port->inv_cycles_per_byte = rte_reciprocal_value(cycles_per_byte);
/* Scheduling loop detection */
port->pipe_loop = RTE_SCHED_PIPE_INVALID;
void
rte_sched_port_free(struct rte_sched_port *port)
{
+ uint32_t qindex;
+ uint32_t n_queues_per_port;
+
/* Check user parameters */
if (port == NULL)
return;
+ n_queues_per_port = rte_sched_port_queues_per_port(port);
+
+ /* Free enqueued mbufs */
+ for (qindex = 0; qindex < n_queues_per_port; qindex++) {
+ struct rte_mbuf **mbufs = rte_sched_port_qbase(port, qindex);
+ uint16_t qsize = rte_sched_port_qsize(port, qindex);
+ struct rte_sched_queue *queue = port->queue + qindex;
+ uint16_t qr = queue->qr & (qsize - 1);
+ uint16_t qw = queue->qw & (qsize - 1);
+
+ for (; qr != qw; qr = (qr + 1) & (qsize - 1))
+ rte_pktmbuf_free(mbufs[qr]);
+ }
+
rte_bitmap_free(port->bmp);
rte_free(port);
}
struct rte_sched_port_hierarchy *sched
= (struct rte_sched_port_hierarchy *) &pkt->hash.sched;
+ RTE_BUILD_BUG_ON(sizeof(*sched) > sizeof(pkt->hash.sched));
+
sched->color = (uint32_t) color;
sched->subport = subport;
sched->pipe = pipe;
memcpy(stats, &s->stats, sizeof(struct rte_sched_subport_stats));
memset(&s->stats, 0, sizeof(struct rte_sched_subport_stats));
- /* Subport TC ovesubscription status */
+ /* Subport TC oversubscription status */
*tc_ov = s->tc_ov;
return 0;
return result;
}
-static inline struct rte_mbuf **
-rte_sched_port_qbase(struct rte_sched_port *port, uint32_t qindex)
-{
- uint32_t pindex = qindex >> 4;
- uint32_t qpos = qindex & 0xF;
-
- return (port->queue_array + pindex * port->qsize_sum + port->qsize_add[qpos]);
-}
-
-static inline uint16_t
-rte_sched_port_qsize(struct rte_sched_port *port, uint32_t qindex)
-{
- uint32_t tc = (qindex >> 2) & 0x3;
-
- return port->qsize[tc];
-}
-
#ifdef RTE_SCHED_DEBUG
static inline int
{
struct rte_sched_queue *queue = port->queue + qindex;
- return (queue->qr == queue->qw);
-}
-
-static inline int
-rte_sched_port_queue_is_full(struct rte_sched_port *port, uint32_t qindex)
-{
- struct rte_sched_queue *queue = port->queue + qindex;
- uint16_t qsize = rte_sched_port_qsize(port, qindex);
- uint16_t qlen = queue->qw - queue->qr;
-
- return (qlen >= qsize);
+ return queue->qr == queue->qw;
}
#endif /* RTE_SCHED_DEBUG */
s->stats.n_bytes_tc[tc_index] += pkt_len;
}
+#ifdef RTE_SCHED_RED
static inline void
-rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
+rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port,
+ uint32_t qindex,
+ struct rte_mbuf *pkt, uint32_t red)
+#else
+static inline void
+rte_sched_port_update_subport_stats_on_drop(struct rte_sched_port *port,
+ uint32_t qindex,
+ struct rte_mbuf *pkt, __rte_unused uint32_t red)
+#endif
{
struct rte_sched_subport *s = port->subport + (qindex / rte_sched_port_queues_per_subport(port));
uint32_t tc_index = (qindex >> 2) & 0x3;
s->stats.n_pkts_tc_dropped[tc_index] += 1;
s->stats.n_bytes_tc_dropped[tc_index] += pkt_len;
+#ifdef RTE_SCHED_RED
+ s->stats.n_pkts_red_dropped[tc_index] += red;
+#endif
}
static inline void
qe->stats.n_bytes += pkt_len;
}
+#ifdef RTE_SCHED_RED
static inline void
-rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port, uint32_t qindex, struct rte_mbuf *pkt)
+rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port,
+ uint32_t qindex,
+ struct rte_mbuf *pkt, uint32_t red)
+#else
+static inline void
+rte_sched_port_update_queue_stats_on_drop(struct rte_sched_port *port,
+ uint32_t qindex,
+ struct rte_mbuf *pkt, __rte_unused uint32_t red)
+#endif
{
struct rte_sched_queue_extra *qe = port->queue_extra + qindex;
uint32_t pkt_len = pkt->pkt_len;
qe->stats.n_pkts_dropped += 1;
qe->stats.n_bytes_dropped += pkt_len;
+#ifdef RTE_SCHED_RED
+ qe->stats.n_pkts_red_dropped += red;
+#endif
}
#endif /* RTE_SCHED_COLLECT_STATS */
#ifdef RTE_SCHED_DEBUG
-static inline int
-debug_pipe_is_empty(struct rte_sched_port *port, uint32_t pindex)
-{
- uint32_t qindex, i;
-
- qindex = pindex << 4;
-
- for (i = 0; i < 16; i++) {
- uint32_t queue_empty = rte_sched_port_queue_is_empty(port, qindex + i);
- uint32_t bmp_bit_clear = (rte_bitmap_get(port->bmp, qindex + i) == 0);
-
- if (queue_empty != bmp_bit_clear)
- rte_panic("Queue status mismatch for queue %u of pipe %u\n", i, pindex);
-
- if (!queue_empty)
- return 0;
- }
-
- return 1;
-}
-
static inline void
debug_check_queue_slab(struct rte_sched_port *port, uint32_t bmp_pos,
uint64_t bmp_slab)
(qlen >= qsize))) {
rte_pktmbuf_free(pkt);
#ifdef RTE_SCHED_COLLECT_STATS
- rte_sched_port_update_subport_stats_on_drop(port, qindex, pkt);
- rte_sched_port_update_queue_stats_on_drop(port, qindex, pkt);
+ rte_sched_port_update_subport_stats_on_drop(port, qindex, pkt,
+ qlen < qsize);
+ rte_sched_port_update_queue_stats_on_drop(port, qindex, pkt,
+ qlen < qsize);
#endif
return 0;
}
return 1;
}
-#ifdef RTE_SCHED_VECTOR
+#ifdef SCHED_VECTOR_SSE4
static inline int
grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe)
return 1;
}
+#elif defined(SCHED_VECTOR_NEON)
+
+static inline int
+grinder_pipe_exists(struct rte_sched_port *port, uint32_t base_pipe)
+{
+ uint32x4_t index, pipes;
+ uint32_t *pos = (uint32_t *)port->grinder_base_bmp_pos;
+
+ index = vmovq_n_u32(base_pipe);
+ pipes = vld1q_u32(pos);
+ if (!vminvq_u32(veorq_u32(pipes, index)))
+ return 1;
+
+ pipes = vld1q_u32(pos + 4);
+ if (!vminvq_u32(veorq_u32(pipes, index)))
+ return 1;
+
+ return 0;
+}
+
#else
static inline int
{
uint64_t cycles = rte_get_tsc_cycles();
uint64_t cycles_diff = cycles - port->time_cpu_cycles;
- double bytes_diff = ((double) cycles_diff) / port->cycles_per_byte;
+ uint64_t bytes_diff;
+
+ /* Compute elapsed time in bytes */
+ bytes_diff = rte_reciprocal_divide(cycles_diff << RTE_SCHED_TIME_SHIFT,
+ port->inv_cycles_per_byte);
/* Advance port time */
port->time_cpu_cycles = cycles;
- port->time_cpu_bytes += (uint64_t) bytes_diff;
+ port->time_cpu_bytes += bytes_diff;
if (port->time < port->time_cpu_bytes)
port->time = port->time_cpu_bytes;
git.droids-corp.org / dpdk.git / blobdiff