#include <rte_memcpy.h>
#include <rte_memzone.h>
#include <rte_eal.h>
-#include <rte_per_lcore.h>
#include <rte_launch.h>
#include <rte_atomic.h>
#include <rte_cycles.h>
#include <rte_tcp.h>
#include <rte_udp.h>
#include <rte_string_fns.h>
+#include <rte_pause.h>
#include <cmdline_parse.h>
#include <cmdline_parse_etheraddr.h>
#define APP_LOOKUP_METHOD APP_LOOKUP_LPM
#endif
+#ifndef __GLIBC__ /* sched_getcpu() is glibc specific */
+#define sched_getcpu() rte_lcore_id()
+#endif
+
static int
check_ptype(int portid)
{
* When set to one, optimized forwarding path is enabled.
* Note that LPM optimisation path uses SSE4.1 instructions.
*/
-#if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && !defined(__SSE4_1__))
-#define ENABLE_MULTI_BUFFER_OPTIMIZE 0
-#else
#define ENABLE_MULTI_BUFFER_OPTIMIZE 1
-#endif
#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
#include <rte_hash.h>
*/
#define NB_MBUF RTE_MAX(\
- (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \
- nb_ports*nb_lcores*MAX_PKT_BURST + \
- nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \
- nb_lcores*MEMPOOL_CACHE_SIZE), \
+ (nb_ports*nb_rx_queue*nb_rxd + \
+ nb_ports*nb_lcores*MAX_PKT_BURST + \
+ nb_ports*n_tx_queue*nb_txd + \
+ nb_lcores*MEMPOOL_CACHE_SIZE), \
(unsigned)8192)
#define MAX_PKT_BURST 32
static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS];
static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
-static __m128i val_eth[RTE_MAX_ETHPORTS];
+static xmm_t val_eth[RTE_MAX_ETHPORTS];
/* replace first 12B of the ethernet header. */
#define MASK_ETH 0x3f
.hw_ip_checksum = 1, /**< IP checksum offload enabled */
.hw_vlan_filter = 0, /**< VLAN filtering disabled */
.jumbo_frame = 0, /**< Jumbo Frame Support disabled */
- .hw_strip_crc = 0, /**< CRC stripped by hardware */
+ .hw_strip_crc = 1, /**< CRC stripped by hardware */
},
.rx_adv_conf = {
.rss_conf = {
#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
#include <rte_hash_crc.h>
#define DEFAULT_HASH_FUNC rte_hash_crc
-#else
-#include <rte_jhash.h>
-#define DEFAULT_HASH_FUNC rte_jhash
-#endif
struct ipv4_5tuple {
uint32_t ip_dst;
t = k->proto;
p = (const uint32_t *)&k->port_src;
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
init_val = rte_hash_crc_4byte(t, init_val);
init_val = rte_hash_crc_4byte(k->ip_src, init_val);
init_val = rte_hash_crc_4byte(k->ip_dst, init_val);
init_val = rte_hash_crc_4byte(*p, init_val);
-#else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
- init_val = rte_jhash_1word(t, init_val);
- init_val = rte_jhash_1word(k->ip_src, init_val);
- init_val = rte_jhash_1word(k->ip_dst, init_val);
- init_val = rte_jhash_1word(*p, init_val);
-#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
return init_val;
}
const union ipv6_5tuple_host *k;
uint32_t t;
const uint32_t *p;
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
const uint32_t *ip_src0, *ip_src1, *ip_src2, *ip_src3;
const uint32_t *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3;
-#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
k = data;
t = k->proto;
p = (const uint32_t *)&k->port_src;
-#ifdef RTE_MACHINE_CPUFLAG_SSE4_2
ip_src0 = (const uint32_t *) k->ip_src;
ip_src1 = (const uint32_t *)(k->ip_src + 4);
ip_src2 = (const uint32_t *)(k->ip_src + 8);
init_val = rte_hash_crc_4byte(*ip_dst2, init_val);
init_val = rte_hash_crc_4byte(*ip_dst3, init_val);
init_val = rte_hash_crc_4byte(*p, init_val);
-#else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
- init_val = rte_jhash_1word(t, init_val);
- init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
- init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
- init_val = rte_jhash_1word(*p, init_val);
-#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
return init_val;
}
#if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
(ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
-static inline __attribute__((always_inline)) void
+static __rte_always_inline void
send_packetsx4(uint8_t port,
struct rte_mbuf *m[], uint32_t num)
{
case 0:
qconf->tx_mbufs[port].m_table[len + j] = m[j];
j++;
+ /* fall-through */
case 3:
qconf->tx_mbufs[port].m_table[len + j] = m[j];
j++;
+ /* fall-through */
case 2:
qconf->tx_mbufs[port].m_table[len + j] = m[j];
j++;
+ /* fall-through */
case 1:
qconf->tx_mbufs[port].m_table[len + j] = m[j];
j++;
case 0:
qconf->tx_mbufs[port].m_table[j] = m[n + j];
j++;
+ /* fall-through */
case 3:
qconf->tx_mbufs[port].m_table[j] = m[n + j];
j++;
+ /* fall-through */
case 2:
qconf->tx_mbufs[port].m_table[j] = m[n + j];
j++;
+ /* fall-through */
case 1:
qconf->tx_mbufs[port].m_table[j] = m[n + j];
j++;
get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid,
lookup6_struct_t *ipv6_l3fwd_lookup_struct)
{
- uint8_t next_hop;
+ uint32_t next_hop;
return (uint8_t) ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct,
((struct ipv6_hdr *)ipv6_hdr)->dst_addr, &next_hop) == 0) ?
}
#endif /* APP_LOOKUP_METHOD */
-static inline __attribute__((always_inline)) void
+static __rte_always_inline void
l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid)
{
struct ether_hdr *eth_hdr;
* If we encounter invalid IPV4 packet, then set destination port for it
* to BAD_PORT value.
*/
-static inline __attribute__((always_inline)) void
+static __rte_always_inline void
rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype)
{
uint8_t ihl;
#if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
(ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
-static inline __attribute__((always_inline)) uint16_t
+static __rte_always_inline uint16_t
get_dst_port(struct rte_mbuf *pkt, uint32_t dst_ipv4, uint8_t portid)
{
- uint32_t next_hop_ipv4;
- uint8_t next_hop_ipv6;
+ uint32_t next_hop;
struct ipv6_hdr *ipv6_hdr;
struct ether_hdr *eth_hdr;
if (RTE_ETH_IS_IPV4_HDR(pkt->packet_type)) {
return (uint16_t) ((rte_lpm_lookup(
RTE_PER_LCORE(lcore_conf)->ipv4_lookup_struct, dst_ipv4,
- &next_hop_ipv4) == 0) ? next_hop_ipv4 : portid);
+ &next_hop) == 0) ? next_hop : portid);
} else if (RTE_ETH_IS_IPV6_HDR(pkt->packet_type)) {
return (uint16_t) ((rte_lpm6_lookup(
RTE_PER_LCORE(lcore_conf)->ipv6_lookup_struct,
- ipv6_hdr->dst_addr, &next_hop_ipv6) == 0) ? next_hop_ipv6 :
- portid);
+ ipv6_hdr->dst_addr, &next_hop) == 0) ?
+ next_hop : portid);
}
* Suppose we have array of destionation ports:
* dst_port[] = {a, b, c, d,, e, ... }
* dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>.
- * We doing 4 comparisions at once and the result is 4 bit mask.
+ * We doing 4 comparisons at once and the result is 4 bit mask.
* This mask is used as an index into prebuild array of pnum values.
*/
static inline uint16_t *
process_packet(pkts_burst[j], dst_port + j, portid);
GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
j++;
+ /* fall-through */
case 2:
process_packet(pkts_burst[j], dst_port + j, portid);
GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
j++;
+ /* fall-through */
case 1:
process_packet(pkts_burst[j], dst_port + j, portid);
GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
*/
SET_CPU_BUSY(tx_conf, CPU_POLL);
nb_rx = rte_ring_sc_dequeue_burst(ring, (void **)pkts_burst,
- MAX_PKT_BURST);
+ MAX_PKT_BURST, NULL);
SET_CPU_IDLE(tx_conf, CPU_POLL);
if (nb_rx > 0) {
ret = rte_ring_sp_enqueue_burst(
rx_conf->ring[worker_id],
(void **) pkts_burst,
- nb_rx);
+ nb_rx, NULL);
new_len = old_len + ret;
*/
SET_CPU_BUSY(tx_conf, CPU_POLL);
nb_rx = rte_ring_sc_dequeue_burst(tx_conf->ring,
- (void **)pkts_burst, MAX_PKT_BURST);
+ (void **)pkts_burst, MAX_PKT_BURST, NULL);
SET_CPU_IDLE(tx_conf, CPU_POLL);
if (unlikely(nb_rx == 0)) {
SET_CPU_BUSY(rx_conf, CPU_PROCESS);
worker_id = (worker_id + 1) % rx_conf->n_ring;
n = rte_ring_sp_enqueue_burst(rx_conf->ring[worker_id],
- (void **)pkts_burst, nb_rx);
+ (void **)pkts_burst, nb_rx, NULL);
if (unlikely(n != nb_rx)) {
uint32_t k;
rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%d\n",
ret, portid);
+ ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
+ &nb_txd);
+ if (ret < 0)
+ rte_exit(EXIT_FAILURE,
+ "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d, port=%d\n",
+ ret, portid);
+
rte_eth_macaddr_get(portid, &ports_eth_addr[portid]);
print_ethaddr(" Address:", &ports_eth_addr[portid]);
printf(", ");