#include <getopt.h>
#include <rte_common.h>
-#include <rte_common_vect.h>
+#include <rte_vect.h>
#include <rte_byteorder.h>
#include <rte_log.h>
#include <rte_memory.h>
#include <rte_memcpy.h>
#include <rte_memzone.h>
-#include <rte_tailq.h>
#include <rte_eal.h>
#include <rte_per_lcore.h>
#include <rte_launch.h>
#include <rte_udp.h>
#include <rte_string_fns.h>
-#include "main.h"
+#include <cmdline_parse.h>
+#include <cmdline_parse_etheraddr.h>
#define APP_LOOKUP_EXACT_MATCH 0
#define APP_LOOKUP_LPM 1
#define MEMPOOL_CACHE_SIZE 256
-#define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
-
/*
* This expression is used to calculate the number of mbufs needed depending on user input, taking
* into account memory for rx and tx hardware rings, cache per lcore and mtable per port per lcore.
nb_lcores*MEMPOOL_CACHE_SIZE), \
(unsigned)8192)
-/*
- * RX and TX Prefetch, Host, and Write-back threshold values should be
- * carefully set for optimal performance. Consult the network
- * controller's datasheet and supporting DPDK documentation for guidance
- * on how these parameters should be set.
- */
-#define RX_PTHRESH 8 /**< Default values of RX prefetch threshold reg. */
-#define RX_HTHRESH 8 /**< Default values of RX host threshold reg. */
-#define RX_WTHRESH 4 /**< Default values of RX write-back threshold reg. */
-
-/*
- * These default values are optimized for use with the Intel(R) 82599 10 GbE
- * Controller and the DPDK ixgbe PMD. Consider using other values for other
- * network controllers and/or network drivers.
- */
-#define TX_PTHRESH 36 /**< Default values of TX prefetch threshold reg. */
-#define TX_HTHRESH 0 /**< Default values of TX host threshold reg. */
-#define TX_WTHRESH 0 /**< Default values of TX write-back threshold reg. */
-
#define MAX_PKT_BURST 32
#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
/* ethernet addresses of ports */
+static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS];
static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
static __m128i val_eth[RTE_MAX_ETHPORTS];
.rx_adv_conf = {
.rss_conf = {
.rss_key = NULL,
- .rss_hf = ETH_RSS_IPV4 | ETH_RSS_IPV6,
+ .rss_hf = ETH_RSS_IP,
},
},
.txmode = {
},
};
-static const struct rte_eth_rxconf rx_conf = {
- .rx_thresh = {
- .pthresh = RX_PTHRESH,
- .hthresh = RX_HTHRESH,
- .wthresh = RX_WTHRESH,
- },
- .rx_free_thresh = 32,
-};
-
-static struct rte_eth_txconf tx_conf = {
- .tx_thresh = {
- .pthresh = TX_PTHRESH,
- .hthresh = TX_HTHRESH,
- .wthresh = TX_WTHRESH,
- },
- .tx_free_thresh = 0, /* Use PMD default values */
- .tx_rs_thresh = 0, /* Use PMD default values */
- .txq_flags = (ETH_TXQ_FLAGS_NOMULTSEGS |
- ETH_TXQ_FLAGS_NOVLANOFFL |
- ETH_TXQ_FLAGS_NOXSUMSCTP |
- ETH_TXQ_FLAGS_NOXSUMUDP |
- ETH_TXQ_FLAGS_NOXSUMTCP)
-
-};
-
static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
}
#endif
+static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid,
+ struct lcore_conf *qconf) __attribute__((unused));
+
#if ((APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) && \
(ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
-static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, struct lcore_conf *qconf);
#define MASK_ALL_PKTS 0xf
#define EXECLUDE_1ST_PKT 0xe
{
struct ether_hdr *eth_hdr[4];
struct ipv4_hdr *ipv4_hdr[4];
- void *d_addr_bytes[4];
uint8_t dst_port[4];
int32_t ret[4];
union ipv4_5tuple_host key[4];
eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
/* Handle IPv4 headers.*/
- ipv4_hdr[0] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[0], unsigned char *) +
- sizeof(struct ether_hdr));
- ipv4_hdr[1] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[1], unsigned char *) +
- sizeof(struct ether_hdr));
- ipv4_hdr[2] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[2], unsigned char *) +
- sizeof(struct ether_hdr));
- ipv4_hdr[3] = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m[3], unsigned char *) +
- sizeof(struct ether_hdr));
+ ipv4_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv4_hdr *,
+ sizeof(struct ether_hdr));
+ ipv4_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv4_hdr *,
+ sizeof(struct ether_hdr));
+ ipv4_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv4_hdr *,
+ sizeof(struct ether_hdr));
+ ipv4_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv4_hdr *,
+ sizeof(struct ether_hdr));
#ifdef DO_RFC_1812_CHECKS
/* Check to make sure the packet is valid (RFC1812) */
uint8_t valid_mask = MASK_ALL_PKTS;
- if (is_valid_ipv4_pkt(ipv4_hdr[0], m[0]->pkt.pkt_len) < 0) {
+ if (is_valid_ipv4_pkt(ipv4_hdr[0], m[0]->pkt_len) < 0) {
rte_pktmbuf_free(m[0]);
valid_mask &= EXECLUDE_1ST_PKT;
}
- if (is_valid_ipv4_pkt(ipv4_hdr[1], m[1]->pkt.pkt_len) < 0) {
+ if (is_valid_ipv4_pkt(ipv4_hdr[1], m[1]->pkt_len) < 0) {
rte_pktmbuf_free(m[1]);
valid_mask &= EXECLUDE_2ND_PKT;
}
- if (is_valid_ipv4_pkt(ipv4_hdr[2], m[2]->pkt.pkt_len) < 0) {
+ if (is_valid_ipv4_pkt(ipv4_hdr[2], m[2]->pkt_len) < 0) {
rte_pktmbuf_free(m[2]);
valid_mask &= EXECLUDE_3RD_PKT;
}
- if (is_valid_ipv4_pkt(ipv4_hdr[3], m[3]->pkt.pkt_len) < 0) {
+ if (is_valid_ipv4_pkt(ipv4_hdr[3], m[3]->pkt_len) < 0) {
rte_pktmbuf_free(m[3]);
valid_mask &= EXECLUDE_4TH_PKT;
}
}
#endif // End of #ifdef DO_RFC_1812_CHECKS
- data[0] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[0], unsigned char *) +
- sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live)));
- data[1] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[1], unsigned char *) +
- sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live)));
- data[2] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[2], unsigned char *) +
- sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live)));
- data[3] = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m[3], unsigned char *) +
- sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live)));
+ data[0] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[0], __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live)));
+ data[1] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[1], __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live)));
+ data[2] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[2], __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live)));
+ data[3] = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m[3], __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv4_hdr, time_to_live)));
key[0].xmm = _mm_and_si128(data[0], mask0);
key[1].xmm = _mm_and_si128(data[1], mask0);
if (dst_port[3] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[3]) == 0)
dst_port[3] = portid;
- /* 02:00:00:00:00:xx */
- d_addr_bytes[0] = ð_hdr[0]->d_addr.addr_bytes[0];
- d_addr_bytes[1] = ð_hdr[1]->d_addr.addr_bytes[0];
- d_addr_bytes[2] = ð_hdr[2]->d_addr.addr_bytes[0];
- d_addr_bytes[3] = ð_hdr[3]->d_addr.addr_bytes[0];
- *((uint64_t *)d_addr_bytes[0]) = 0x000000000002 + ((uint64_t)dst_port[0] << 40);
- *((uint64_t *)d_addr_bytes[1]) = 0x000000000002 + ((uint64_t)dst_port[1] << 40);
- *((uint64_t *)d_addr_bytes[2]) = 0x000000000002 + ((uint64_t)dst_port[2] << 40);
- *((uint64_t *)d_addr_bytes[3]) = 0x000000000002 + ((uint64_t)dst_port[3] << 40);
-
#ifdef DO_RFC_1812_CHECKS
/* Update time to live and header checksum */
--(ipv4_hdr[0]->time_to_live);
++(ipv4_hdr[3]->hdr_checksum);
#endif
+ /* dst addr */
+ *(uint64_t *)ð_hdr[0]->d_addr = dest_eth_addr[dst_port[0]];
+ *(uint64_t *)ð_hdr[1]->d_addr = dest_eth_addr[dst_port[1]];
+ *(uint64_t *)ð_hdr[2]->d_addr = dest_eth_addr[dst_port[2]];
+ *(uint64_t *)ð_hdr[3]->d_addr = dest_eth_addr[dst_port[3]];
+
/* src addr */
ether_addr_copy(&ports_eth_addr[dst_port[0]], ð_hdr[0]->s_addr);
ether_addr_copy(&ports_eth_addr[dst_port[1]], ð_hdr[1]->s_addr);
static inline void get_ipv6_5tuple(struct rte_mbuf* m0, __m128i mask0, __m128i mask1,
union ipv6_5tuple_host * key)
{
- __m128i tmpdata0 = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m0, unsigned char *)
- + sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len)));
- __m128i tmpdata1 = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m0, unsigned char *)
- + sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len)
- + sizeof(__m128i)));
- __m128i tmpdata2 = _mm_loadu_si128((__m128i*)(rte_pktmbuf_mtod(m0, unsigned char *)
- + sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len)
- + sizeof(__m128i) + sizeof(__m128i)));
+ __m128i tmpdata0 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len)));
+ __m128i tmpdata1 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i)));
+ __m128i tmpdata2 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i) + sizeof(__m128i)));
key->xmm[0] = _mm_and_si128(tmpdata0, mask0);
key->xmm[1] = tmpdata1;
key->xmm[2] = _mm_and_si128(tmpdata2, mask1);
{
struct ether_hdr *eth_hdr[4];
__attribute__((unused)) struct ipv6_hdr *ipv6_hdr[4];
- void *d_addr_bytes[4];
uint8_t dst_port[4];
int32_t ret[4];
union ipv6_5tuple_host key[4];
eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
/* Handle IPv6 headers.*/
- ipv6_hdr[0] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[0], unsigned char *) +
- sizeof(struct ether_hdr));
- ipv6_hdr[1] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[1], unsigned char *) +
- sizeof(struct ether_hdr));
- ipv6_hdr[2] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[2], unsigned char *) +
- sizeof(struct ether_hdr));
- ipv6_hdr[3] = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m[3], unsigned char *) +
- sizeof(struct ether_hdr));
+ ipv6_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv6_hdr *,
+ sizeof(struct ether_hdr));
+ ipv6_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv6_hdr *,
+ sizeof(struct ether_hdr));
+ ipv6_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv6_hdr *,
+ sizeof(struct ether_hdr));
+ ipv6_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv6_hdr *,
+ sizeof(struct ether_hdr));
get_ipv6_5tuple(m[0], mask1, mask2, &key[0]);
get_ipv6_5tuple(m[1], mask1, mask2, &key[1]);
if (dst_port[3] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[3]) == 0)
dst_port[3] = portid;
- /* 02:00:00:00:00:xx */
- d_addr_bytes[0] = ð_hdr[0]->d_addr.addr_bytes[0];
- d_addr_bytes[1] = ð_hdr[1]->d_addr.addr_bytes[0];
- d_addr_bytes[2] = ð_hdr[2]->d_addr.addr_bytes[0];
- d_addr_bytes[3] = ð_hdr[3]->d_addr.addr_bytes[0];
- *((uint64_t *)d_addr_bytes[0]) = 0x000000000002 + ((uint64_t)dst_port[0] << 40);
- *((uint64_t *)d_addr_bytes[1]) = 0x000000000002 + ((uint64_t)dst_port[1] << 40);
- *((uint64_t *)d_addr_bytes[2]) = 0x000000000002 + ((uint64_t)dst_port[2] << 40);
- *((uint64_t *)d_addr_bytes[3]) = 0x000000000002 + ((uint64_t)dst_port[3] << 40);
+ /* dst addr */
+ *(uint64_t *)ð_hdr[0]->d_addr = dest_eth_addr[dst_port[0]];
+ *(uint64_t *)ð_hdr[1]->d_addr = dest_eth_addr[dst_port[1]];
+ *(uint64_t *)ð_hdr[2]->d_addr = dest_eth_addr[dst_port[2]];
+ *(uint64_t *)ð_hdr[3]->d_addr = dest_eth_addr[dst_port[3]];
/* src addr */
ether_addr_copy(&ports_eth_addr[dst_port[0]], ð_hdr[0]->s_addr);
{
struct ether_hdr *eth_hdr;
struct ipv4_hdr *ipv4_hdr;
- void *d_addr_bytes;
uint8_t dst_port;
eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
if (m->ol_flags & PKT_RX_IPV4_HDR) {
/* Handle IPv4 headers.*/
- ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m, unsigned char *) +
- sizeof(struct ether_hdr));
+ ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *,
+ sizeof(struct ether_hdr));
#ifdef DO_RFC_1812_CHECKS
/* Check to make sure the packet is valid (RFC1812) */
- if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt.pkt_len) < 0) {
+ if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) {
rte_pktmbuf_free(m);
return;
}
(enabled_port_mask & 1 << dst_port) == 0)
dst_port = portid;
- /* 02:00:00:00:00:xx */
- d_addr_bytes = ð_hdr->d_addr.addr_bytes[0];
- *((uint64_t *)d_addr_bytes) = ETHER_LOCAL_ADMIN_ADDR +
- ((uint64_t)dst_port << 40);
-
#ifdef DO_RFC_1812_CHECKS
/* Update time to live and header checksum */
--(ipv4_hdr->time_to_live);
++(ipv4_hdr->hdr_checksum);
#endif
+ /* dst addr */
+ *(uint64_t *)ð_hdr->d_addr = dest_eth_addr[dst_port];
/* src addr */
ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr);
/* Handle IPv6 headers.*/
struct ipv6_hdr *ipv6_hdr;
- ipv6_hdr = (struct ipv6_hdr *)(rte_pktmbuf_mtod(m, unsigned char *) +
- sizeof(struct ether_hdr));
+ ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *,
+ sizeof(struct ether_hdr));
dst_port = get_ipv6_dst_port(ipv6_hdr, portid, qconf->ipv6_lookup_struct);
if (dst_port >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port) == 0)
dst_port = portid;
- /* 02:00:00:00:00:xx */
- d_addr_bytes = ð_hdr->d_addr.addr_bytes[0];
- *((uint64_t *)d_addr_bytes) = ETHER_LOCAL_ADMIN_ADDR +
- ((uint64_t)dst_port << 40);
+ /* dst addr */
+ *(uint64_t *)ð_hdr->d_addr = dest_eth_addr[dst_port];
/* src addr */
ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr);
if (likely(flag != 0)) {
rte_lpm_lookupx4(qconf->ipv4_lookup_struct, dip, dprt, portid);
} else {
- dst.m = dip;
+ dst.x = dip;
dprt[0] = get_dst_port(qconf, pkt[0], dst.u32[0], portid);
dprt[1] = get_dst_port(qconf, pkt[1], dst.u32[1], portid);
dprt[2] = get_dst_port(qconf, pkt[2], dst.u32[2], portid);
__m128i ve[FWDSTEP];
__m128i *p[FWDSTEP];
- p[0] = (rte_pktmbuf_mtod(pkt[0], __m128i *));
- p[1] = (rte_pktmbuf_mtod(pkt[1], __m128i *));
- p[2] = (rte_pktmbuf_mtod(pkt[2], __m128i *));
- p[3] = (rte_pktmbuf_mtod(pkt[3], __m128i *));
+ p[0] = rte_pktmbuf_mtod(pkt[0], __m128i *);
+ p[1] = rte_pktmbuf_mtod(pkt[1], __m128i *);
+ p[2] = rte_pktmbuf_mtod(pkt[2], __m128i *);
+ p[3] = rte_pktmbuf_mtod(pkt[3], __m128i *);
ve[0] = val_eth[dst_port[0]];
te[0] = _mm_load_si128(p[0]);
&dst_port[3], pkt[3]->ol_flags);
}
+/*
+ * We group consecutive packets with the same destionation port into one burst.
+ * To avoid extra latency this is done together with some other packet
+ * processing, but after we made a final decision about packet's destination.
+ * To do this we maintain:
+ * pnum - array of number of consecutive packets with the same dest port for
+ * each packet in the input burst.
+ * lp - pointer to the last updated element in the pnum.
+ * dlp - dest port value lp corresponds to.
+ */
+
+#define GRPSZ (1 << FWDSTEP)
+#define GRPMSK (GRPSZ - 1)
+
+#define GROUP_PORT_STEP(dlp, dcp, lp, pn, idx) do { \
+ if (likely((dlp) == (dcp)[(idx)])) { \
+ (lp)[0]++; \
+ } else { \
+ (dlp) = (dcp)[idx]; \
+ (lp) = (pn) + (idx); \
+ (lp)[0] = 1; \
+ } \
+} while (0)
+
+/*
+ * Group consecutive packets with the same destination port in bursts of 4.
+ * Suppose we have array of destionation ports:
+ * dst_port[] = {a, b, c, d,, e, ... }
+ * dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>.
+ * We doing 4 comparisions at once and the result is 4 bit mask.
+ * This mask is used as an index into prebuild array of pnum values.
+ */
+static inline uint16_t *
+port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2)
+{
+ static const struct {
+ uint64_t pnum; /* prebuild 4 values for pnum[]. */
+ int32_t idx; /* index for new last updated elemnet. */
+ uint16_t lpv; /* add value to the last updated element. */
+ } gptbl[GRPSZ] = {
+ {
+ /* 0: a != b, b != c, c != d, d != e */
+ .pnum = UINT64_C(0x0001000100010001),
+ .idx = 4,
+ .lpv = 0,
+ },
+ {
+ /* 1: a == b, b != c, c != d, d != e */
+ .pnum = UINT64_C(0x0001000100010002),
+ .idx = 4,
+ .lpv = 1,
+ },
+ {
+ /* 2: a != b, b == c, c != d, d != e */
+ .pnum = UINT64_C(0x0001000100020001),
+ .idx = 4,
+ .lpv = 0,
+ },
+ {
+ /* 3: a == b, b == c, c != d, d != e */
+ .pnum = UINT64_C(0x0001000100020003),
+ .idx = 4,
+ .lpv = 2,
+ },
+ {
+ /* 4: a != b, b != c, c == d, d != e */
+ .pnum = UINT64_C(0x0001000200010001),
+ .idx = 4,
+ .lpv = 0,
+ },
+ {
+ /* 5: a == b, b != c, c == d, d != e */
+ .pnum = UINT64_C(0x0001000200010002),
+ .idx = 4,
+ .lpv = 1,
+ },
+ {
+ /* 6: a != b, b == c, c == d, d != e */
+ .pnum = UINT64_C(0x0001000200030001),
+ .idx = 4,
+ .lpv = 0,
+ },
+ {
+ /* 7: a == b, b == c, c == d, d != e */
+ .pnum = UINT64_C(0x0001000200030004),
+ .idx = 4,
+ .lpv = 3,
+ },
+ {
+ /* 8: a != b, b != c, c != d, d == e */
+ .pnum = UINT64_C(0x0002000100010001),
+ .idx = 3,
+ .lpv = 0,
+ },
+ {
+ /* 9: a == b, b != c, c != d, d == e */
+ .pnum = UINT64_C(0x0002000100010002),
+ .idx = 3,
+ .lpv = 1,
+ },
+ {
+ /* 0xa: a != b, b == c, c != d, d == e */
+ .pnum = UINT64_C(0x0002000100020001),
+ .idx = 3,
+ .lpv = 0,
+ },
+ {
+ /* 0xb: a == b, b == c, c != d, d == e */
+ .pnum = UINT64_C(0x0002000100020003),
+ .idx = 3,
+ .lpv = 2,
+ },
+ {
+ /* 0xc: a != b, b != c, c == d, d == e */
+ .pnum = UINT64_C(0x0002000300010001),
+ .idx = 2,
+ .lpv = 0,
+ },
+ {
+ /* 0xd: a == b, b != c, c == d, d == e */
+ .pnum = UINT64_C(0x0002000300010002),
+ .idx = 2,
+ .lpv = 1,
+ },
+ {
+ /* 0xe: a != b, b == c, c == d, d == e */
+ .pnum = UINT64_C(0x0002000300040001),
+ .idx = 1,
+ .lpv = 0,
+ },
+ {
+ /* 0xf: a == b, b == c, c == d, d == e */
+ .pnum = UINT64_C(0x0002000300040005),
+ .idx = 0,
+ .lpv = 4,
+ },
+ };
+
+ union {
+ uint16_t u16[FWDSTEP + 1];
+ uint64_t u64;
+ } *pnum = (void *)pn;
+
+ int32_t v;
+
+ dp1 = _mm_cmpeq_epi16(dp1, dp2);
+ dp1 = _mm_unpacklo_epi16(dp1, dp1);
+ v = _mm_movemask_ps((__m128)dp1);
+
+ /* update last port counter. */
+ lp[0] += gptbl[v].lpv;
+
+ /* if dest port value has changed. */
+ if (v != GRPMSK) {
+ lp = pnum->u16 + gptbl[v].idx;
+ lp[0] = 1;
+ pnum->u64 = gptbl[v].pnum;
+ }
+
+ return lp;
+}
+
#endif /* APP_LOOKUP_METHOD */
/* main processing loop */
#if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
(ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
int32_t k;
+ uint16_t dlp;
+ uint16_t *lp;
uint16_t dst_port[MAX_PKT_BURST];
__m128i dip[MAX_PKT_BURST / FWDSTEP];
uint32_t flag[MAX_PKT_BURST / FWDSTEP];
+ uint16_t pnum[MAX_PKT_BURST + 1];
#endif
prev_tsc = 0;
&pkts_burst[j], &dst_port[j]);
}
+ /*
+ * Finish packet processing and group consecutive
+ * packets with the same destination port.
+ */
k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
- for (j = 0; j != k; j += FWDSTEP) {
- processx4_step3(&pkts_burst[j], &dst_port[j]);
+ if (k != 0) {
+ __m128i dp1, dp2;
+
+ lp = pnum;
+ lp[0] = 1;
+
+ processx4_step3(pkts_burst, dst_port);
+
+ /* dp1: <d[0], d[1], d[2], d[3], ... > */
+ dp1 = _mm_loadu_si128((__m128i *)dst_port);
+
+ for (j = FWDSTEP; j != k; j += FWDSTEP) {
+ processx4_step3(&pkts_burst[j],
+ &dst_port[j]);
+
+ /*
+ * dp2:
+ * <d[j-3], d[j-2], d[j-1], d[j], ... >
+ */
+ dp2 = _mm_loadu_si128((__m128i *)
+ &dst_port[j - FWDSTEP + 1]);
+ lp = port_groupx4(&pnum[j - FWDSTEP],
+ lp, dp1, dp2);
+
+ /*
+ * dp1:
+ * <d[j], d[j+1], d[j+2], d[j+3], ... >
+ */
+ dp1 = _mm_srli_si128(dp2,
+ (FWDSTEP - 1) *
+ sizeof(dst_port[0]));
+ }
+
+ /*
+ * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... >
+ */
+ dp2 = _mm_shufflelo_epi16(dp1, 0xf9);
+ lp = port_groupx4(&pnum[j - FWDSTEP], lp,
+ dp1, dp2);
+
+ /*
+ * remove values added by the last repeated
+ * dst port.
+ */
+ lp[0]--;
+ dlp = dst_port[j - 1];
+ } else {
+ /* set dlp and lp to the never used values. */
+ dlp = BAD_PORT - 1;
+ lp = pnum + MAX_PKT_BURST;
}
/* Process up to last 3 packets one by one. */
case 3:
process_packet(qconf, pkts_burst[j],
dst_port + j, portid);
+ GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
j++;
case 2:
process_packet(qconf, pkts_burst[j],
dst_port + j, portid);
+ GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
j++;
case 1:
process_packet(qconf, pkts_burst[j],
dst_port + j, portid);
+ GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
j++;
}
/*
* Send packets out, through destination port.
- * Try to group packets with the same destination port.
+ * Consecuteve pacekts with the same destination port
+ * are already grouped together.
* If destination port for the packet equals BAD_PORT,
* then free the packet without sending it out.
*/
- for (j = 0; j < nb_rx; j = k) {
+ for (j = 0; j < nb_rx; j += k) {
- uint16_t cn, pn = dst_port[j];
+ int32_t m;
+ uint16_t pn;
- k = j;
- do {
- cn = dst_port[k];
- } while (cn != BAD_PORT && pn == cn &&
- ++k < nb_rx);
+ pn = dst_port[j];
+ k = pnum[j];
- send_packetsx4(qconf, pn, pkts_burst + j,
- k - j);
-
- if (cn == BAD_PORT) {
- rte_pktmbuf_free(pkts_burst[k]);
- k += 1;
+ if (likely(pn != BAD_PORT)) {
+ send_packetsx4(qconf, pn,
+ pkts_burst + j, k);
+ } else {
+ for (m = j; m != j + k; m++)
+ rte_pktmbuf_free(pkts_burst[m]);
}
}
" -p PORTMASK: hexadecimal bitmask of ports to configure\n"
" -P : enable promiscuous mode\n"
" --config (port,queue,lcore): rx queues configuration\n"
+ " --eth-dest=X,MM:MM:MM:MM:MM:MM: optional, ethernet destination for port X\n"
" --no-numa: optional, disable numa awareness\n"
" --ipv6: optional, specify it if running ipv6 packets\n"
" --enable-jumbo: enable jumbo frame"
if(size >= sizeof(s))
return -1;
- rte_snprintf(s, sizeof(s), "%.*s", size, p);
+ snprintf(s, sizeof(s), "%.*s", size, p);
if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD)
return -1;
for (i = 0; i < _NUM_FLD; i++){
return 0;
}
+static void
+parse_eth_dest(const char *optarg)
+{
+ uint8_t portid;
+ char *port_end;
+ uint8_t c, *dest, peer_addr[6];
+
+ errno = 0;
+ portid = strtoul(optarg, &port_end, 10);
+ if (errno != 0 || port_end == optarg || *port_end++ != ',')
+ rte_exit(EXIT_FAILURE,
+ "Invalid eth-dest: %s", optarg);
+ if (portid >= RTE_MAX_ETHPORTS)
+ rte_exit(EXIT_FAILURE,
+ "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n",
+ portid, RTE_MAX_ETHPORTS);
+
+ if (cmdline_parse_etheraddr(NULL, port_end,
+ &peer_addr, sizeof(peer_addr)) < 0)
+ rte_exit(EXIT_FAILURE,
+ "Invalid ethernet address: %s\n",
+ port_end);
+ dest = (uint8_t *)&dest_eth_addr[portid];
+ for (c = 0; c < 6; c++)
+ dest[c] = peer_addr[c];
+ *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
+}
+
#define CMD_LINE_OPT_CONFIG "config"
+#define CMD_LINE_OPT_ETH_DEST "eth-dest"
#define CMD_LINE_OPT_NO_NUMA "no-numa"
#define CMD_LINE_OPT_IPV6 "ipv6"
#define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo"
char *prgname = argv[0];
static struct option lgopts[] = {
{CMD_LINE_OPT_CONFIG, 1, 0, 0},
+ {CMD_LINE_OPT_ETH_DEST, 1, 0, 0},
{CMD_LINE_OPT_NO_NUMA, 0, 0, 0},
{CMD_LINE_OPT_IPV6, 0, 0, 0},
{CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0},
}
}
+ if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ETH_DEST,
+ sizeof(CMD_LINE_OPT_CONFIG))) {
+ parse_eth_dest(optarg);
+ }
+
if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_NUMA,
sizeof(CMD_LINE_OPT_NO_NUMA))) {
printf("numa is disabled \n");
printf("jumbo frame is enabled - disabling simple TX path\n");
port_conf.rxmode.jumbo_frame = 1;
- tx_conf.txq_flags = 0;
/* if no max-pkt-len set, use the default value ETHER_MAX_LEN */
if (0 == getopt_long(argc, argvopt, "", &lenopts, &option_index)) {
static void
print_ethaddr(const char *name, const struct ether_addr *eth_addr)
{
- printf ("%s%02X:%02X:%02X:%02X:%02X:%02X", name,
- eth_addr->addr_bytes[0],
- eth_addr->addr_bytes[1],
- eth_addr->addr_bytes[2],
- eth_addr->addr_bytes[3],
- eth_addr->addr_bytes[4],
- eth_addr->addr_bytes[5]);
+ char buf[ETHER_ADDR_FMT_SIZE];
+ ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr);
+ printf("%s%s", name, buf);
}
#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
char s[64];
/* create ipv4 hash */
- rte_snprintf(s, sizeof(s), "ipv4_l3fwd_hash_%d", socketid);
+ snprintf(s, sizeof(s), "ipv4_l3fwd_hash_%d", socketid);
ipv4_l3fwd_hash_params.name = s;
ipv4_l3fwd_hash_params.socket_id = socketid;
ipv4_l3fwd_lookup_struct[socketid] = rte_hash_create(&ipv4_l3fwd_hash_params);
"socket %d\n", socketid);
/* create ipv6 hash */
- rte_snprintf(s, sizeof(s), "ipv6_l3fwd_hash_%d", socketid);
+ snprintf(s, sizeof(s), "ipv6_l3fwd_hash_%d", socketid);
ipv6_l3fwd_hash_params.name = s;
ipv6_l3fwd_hash_params.socket_id = socketid;
ipv6_l3fwd_lookup_struct[socketid] = rte_hash_create(&ipv6_l3fwd_hash_params);
char s[64];
/* create the LPM table */
- rte_snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid);
+ snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid);
ipv4_l3fwd_lookup_struct[socketid] = rte_lpm_create(s, socketid,
IPV4_L3FWD_LPM_MAX_RULES, 0);
if (ipv4_l3fwd_lookup_struct[socketid] == NULL)
}
/* create the LPM6 table */
- rte_snprintf(s, sizeof(s), "IPV6_L3FWD_LPM_%d", socketid);
+ snprintf(s, sizeof(s), "IPV6_L3FWD_LPM_%d", socketid);
config.max_rules = IPV6_L3FWD_LPM_MAX_RULES;
config.number_tbl8s = IPV6_L3FWD_LPM_NUMBER_TBL8S;
socketid, lcore_id, NB_SOCKETS);
}
if (pktmbuf_pool[socketid] == NULL) {
- rte_snprintf(s, sizeof(s), "mbuf_pool_%d", socketid);
+ snprintf(s, sizeof(s), "mbuf_pool_%d", socketid);
pktmbuf_pool[socketid] =
- rte_mempool_create(s, nb_mbuf, MBUF_SIZE, MEMPOOL_CACHE_SIZE,
- sizeof(struct rte_pktmbuf_pool_private),
- rte_pktmbuf_pool_init, NULL,
- rte_pktmbuf_init, NULL,
- socketid, 0);
+ rte_pktmbuf_pool_create(s, nb_mbuf,
+ MEMPOOL_CACHE_SIZE, 0,
+ RTE_MBUF_DEFAULT_BUF_SIZE, socketid);
if (pktmbuf_pool[socketid] == NULL)
rte_exit(EXIT_FAILURE,
"Cannot init mbuf pool on socket %d\n", socketid);
}
int
-MAIN(int argc, char **argv)
+main(int argc, char **argv)
{
struct lcore_conf *qconf;
+ struct rte_eth_dev_info dev_info;
+ struct rte_eth_txconf *txconf;
int ret;
unsigned nb_ports;
uint16_t queueid;
argc -= ret;
argv += ret;
+ /* pre-init dst MACs for all ports to 02:00:00:00:00:xx */
+ for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
+ dest_eth_addr[portid] = ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40);
+ *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
+ }
+
/* parse application arguments (after the EAL ones) */
ret = parse_args(argc, argv);
if (ret < 0)
if (ret < 0)
rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
-
- if (rte_eal_pci_probe() < 0)
- rte_exit(EXIT_FAILURE, "Cannot probe PCI\n");
-
nb_ports = rte_eth_dev_count();
if (nb_ports > RTE_MAX_ETHPORTS)
nb_ports = RTE_MAX_ETHPORTS;
rte_eth_macaddr_get(portid, &ports_eth_addr[portid]);
print_ethaddr(" Address:", &ports_eth_addr[portid]);
printf(", ");
+ print_ethaddr("Destination:",
+ (const struct ether_addr *)&dest_eth_addr[portid]);
+ printf(", ");
/*
- * prepare dst and src MACs for each port.
+ * prepare src MACs for each port.
*/
- *(uint64_t *)(val_eth + portid) =
- ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40);
ether_addr_copy(&ports_eth_addr[portid],
(struct ether_addr *)(val_eth + portid) + 1);
printf("txq=%u,%d,%d ", lcore_id, queueid, socketid);
fflush(stdout);
+
+ rte_eth_dev_info_get(portid, &dev_info);
+ txconf = &dev_info.default_txconf;
+ if (port_conf.rxmode.jumbo_frame)
+ txconf->txq_flags = 0;
ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
- socketid, &tx_conf);
+ socketid, txconf);
if (ret < 0)
rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: err=%d, "
"port=%d\n", ret, portid);
fflush(stdout);
ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd,
- socketid, &rx_conf, pktmbuf_pool[socketid]);
+ socketid,
+ NULL,
+ pktmbuf_pool[socketid]);
if (ret < 0)
rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d,"
"port=%d\n", ret, portid);
git.droids-corp.org / dpdk.git / blobdiff