X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=examples%2Fperformance-thread%2Fl3fwd-thread%2Fmain.c;h=fa65234f3ca2e819138003de0bdfc3c96a15339f;hb=4daae08e7f5894cfa4cc729b21c1ac9750c2084c;hp=59b7678473241b828a5f3e54e0303df30b5d483a;hpb=dc81ebbacaeb87d9dab302576ab676564c78557e;p=dpdk.git diff --git a/examples/performance-thread/l3fwd-thread/main.c b/examples/performance-thread/l3fwd-thread/main.c index 59b7678473..fa65234f3c 100644 --- a/examples/performance-thread/l3fwd-thread/main.c +++ b/examples/performance-thread/l3fwd-thread/main.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -50,9 +50,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -61,7 +59,6 @@ #include #include #include -#include #include #include #include @@ -73,6 +70,7 @@ #include #include #include +#include #include #include @@ -90,16 +88,74 @@ #define APP_LOOKUP_METHOD APP_LOOKUP_LPM #endif +#ifndef __GLIBC__ /* sched_getcpu() is glibc specific */ +#define sched_getcpu() rte_lcore_id() +#endif + +static int +check_ptype(int portid) +{ + int i, ret; + int ipv4 = 0, ipv6 = 0; + + ret = rte_eth_dev_get_supported_ptypes(portid, RTE_PTYPE_L3_MASK, NULL, + 0); + if (ret <= 0) + return 0; + + uint32_t ptypes[ret]; + + ret = rte_eth_dev_get_supported_ptypes(portid, RTE_PTYPE_L3_MASK, + ptypes, ret); + for (i = 0; i < ret; ++i) { + if (ptypes[i] & RTE_PTYPE_L3_IPV4) + ipv4 = 1; + if (ptypes[i] & RTE_PTYPE_L3_IPV6) + ipv6 = 1; + } + + if (ipv4 && ipv6) + return 1; + + return 0; +} + +static inline void +parse_ptype(struct rte_mbuf *m) +{ + struct ether_hdr *eth_hdr; + uint32_t packet_type = RTE_PTYPE_UNKNOWN; + uint16_t ether_type; + + eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + ether_type = eth_hdr->ether_type; + if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv4)) + packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + else if (ether_type == rte_cpu_to_be_16(ETHER_TYPE_IPv6)) + packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + + m->packet_type = packet_type; +} + +static uint16_t +cb_parse_ptype(__rte_unused uint16_t port, __rte_unused uint16_t queue, + struct rte_mbuf *pkts[], uint16_t nb_pkts, + __rte_unused uint16_t max_pkts, __rte_unused void *user_param) +{ + unsigned int i; + + for (i = 0; i < nb_pkts; i++) + parse_ptype(pkts[i]); + + return nb_pkts; +} + /* * When set to zero, simple forwaring path is eanbled. * When set to one, optimized forwarding path is enabled. * Note that LPM optimisation path uses SSE4.1 instructions. */ -#if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && !defined(__SSE4_1__)) -#define ENABLE_MULTI_BUFFER_OPTIMIZE 0 -#else #define ENABLE_MULTI_BUFFER_OPTIMIZE 1 -#endif #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) #include @@ -126,10 +182,10 @@ */ #define NB_MBUF RTE_MAX(\ - (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \ - nb_ports*nb_lcores*MAX_PKT_BURST + \ - nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \ - nb_lcores*MEMPOOL_CACHE_SIZE), \ + (nb_ports*nb_rx_queue*nb_rxd + \ + nb_ports*nb_lcores*MAX_PKT_BURST + \ + nb_ports*n_tx_queue*nb_txd + \ + nb_lcores*MEMPOOL_CACHE_SIZE), \ (unsigned)8192) #define MAX_PKT_BURST 32 @@ -163,15 +219,16 @@ static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS]; static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; -static __m128i val_eth[RTE_MAX_ETHPORTS]; +static xmm_t val_eth[RTE_MAX_ETHPORTS]; /* replace first 12B of the ethernet header. */ #define MASK_ETH 0x3f /* mask of enabled ports */ static uint32_t enabled_port_mask; -static int promiscuous_on; /**< $et in promiscuous mode off by default. */ +static int promiscuous_on; /**< Set in promiscuous mode off by default. */ static int numa_on = 1; /**< NUMA is enabled by default. */ +static int parse_ptype_on; #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) static int ipv6; /**< ipv6 is false by default. */ @@ -218,7 +275,7 @@ struct mbuf_table { }; struct lcore_rx_queue { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; } __rte_cache_aligned; @@ -228,7 +285,7 @@ struct lcore_rx_queue { #define MAX_LCORE_PARAMS 1024 struct rx_thread_params { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; uint8_t lcore_id; uint8_t thread_id; @@ -282,7 +339,7 @@ static struct rte_eth_conf port_conf = { .hw_ip_checksum = 1, /**< IP checksum offload enabled */ .hw_vlan_filter = 0, /**< VLAN filtering disabled */ .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .rx_adv_conf = { .rss_conf = { @@ -299,13 +356,8 @@ static struct rte_mempool *pktmbuf_pool[NB_SOCKETS]; #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 #include #define DEFAULT_HASH_FUNC rte_hash_crc -#else -#include -#define DEFAULT_HASH_FUNC rte_jhash -#endif struct ipv4_5tuple { uint32_t ip_dst; @@ -422,17 +474,10 @@ ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len, t = k->proto; p = (const uint32_t *)&k->port_src; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 init_val = rte_hash_crc_4byte(t, init_val); init_val = rte_hash_crc_4byte(k->ip_src, init_val); init_val = rte_hash_crc_4byte(k->ip_dst, init_val); init_val = rte_hash_crc_4byte(*p, init_val); -#else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ - init_val = rte_jhash_1word(t, init_val); - init_val = rte_jhash_1word(k->ip_src, init_val); - init_val = rte_jhash_1word(k->ip_dst, init_val); - init_val = rte_jhash_1word(*p, init_val); -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ return init_val; } @@ -443,16 +488,13 @@ ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, const union ipv6_5tuple_host *k; uint32_t t; const uint32_t *p; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 const uint32_t *ip_src0, *ip_src1, *ip_src2, *ip_src3; const uint32_t *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3; -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ k = data; t = k->proto; p = (const uint32_t *)&k->port_src; -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 ip_src0 = (const uint32_t *) k->ip_src; ip_src1 = (const uint32_t *)(k->ip_src + 4); ip_src2 = (const uint32_t *)(k->ip_src + 8); @@ -471,12 +513,6 @@ ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, init_val = rte_hash_crc_4byte(*ip_dst2, init_val); init_val = rte_hash_crc_4byte(*ip_dst3, init_val); init_val = rte_hash_crc_4byte(*p, init_val); -#else /* RTE_MACHINE_CPUFLAG_SSE4_2 */ - init_val = rte_jhash_1word(t, init_val); - init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); - init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val); - init_val = rte_jhash_1word(*p, init_val); -#endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */ return init_val; } @@ -610,7 +646,7 @@ struct thread_tx_conf tx_thread[MAX_TX_THREAD]; /* Send burst of packets on an output interface */ static inline int -send_burst(struct thread_tx_conf *qconf, uint16_t n, uint8_t port) +send_burst(struct thread_tx_conf *qconf, uint16_t n, uint16_t port) { struct rte_mbuf **m_table; int ret; @@ -631,7 +667,7 @@ send_burst(struct thread_tx_conf *qconf, uint16_t n, uint8_t port) /* Enqueue a single packet, and send burst if queue is filled */ static inline int -send_single_packet(struct rte_mbuf *m, uint8_t port) +send_single_packet(struct rte_mbuf *m, uint16_t port) { uint16_t len; struct thread_tx_conf *qconf; @@ -655,9 +691,10 @@ send_single_packet(struct rte_mbuf *m, uint8_t port) return 0; } -#if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) -static inline __attribute__((always_inline)) void -send_packetsx4(uint8_t port, +#if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \ + (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)) +static __rte_always_inline void +send_packetsx4(uint16_t port, struct rte_mbuf *m[], uint32_t num) { uint32_t len, j, n; @@ -697,12 +734,15 @@ send_packetsx4(uint8_t port, case 0: qconf->tx_mbufs[port].m_table[len + j] = m[j]; j++; + /* fall-through */ case 3: qconf->tx_mbufs[port].m_table[len + j] = m[j]; j++; + /* fall-through */ case 2: qconf->tx_mbufs[port].m_table[len + j] = m[j]; j++; + /* fall-through */ case 1: qconf->tx_mbufs[port].m_table[len + j] = m[j]; j++; @@ -724,12 +764,15 @@ send_packetsx4(uint8_t port, case 0: qconf->tx_mbufs[port].m_table[j] = m[n + j]; j++; + /* fall-through */ case 3: qconf->tx_mbufs[port].m_table[j] = m[n + j]; j++; + /* fall-through */ case 2: qconf->tx_mbufs[port].m_table[j] = m[n + j]; j++; + /* fall-through */ case 1: qconf->tx_mbufs[port].m_table[j] = m[n + j]; j++; @@ -787,8 +830,8 @@ is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len) static __m128i mask0; static __m128i mask1; static __m128i mask2; -static inline uint8_t -get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, +static inline uint16_t +get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) { int ret = 0; @@ -801,11 +844,11 @@ get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, key.xmm = _mm_and_si128(data, mask0); /* Find destination port */ ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]); + return ((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]); } -static inline uint8_t -get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, +static inline uint16_t +get_ipv6_dst_port(void *ipv6_hdr, uint16_t portid, lookup_struct_t *ipv6_l3fwd_lookup_struct) { int ret = 0; @@ -828,36 +871,36 @@ get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, /* Find destination port */ ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]); + return ((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]); } #endif #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) -static inline uint8_t -get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, +static inline uint16_t +get_ipv4_dst_port(void *ipv4_hdr, uint16_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) { uint32_t next_hop; - return (uint8_t)((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, + return ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr), &next_hop) == 0) ? next_hop : portid); } -static inline uint8_t -get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, +static inline uint16_t +get_ipv6_dst_port(void *ipv6_hdr, uint16_t portid, lookup6_struct_t *ipv6_l3fwd_lookup_struct) { - uint8_t next_hop; + uint32_t next_hop; - return (uint8_t) ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct, + return ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct, ((struct ipv6_hdr *)ipv6_hdr)->dst_addr, &next_hop) == 0) ? next_hop : portid); } #endif -static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid) +static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint16_t portid) __attribute__((unused)); #if ((APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) && \ @@ -874,11 +917,11 @@ static inline void l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid) #define EXCLUDE_8TH_PKT 0x7f static inline void -simple_ipv4_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) +simple_ipv4_fwd_8pkts(struct rte_mbuf *m[8], uint16_t portid) { struct ether_hdr *eth_hdr[8]; struct ipv4_hdr *ipv4_hdr[8]; - uint8_t dst_port[8]; + uint16_t dst_port[8]; int32_t ret[8]; union ipv4_5tuple_host key[8]; __m128i data[8]; @@ -995,16 +1038,16 @@ simple_ipv4_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3], &key[4], &key[5], &key[6], &key[7]}; - rte_hash_lookup_multi(RTE_PER_LCORE(lcore_conf)->ipv4_lookup_struct, + rte_hash_lookup_bulk(RTE_PER_LCORE(lcore_conf)->ipv4_lookup_struct, &key_array[0], 8, ret); - dst_port[0] = (uint8_t) ((ret[0] < 0) ? portid : ipv4_l3fwd_out_if[ret[0]]); - dst_port[1] = (uint8_t) ((ret[1] < 0) ? portid : ipv4_l3fwd_out_if[ret[1]]); - dst_port[2] = (uint8_t) ((ret[2] < 0) ? portid : ipv4_l3fwd_out_if[ret[2]]); - dst_port[3] = (uint8_t) ((ret[3] < 0) ? portid : ipv4_l3fwd_out_if[ret[3]]); - dst_port[4] = (uint8_t) ((ret[4] < 0) ? portid : ipv4_l3fwd_out_if[ret[4]]); - dst_port[5] = (uint8_t) ((ret[5] < 0) ? portid : ipv4_l3fwd_out_if[ret[5]]); - dst_port[6] = (uint8_t) ((ret[6] < 0) ? portid : ipv4_l3fwd_out_if[ret[6]]); - dst_port[7] = (uint8_t) ((ret[7] < 0) ? portid : ipv4_l3fwd_out_if[ret[7]]); + dst_port[0] = ((ret[0] < 0) ? portid : ipv4_l3fwd_out_if[ret[0]]); + dst_port[1] = ((ret[1] < 0) ? portid : ipv4_l3fwd_out_if[ret[1]]); + dst_port[2] = ((ret[2] < 0) ? portid : ipv4_l3fwd_out_if[ret[2]]); + dst_port[3] = ((ret[3] < 0) ? portid : ipv4_l3fwd_out_if[ret[3]]); + dst_port[4] = ((ret[4] < 0) ? portid : ipv4_l3fwd_out_if[ret[4]]); + dst_port[5] = ((ret[5] < 0) ? portid : ipv4_l3fwd_out_if[ret[5]]); + dst_port[6] = ((ret[6] < 0) ? portid : ipv4_l3fwd_out_if[ret[6]]); + dst_port[7] = ((ret[7] < 0) ? portid : ipv4_l3fwd_out_if[ret[7]]); if (dst_port[0] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[0]) == 0) @@ -1101,10 +1144,10 @@ static inline void get_ipv6_5tuple(struct rte_mbuf *m0, __m128i mask0, } static inline void -simple_ipv6_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) +simple_ipv6_fwd_8pkts(struct rte_mbuf *m[8], uint16_t portid) { int32_t ret[8]; - uint8_t dst_port[8]; + uint16_t dst_port[8]; struct ether_hdr *eth_hdr[8]; union ipv6_5tuple_host key[8]; @@ -1149,16 +1192,16 @@ simple_ipv6_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3], &key[4], &key[5], &key[6], &key[7]}; - rte_hash_lookup_multi(RTE_PER_LCORE(lcore_conf)->ipv6_lookup_struct, + rte_hash_lookup_bulk(RTE_PER_LCORE(lcore_conf)->ipv6_lookup_struct, &key_array[0], 4, ret); - dst_port[0] = (uint8_t) ((ret[0] < 0) ? portid : ipv6_l3fwd_out_if[ret[0]]); - dst_port[1] = (uint8_t) ((ret[1] < 0) ? portid : ipv6_l3fwd_out_if[ret[1]]); - dst_port[2] = (uint8_t) ((ret[2] < 0) ? portid : ipv6_l3fwd_out_if[ret[2]]); - dst_port[3] = (uint8_t) ((ret[3] < 0) ? portid : ipv6_l3fwd_out_if[ret[3]]); - dst_port[4] = (uint8_t) ((ret[4] < 0) ? portid : ipv6_l3fwd_out_if[ret[4]]); - dst_port[5] = (uint8_t) ((ret[5] < 0) ? portid : ipv6_l3fwd_out_if[ret[5]]); - dst_port[6] = (uint8_t) ((ret[6] < 0) ? portid : ipv6_l3fwd_out_if[ret[6]]); - dst_port[7] = (uint8_t) ((ret[7] < 0) ? portid : ipv6_l3fwd_out_if[ret[7]]); + dst_port[0] = ((ret[0] < 0) ? portid : ipv6_l3fwd_out_if[ret[0]]); + dst_port[1] = ((ret[1] < 0) ? portid : ipv6_l3fwd_out_if[ret[1]]); + dst_port[2] = ((ret[2] < 0) ? portid : ipv6_l3fwd_out_if[ret[2]]); + dst_port[3] = ((ret[3] < 0) ? portid : ipv6_l3fwd_out_if[ret[3]]); + dst_port[4] = ((ret[4] < 0) ? portid : ipv6_l3fwd_out_if[ret[4]]); + dst_port[5] = ((ret[5] < 0) ? portid : ipv6_l3fwd_out_if[ret[5]]); + dst_port[6] = ((ret[6] < 0) ? portid : ipv6_l3fwd_out_if[ret[6]]); + dst_port[7] = ((ret[7] < 0) ? portid : ipv6_l3fwd_out_if[ret[7]]); if (dst_port[0] >= RTE_MAX_ETHPORTS || (enabled_port_mask & 1 << dst_port[0]) == 0) @@ -1205,24 +1248,24 @@ simple_ipv6_fwd_8pkts(struct rte_mbuf *m[8], uint8_t portid) ether_addr_copy(&ports_eth_addr[dst_port[6]], ð_hdr[6]->s_addr); ether_addr_copy(&ports_eth_addr[dst_port[7]], ð_hdr[7]->s_addr); - send_single_packet(m[0], (uint8_t)dst_port[0]); - send_single_packet(m[1], (uint8_t)dst_port[1]); - send_single_packet(m[2], (uint8_t)dst_port[2]); - send_single_packet(m[3], (uint8_t)dst_port[3]); - send_single_packet(m[4], (uint8_t)dst_port[4]); - send_single_packet(m[5], (uint8_t)dst_port[5]); - send_single_packet(m[6], (uint8_t)dst_port[6]); - send_single_packet(m[7], (uint8_t)dst_port[7]); + send_single_packet(m[0], dst_port[0]); + send_single_packet(m[1], dst_port[1]); + send_single_packet(m[2], dst_port[2]); + send_single_packet(m[3], dst_port[3]); + send_single_packet(m[4], dst_port[4]); + send_single_packet(m[5], dst_port[5]); + send_single_packet(m[6], dst_port[6]); + send_single_packet(m[7], dst_port[7]); } #endif /* APP_LOOKUP_METHOD */ -static inline __attribute__((always_inline)) void -l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid) +static __rte_always_inline void +l3fwd_simple_forward(struct rte_mbuf *m, uint16_t portid) { struct ether_hdr *eth_hdr; struct ipv4_hdr *ipv4_hdr; - uint8_t dst_port; + uint16_t dst_port; eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); @@ -1305,8 +1348,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid) * If we encounter invalid IPV4 packet, then set destination port for it * to BAD_PORT value. */ -static inline __attribute__((always_inline)) void -rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint32_t *dp, uint32_t ptype) +static __rte_always_inline void +rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype) { uint8_t ihl; @@ -1325,7 +1368,7 @@ rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint32_t *dp, uint32_t ptype) } #else -#define rfc1812_process(mb, dp) do { } while (0) +#define rfc1812_process(mb, dp, ptype) do { } while (0) #endif /* DO_RFC_1812_CHECKS */ #endif /* APP_LOOKUP_LPM && ENABLE_MULTI_BUFFER_OPTIMIZE */ @@ -1333,37 +1376,35 @@ rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint32_t *dp, uint32_t ptype) #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \ (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)) -static inline __attribute__((always_inline)) uint16_t -get_dst_port(struct rte_mbuf *pkt, uint32_t dst_ipv4, uint8_t portid) +static __rte_always_inline uint16_t +get_dst_port(struct rte_mbuf *pkt, uint32_t dst_ipv4, uint16_t portid) { - uint32_t next_hop_ipv4; - uint8_t next_hop_ipv6; + uint32_t next_hop; struct ipv6_hdr *ipv6_hdr; struct ether_hdr *eth_hdr; if (RTE_ETH_IS_IPV4_HDR(pkt->packet_type)) { - if (rte_lpm_lookup(RTE_PER_LCORE(lcore_conf)->ipv4_lookup_struct, - dst_ipv4, &next_hop_ipv4) != 0) { - next_hop_ipv4 = portid; - return next_hop_ipv4; - } + return (uint16_t) ((rte_lpm_lookup( + RTE_PER_LCORE(lcore_conf)->ipv4_lookup_struct, dst_ipv4, + &next_hop) == 0) ? next_hop : portid); + } else if (RTE_ETH_IS_IPV6_HDR(pkt->packet_type)) { + eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *); ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1); - if (rte_lpm6_lookup(RTE_PER_LCORE(lcore_conf)->ipv6_lookup_struct, - ipv6_hdr->dst_addr, &next_hop_ipv6) != 0) { - next_hop_ipv6 = portid; - return next_hop_ipv6; - } - } else { - next_hop_ipv4 = portid; - return next_hop_ipv4; + + return (uint16_t) ((rte_lpm6_lookup( + RTE_PER_LCORE(lcore_conf)->ipv6_lookup_struct, + ipv6_hdr->dst_addr, &next_hop) == 0) ? + next_hop : portid); + } + return portid; } static inline void -process_packet(struct rte_mbuf *pkt, uint32_t *dst_port, uint8_t portid) +process_packet(struct rte_mbuf *pkt, uint16_t *dst_port, uint16_t portid) { struct ether_hdr *eth_hdr; struct ipv4_hdr *ipv4_hdr; @@ -1430,9 +1471,9 @@ processx4_step1(struct rte_mbuf *pkt[FWDSTEP], static inline void processx4_step2(__m128i dip, uint32_t ipv4_flag, - uint32_t portid, + uint16_t portid, struct rte_mbuf *pkt[FWDSTEP], - uint32_t dprt[FWDSTEP]) + uint16_t dprt[FWDSTEP]) { rte_xmm_t dst; const __m128i bswap_mask = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, @@ -1444,7 +1485,11 @@ processx4_step2(__m128i dip, /* if all 4 packets are IPV4. */ if (likely(ipv4_flag)) { rte_lpm_lookupx4(RTE_PER_LCORE(lcore_conf)->ipv4_lookup_struct, dip, - dprt, portid); + dst.u32, portid); + + /* get rid of unused upper 16 bit for each dport. */ + dst.x = _mm_packs_epi32(dst.x, dst.x); + *(uint64_t *)dprt = dst.u64[0]; } else { dst.x = dip; dprt[0] = get_dst_port(pkt[0], dst.u32[0], portid); @@ -1459,7 +1504,7 @@ processx4_step2(__m128i dip, * Perform RFC1812 checks and updates for IPV4 packets. */ static inline void -processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint32_t dst_port[FWDSTEP]) +processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP]) { __m128i te[FWDSTEP]; __m128i ve[FWDSTEP]; @@ -1532,7 +1577,7 @@ processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint32_t dst_port[FWDSTEP]) * Suppose we have array of destionation ports: * dst_port[] = {a, b, c, d,, e, ... } * dp1 should contain: , dp2: . - * We doing 4 comparisions at once and the result is 4 bit mask. + * We doing 4 comparisons at once and the result is 4 bit mask. * This mask is used as an index into prebuild array of pnum values. */ static inline uint16_t * @@ -1657,9 +1702,9 @@ port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2) /* if dest port value has changed. */ if (v != GRPMSK) { - lp = pnum->u16 + gptbl[v].idx; - lp[0] = 1; pnum->u64 = gptbl[v].pnum; + pnum->u16[FWDSTEP] = 1; + lp = pnum->u16 + gptbl[v].idx; } return lp; @@ -1669,7 +1714,8 @@ port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2) static void process_burst(struct rte_mbuf *pkts_burst[MAX_PKT_BURST], int nb_rx, - uint8_t portid) { + uint16_t portid) +{ int j; @@ -1678,7 +1724,7 @@ process_burst(struct rte_mbuf *pkts_burst[MAX_PKT_BURST], int nb_rx, int32_t k; uint16_t dlp; uint16_t *lp; - uint32_t dst_port[MAX_PKT_BURST]; + uint16_t dst_port[MAX_PKT_BURST]; __m128i dip[MAX_PKT_BURST / FWDSTEP]; uint32_t ipv4_flag[MAX_PKT_BURST / FWDSTEP]; uint16_t pnum[MAX_PKT_BURST + 1]; @@ -1794,10 +1840,12 @@ process_burst(struct rte_mbuf *pkts_burst[MAX_PKT_BURST], int nb_rx, process_packet(pkts_burst[j], dst_port + j, portid); GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); j++; + /* fall-through */ case 2: process_packet(pkts_burst[j], dst_port + j, portid); GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); j++; + /* fall-through */ case 1: process_packet(pkts_burst[j], dst_port + j, portid); GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j); @@ -1838,12 +1886,12 @@ process_burst(struct rte_mbuf *pkts_burst[MAX_PKT_BURST], int nb_rx, for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ j + PREFETCH_OFFSET], void *)); - l3fwd_simple_forward(pkts_burst[j], portid, qconf); + l3fwd_simple_forward(pkts_burst[j], portid); } /* Forward remaining prefetched packets */ for (; j < nb_rx; j++) - l3fwd_simple_forward(pkts_burst[j], portid, qconf); + l3fwd_simple_forward(pkts_burst[j], portid); #endif /* ENABLE_MULTI_BUFFER_OPTIMIZE */ @@ -1987,7 +2035,7 @@ static void lthread_tx_per_ring(void *dummy) { int nb_rx; - uint8_t portid; + uint16_t portid; struct rte_ring *ring; struct thread_tx_conf *tx_conf; struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; @@ -2015,7 +2063,7 @@ lthread_tx_per_ring(void *dummy) */ SET_CPU_BUSY(tx_conf, CPU_POLL); nb_rx = rte_ring_sc_dequeue_burst(ring, (void **)pkts_burst, - MAX_PKT_BURST); + MAX_PKT_BURST, NULL); SET_CPU_IDLE(tx_conf, CPU_POLL); if (nb_rx > 0) { @@ -2042,7 +2090,7 @@ lthread_tx(void *args) struct lthread *lt; unsigned lcore_id; - uint8_t portid; + uint16_t portid; struct thread_tx_conf *tx_conf; tx_conf = (struct thread_tx_conf *)args; @@ -2089,7 +2137,8 @@ lthread_rx(void *dummy) int ret; uint16_t nb_rx; int i; - uint8_t portid, queueid; + uint16_t portid; + uint8_t queueid; int worker_id; int len[RTE_MAX_LCORE] = { 0 }; int old_len, new_len; @@ -2115,7 +2164,8 @@ lthread_rx(void *dummy) portid = rx_conf->rx_queue_list[i].port_id; queueid = rx_conf->rx_queue_list[i].queue_id; - RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", + RTE_LOG(INFO, L3FWD, + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", rte_lcore_id(), portid, queueid); } @@ -2151,7 +2201,7 @@ lthread_rx(void *dummy) ret = rte_ring_sp_enqueue_burst( rx_conf->ring[worker_id], (void **) pkts_burst, - nb_rx); + nb_rx, NULL); new_len = old_len + ret; @@ -2274,7 +2324,7 @@ pthread_tx(void *dummy) struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; uint64_t prev_tsc, diff_tsc, cur_tsc; int nb_rx; - uint8_t portid; + uint16_t portid; struct thread_tx_conf *tx_conf; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / @@ -2319,7 +2369,7 @@ pthread_tx(void *dummy) */ SET_CPU_BUSY(tx_conf, CPU_POLL); nb_rx = rte_ring_sc_dequeue_burst(tx_conf->ring, - (void **)pkts_burst, MAX_PKT_BURST); + (void **)pkts_burst, MAX_PKT_BURST, NULL); SET_CPU_IDLE(tx_conf, CPU_POLL); if (unlikely(nb_rx == 0)) { @@ -2343,7 +2393,8 @@ pthread_rx(void *dummy) uint32_t n; uint32_t nb_rx; unsigned lcore_id; - uint8_t portid, queueid; + uint8_t queueid; + uint16_t portid; struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; struct thread_rx_conf *rx_conf; @@ -2362,7 +2413,8 @@ pthread_rx(void *dummy) portid = rx_conf->rx_queue_list[i].port_id; queueid = rx_conf->rx_queue_list[i].queue_id; - RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", + RTE_LOG(INFO, L3FWD, + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", lcore_id, portid, queueid); } @@ -2391,7 +2443,7 @@ pthread_rx(void *dummy) SET_CPU_BUSY(rx_conf, CPU_PROCESS); worker_id = (worker_id + 1) % rx_conf->n_ring; n = rte_ring_sp_enqueue_burst(rx_conf->ring[worker_id], - (void **)pkts_burst, nb_rx); + (void **)pkts_burst, nb_rx, NULL); if (unlikely(n != nb_rx)) { uint32_t k; @@ -2490,7 +2542,7 @@ check_port_config(const unsigned nb_ports) } static uint8_t -get_port_n_rx_queues(const uint8_t port) +get_port_n_rx_queues(const uint16_t port) { int queue = -1; uint16_t i; @@ -2606,6 +2658,7 @@ print_usage(const char *prgname) " [--rx (port,queue,lcore,thread)[,(port,queue,lcore,thread]]" " [--tx (lcore,thread)[,(lcore,thread]]" " [--enable-jumbo [--max-pkt-len PKTLEN]]\n" + " [--parse-ptype]\n\n" " -p PORTMASK: hexadecimal bitmask of ports to configure\n" " -P : enable promiscuous mode\n" " --rx (port,queue,lcore,thread): rx queues configuration\n" @@ -2617,7 +2670,8 @@ print_usage(const char *prgname) " --enable-jumbo: enable jumbo frame" " which max packet len is PKTLEN in decimal (64-9600)\n" " --hash-entry-num: specify the hash entry number in hexadecimal to be setup\n" - " --no-lthreads: turn off lthread model\n", + " --no-lthreads: turn off lthread model\n" + " --parse-ptype: set to use software to analyze packet type\n\n", prgname); } @@ -2718,7 +2772,7 @@ parse_rx_config(const char *q_arg) return -1; } rx_thread_params_array[nb_rx_thread_params].port_id = - (uint8_t)int_fld[FLD_PORT]; + int_fld[FLD_PORT]; rx_thread_params_array[nb_rx_thread_params].queue_id = (uint8_t)int_fld[FLD_QUEUE]; rx_thread_params_array[nb_rx_thread_params].lcore_id = @@ -2802,7 +2856,7 @@ parse_stat_lcore(const char *stat_lcore) static void parse_eth_dest(const char *optarg) { - uint8_t portid; + uint16_t portid; char *port_end; uint8_t c, *dest, peer_addr[6]; @@ -2836,6 +2890,7 @@ parse_eth_dest(const char *optarg) #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo" #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num" #define CMD_LINE_OPT_NO_LTHREADS "no-lthreads" +#define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" /* Parse the argument given in the command line of the application */ static int @@ -2855,6 +2910,7 @@ parse_args(int argc, char **argv) {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0}, {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, 0}, {CMD_LINE_OPT_NO_LTHREADS, 0, 0, 0}, + {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0}, {NULL, 0, 0, 0} }; @@ -2931,6 +2987,12 @@ parse_args(int argc, char **argv) lthreads_on = 0; } + if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_PARSE_PTYPE, + sizeof(CMD_LINE_OPT_PARSE_PTYPE))) { + printf("software packet type parsing enabled\n"); + parse_ptype_on = 1; + } + if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ENABLE_JUMBO, sizeof(CMD_LINE_OPT_ENABLE_JUMBO))) { struct option lenopts = {"max-pkt-len", required_argument, 0, @@ -2979,7 +3041,7 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } @@ -3249,14 +3311,18 @@ static void setup_lpm(int socketid) { struct rte_lpm6_config config; + struct rte_lpm_config lpm_ipv4_config; unsigned i; int ret; char s[64]; /* create the LPM table */ snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid); - ipv4_l3fwd_lookup_struct[socketid] = rte_lpm_create(s, socketid, - IPV4_L3FWD_LPM_MAX_RULES, 0); + lpm_ipv4_config.max_rules = IPV4_L3FWD_LPM_MAX_RULES; + lpm_ipv4_config.number_tbl8s = 256; + lpm_ipv4_config.flags = 0; + ipv4_l3fwd_lookup_struct[socketid] = + rte_lpm_create(s, socketid, &lpm_ipv4_config); if (ipv4_l3fwd_lookup_struct[socketid] == NULL) rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table" " on socket %d\n", socketid); @@ -3373,11 +3439,12 @@ init_mem(unsigned nb_mbuf) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint16_t portid; + uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; printf("\nChecking link status"); @@ -3392,18 +3459,17 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, + printf( + "Port%d Link Up. Speed %u Mbps - %s\n", + portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); else - printf("Port %d Link Down\n", - (uint8_t)portid); + printf("Port %d Link Down\n", portid); continue; } /* clear all_ports_up flag if any link down */ - if (link.link_status == 0) { + if (link.link_status == ETH_LINK_DOWN) { all_ports_up = 0; break; } @@ -3434,10 +3500,10 @@ main(int argc, char **argv) int ret; int i; unsigned nb_ports; - uint16_t queueid; + uint16_t queueid, portid; unsigned lcore_id; uint32_t n_tx_queue, nb_lcores; - uint8_t portid, nb_rx_queue, queue, socketid; + uint8_t nb_rx_queue, queue, socketid; /* init EAL */ ret = rte_eal_init(argc, argv); @@ -3477,8 +3543,6 @@ main(int argc, char **argv) rte_exit(EXIT_FAILURE, "init_rx_rings failed\n"); nb_ports = rte_eth_dev_count(); - if (nb_ports > RTE_MAX_ETHPORTS) - nb_ports = RTE_MAX_ETHPORTS; if (check_port_config(nb_ports) < 0) rte_exit(EXIT_FAILURE, "check_port_config failed\n"); @@ -3509,6 +3573,13 @@ main(int argc, char **argv) rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%d\n", ret, portid); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "rte_eth_dev_adjust_nb_rx_tx_desc: err=%d, port=%d\n", + ret, portid); + rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); print_ethaddr(" Address:", &ports_eth_addr[portid]); printf(", "); @@ -3617,6 +3688,31 @@ main(int argc, char **argv) rte_eth_promiscuous_enable(portid); } + for (i = 0; i < n_rx_thread; i++) { + lcore_id = rx_thread[i].conf.lcore_id; + if (rte_lcore_is_enabled(lcore_id) == 0) + continue; + + /* check if hw packet type is supported */ + for (queue = 0; queue < rx_thread[i].n_rx_queue; ++queue) { + portid = rx_thread[i].rx_queue_list[queue].port_id; + queueid = rx_thread[i].rx_queue_list[queue].queue_id; + + if (parse_ptype_on) { + if (!rte_eth_add_rx_callback(portid, queueid, + cb_parse_ptype, NULL)) + rte_exit(EXIT_FAILURE, + "Failed to add rx callback: " + "port=%d\n", portid); + } else if (!check_ptype(portid)) + rte_exit(EXIT_FAILURE, + "Port %d cannot parse packet type.\n\n" + "Please add --parse-ptype to use sw " + "packet type analyzer.\n\n", + portid); + } + } + check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); if (lthreads_on) {