X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fixgbe%2Fixgbe_rxtx_vec.c;h=9c5390ea92f2911af7045f801f88bc8412a8a0ca;hb=0369957f27aba349bd2b9f8a46c4a75f349aecb8;hp=abd10f6a789a9a6eed9970ad49619c12273473b8;hpb=abf7275bbaa2918a387e1f28f2c352053279c879;p=dpdk.git diff --git a/drivers/net/ixgbe/ixgbe_rxtx_vec.c b/drivers/net/ixgbe/ixgbe_rxtx_vec.c index abd10f6a78..9c5390ea92 100644 --- a/drivers/net/ixgbe/ixgbe_rxtx_vec.c +++ b/drivers/net/ixgbe/ixgbe_rxtx_vec.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -134,6 +134,12 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) */ #ifdef RTE_IXGBE_RX_OLFLAGS_ENABLE +#ifdef RTE_NEXT_ABI +#define OLFLAGS_MASK_V (((uint64_t)PKT_RX_VLAN_PKT << 48) | \ + ((uint64_t)PKT_RX_VLAN_PKT << 32) | \ + ((uint64_t)PKT_RX_VLAN_PKT << 16) | \ + ((uint64_t)PKT_RX_VLAN_PKT)) +#else #define OLFLAGS_MASK ((uint16_t)(PKT_RX_VLAN_PKT | PKT_RX_IPV4_HDR |\ PKT_RX_IPV4_HDR_EXT | PKT_RX_IPV6_HDR |\ PKT_RX_IPV6_HDR_EXT)) @@ -142,11 +148,26 @@ ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq) ((uint64_t)OLFLAGS_MASK << 16) | \ ((uint64_t)OLFLAGS_MASK)) #define PTYPE_SHIFT (1) +#endif /* RTE_NEXT_ABI */ + #define VTAG_SHIFT (3) static inline void desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) { +#ifdef RTE_NEXT_ABI + __m128i vtag0, vtag1; + union { + uint16_t e[4]; + uint64_t dword; + } vol; + + vtag0 = _mm_unpackhi_epi16(descs[0], descs[1]); + vtag1 = _mm_unpackhi_epi16(descs[2], descs[3]); + vtag1 = _mm_unpacklo_epi32(vtag0, vtag1); + vtag1 = _mm_srli_epi16(vtag1, VTAG_SHIFT); + vol.dword = _mm_cvtsi128_si64(vtag1) & OLFLAGS_MASK_V; +#else __m128i ptype0, ptype1, vtag0, vtag1; union { uint16_t e[4]; @@ -166,6 +187,7 @@ desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts) ptype1 = _mm_or_si128(ptype1, vtag1); vol.dword = _mm_cvtsi128_si64(ptype1) & OLFLAGS_MASK_V; +#endif /* RTE_NEXT_ABI */ rx_pkts[0]->ol_flags = vol.e[0]; rx_pkts[1]->ol_flags = vol.e[1]; @@ -196,6 +218,18 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, int pos; uint64_t var; __m128i shuf_msk; +#ifdef RTE_NEXT_ABI + __m128i crc_adjust = _mm_set_epi16( + 0, 0, 0, /* ignore non-length fields */ + -rxq->crc_len, /* sub crc on data_len */ + 0, /* ignore high-16bits of pkt_len */ + -rxq->crc_len, /* sub crc on pkt_len */ + 0, 0 /* ignore pkt_type field */ + ); + __m128i dd_check, eop_check; + __m128i desc_mask = _mm_set_epi32(0xFFFFFFFF, 0xFFFFFFFF, + 0xFFFFFFFF, 0xFFFF07F0); +#else __m128i crc_adjust = _mm_set_epi16( 0, 0, 0, 0, /* ignore non-length fields */ 0, /* ignore high-16bits of pkt_len */ @@ -204,6 +238,7 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 0 /* ignore pkt_type field */ ); __m128i dd_check, eop_check; +#endif /* RTE_NEXT_ABI */ if (unlikely(nb_pkts < RTE_IXGBE_VPMD_RX_BURST)) return 0; @@ -232,6 +267,18 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, eop_check = _mm_set_epi64x(0x0000000200000002LL, 0x0000000200000002LL); /* mask to shuffle from desc. to mbuf */ +#ifdef RTE_NEXT_ABI + shuf_msk = _mm_set_epi8( + 7, 6, 5, 4, /* octet 4~7, 32bits rss */ + 15, 14, /* octet 14~15, low 16 bits vlan_macip */ + 13, 12, /* octet 12~13, 16 bits data_len */ + 0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */ + 13, 12, /* octet 12~13, low 16 bits pkt_len */ + 0xFF, 0xFF, /* skip high 16 bits pkt_type */ + 1, /* octet 1, 8 bits pkt_type field */ + 0 /* octet 0, 4 bits offset 4 pkt_type field */ + ); +#else shuf_msk = _mm_set_epi8( 7, 6, 5, 4, /* octet 4~7, 32bits rss */ 0xFF, 0xFF, /* skip high 16 bits vlan_macip, zero out */ @@ -241,18 +288,28 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, 13, 12, /* octet 12~13, 16 bits data_len */ 0xFF, 0xFF /* skip pkt_type field */ ); +#endif /* RTE_NEXT_ABI */ /* Cache is empty -> need to scan the buffer rings, but first move * the next 'n' mbufs into the cache */ sw_ring = &rxq->sw_ring[rxq->rx_tail]; - /* - * A. load 4 packet in one loop +#ifdef RTE_NEXT_ABI + /* A. load 4 packet in one loop + * [A*. mask out 4 unused dirty field in desc] + * B. copy 4 mbuf point from swring to rx_pkts + * C. calc the number of DD bits among the 4 packets + * [C*. extract the end-of-packet bit, if requested] + * D. fill info. from desc to mbuf + */ +#else + /* A. load 4 packet in one loop * B. copy 4 mbuf point from swring to rx_pkts * C. calc the number of DD bits among the 4 packets * [C*. extract the end-of-packet bit, if requested] * D. fill info. from desc to mbuf */ +#endif /* RTE_NEXT_ABI */ for (pos = 0, nb_pkts_recd = 0; pos < RTE_IXGBE_VPMD_RX_BURST; pos += RTE_IXGBE_DESCS_PER_LOOP, rxdp += RTE_IXGBE_DESCS_PER_LOOP) { @@ -289,6 +346,16 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, /* B.2 copy 2 mbuf point into rx_pkts */ _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2); +#ifdef RTE_NEXT_ABI + /* A* mask out 0~3 bits RSS type */ + descs[3] = _mm_and_si128(descs[3], desc_mask); + descs[2] = _mm_and_si128(descs[2], desc_mask); + + /* A* mask out 0~3 bits RSS type */ + descs[1] = _mm_and_si128(descs[1], desc_mask); + descs[0] = _mm_and_si128(descs[0], desc_mask); +#endif /* RTE_NEXT_ABI */ + /* avoid compiler reorder optimization */ rte_compiler_barrier(); @@ -301,7 +368,11 @@ _recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts, /* C.1 4=>2 filter staterr info only */ sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]); +#ifdef RTE_NEXT_ABI + /* set ol_flags with vlan packet type */ +#else /* set ol_flags with packet type and vlan tag */ +#endif /* RTE_NEXT_ABI */ desc_to_olflags_v(descs, &rx_pkts[pos]); /* D.2 pkt 3,4 set in_port/nb_seg and remove crc */ @@ -537,8 +608,7 @@ ixgbe_tx_free_bufs(struct ixgbe_tx_queue *txq) * first buffer to free from S/W ring is at index * tx_next_dd - (tx_rs_thresh-1) */ - txep = &((struct ixgbe_tx_entry_v *)txq->sw_ring)[txq->tx_next_dd - - (n - 1)]; + txep = &txq->sw_ring_v[txq->tx_next_dd - (n - 1)]; m = __rte_pktmbuf_prefree_seg(txep[0].mbuf); if (likely(m != NULL)) { free[0] = m; @@ -607,7 +677,7 @@ ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, tx_id = txq->tx_tail; txdp = &txq->tx_ring[tx_id]; - txep = &((struct ixgbe_tx_entry_v *)txq->sw_ring)[tx_id]; + txep = &txq->sw_ring_v[tx_id]; txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts); @@ -628,7 +698,7 @@ ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, /* avoid reach the end of ring */ txdp = &(txq->tx_ring[tx_id]); - txep = &(((struct ixgbe_tx_entry_v *)txq->sw_ring)[tx_id]); + txep = &txq->sw_ring_v[tx_id]; } tx_backlog_entry(txep, tx_pkts, nb_commit); @@ -650,49 +720,67 @@ ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, return nb_pkts; } -static void -ixgbe_tx_queue_release_mbufs(struct ixgbe_tx_queue *txq) +static void __attribute__((cold)) +ixgbe_tx_queue_release_mbufs_vec(struct ixgbe_tx_queue *txq) { unsigned i; struct ixgbe_tx_entry_v *txe; - uint16_t nb_free, max_desc; + const uint16_t max_desc = (uint16_t)(txq->nb_tx_desc - 1); - if (txq->sw_ring != NULL) { - /* release the used mbufs in sw_ring */ - nb_free = txq->nb_tx_free; - max_desc = (uint16_t)(txq->nb_tx_desc - 1); - for (i = txq->tx_next_dd - (txq->tx_rs_thresh - 1); - nb_free < max_desc && i != txq->tx_tail; - i = (i + 1) & max_desc) { - txe = (struct ixgbe_tx_entry_v *)&txq->sw_ring[i]; - if (txe->mbuf != NULL) - rte_pktmbuf_free_seg(txe->mbuf); - } - /* reset tx_entry */ - for (i = 0; i < txq->nb_tx_desc; i++) { - txe = (struct ixgbe_tx_entry_v *)&txq->sw_ring[i]; - txe->mbuf = NULL; - } + if (txq->sw_ring == NULL || txq->nb_tx_free == max_desc) + return; + + /* release the used mbufs in sw_ring */ + for (i = txq->tx_next_dd - (txq->tx_rs_thresh - 1); + i != txq->tx_tail; + i = (i + 1) & max_desc) { + txe = &txq->sw_ring_v[i]; + rte_pktmbuf_free_seg(txe->mbuf); } + txq->nb_tx_free = max_desc; + + /* reset tx_entry */ + for (i = 0; i < txq->nb_tx_desc; i++) { + txe = &txq->sw_ring_v[i]; + txe->mbuf = NULL; + } +} + +void __attribute__((cold)) +ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq) +{ + const unsigned mask = rxq->nb_rx_desc - 1; + unsigned i; + + if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc) + return; + + /* free all mbufs that are valid in the ring */ + for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask) + rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf); + rxq->rxrearm_nb = rxq->nb_rx_desc; + + /* set all entries to NULL */ + memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc); } -static void +static void __attribute__((cold)) ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq) { if (txq == NULL) return; if (txq->sw_ring != NULL) { - rte_free((struct ixgbe_rx_entry *)txq->sw_ring - 1); - txq->sw_ring = NULL; + rte_free(txq->sw_ring_v - 1); + txq->sw_ring_v = NULL; } } -static void +static void __attribute__((cold)) ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) { static const union ixgbe_adv_tx_desc zeroed_desc = {{0}}; - struct ixgbe_tx_entry_v *txe = (struct ixgbe_tx_entry_v *)txq->sw_ring; + struct ixgbe_tx_entry_v *txe = txq->sw_ring_v; uint16_t i; /* Zero out HW ring memory */ @@ -723,12 +811,12 @@ ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq) } static const struct ixgbe_txq_ops vec_txq_ops = { - .release_mbufs = ixgbe_tx_queue_release_mbufs, + .release_mbufs = ixgbe_tx_queue_release_mbufs_vec, .free_swring = ixgbe_tx_free_swring, .reset = ixgbe_reset_tx_queue, }; -int +int __attribute__((cold)) ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq) { uintptr_t p; @@ -746,20 +834,21 @@ ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq) return 0; } -int ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq) +int __attribute__((cold)) +ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq) { - if (txq->sw_ring == NULL) + if (txq->sw_ring_v == NULL) return -1; /* leave the first one for overflow */ - txq->sw_ring = (struct ixgbe_tx_entry *) - ((struct ixgbe_tx_entry_v *)txq->sw_ring + 1); + txq->sw_ring_v = txq->sw_ring_v + 1; txq->ops = &vec_txq_ops; return 0; } -int ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) +int __attribute__((cold)) +ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) { #ifndef RTE_LIBRTE_IEEE1588 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;