static inline void
desc_to_olflags_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
{
- __m128i vlan0, vlan1, rss;
- union {
- uint16_t e[4];
- uint64_t dword;
- } vol;
-
- /* mask everything except rss and vlan flags
- *bit2 is for vlan tag, bits 13:12 for rss
- */
- const __m128i rss_vlan_msk = _mm_set_epi16(
- 0x0000, 0x0000, 0x0000, 0x0000,
- 0x3004, 0x3004, 0x3004, 0x3004);
+ __m128i vlan0, vlan1, rss, l3_l4e;
+
+ /* mask everything except RSS, flow director and VLAN flags
+ * bit2 is for VLAN tag, bit11 for flow director indication
+ * bit13:12 for RSS indication.
+ */
+ const __m128i rss_vlan_msk = _mm_set_epi32(
+ 0x1c03804, 0x1c03804, 0x1c03804, 0x1c03804);
/* map rss and vlan type to rss hash and vlan flag */
const __m128i vlan_flags = _mm_set_epi8(0, 0, 0, 0,
0, 0, 0, 0,
- 0, 0, 0, PKT_RX_VLAN_PKT,
+ 0, 0, 0, PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED,
0, 0, 0, 0);
const __m128i rss_flags = _mm_set_epi8(0, 0, 0, 0,
0, 0, 0, 0,
- 0, 0, 0, 0,
- PKT_RX_FDIR, 0, PKT_RX_RSS_HASH, 0);
-
- vlan0 = _mm_unpackhi_epi16(descs[0], descs[1]);
- vlan1 = _mm_unpackhi_epi16(descs[2], descs[3]);
- vlan0 = _mm_unpacklo_epi32(vlan0, vlan1);
+ PKT_RX_RSS_HASH | PKT_RX_FDIR, PKT_RX_RSS_HASH, 0, 0,
+ 0, 0, PKT_RX_FDIR, 0);
+
+ const __m128i l3_l4e_flags = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,
+ PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD,
+ PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD,
+ PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD,
+ PKT_RX_EIP_CKSUM_BAD,
+ PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD,
+ PKT_RX_L4_CKSUM_BAD,
+ PKT_RX_IP_CKSUM_BAD,
+ 0);
+
+ vlan0 = _mm_unpackhi_epi32(descs[0], descs[1]);
+ vlan1 = _mm_unpackhi_epi32(descs[2], descs[3]);
+ vlan0 = _mm_unpacklo_epi64(vlan0, vlan1);
vlan1 = _mm_and_si128(vlan0, rss_vlan_msk);
vlan0 = _mm_shuffle_epi8(vlan_flags, vlan1);
- rss = _mm_srli_epi16(vlan1, 12);
+ rss = _mm_srli_epi32(vlan1, 11);
rss = _mm_shuffle_epi8(rss_flags, rss);
+ l3_l4e = _mm_srli_epi32(vlan1, 22);
+ l3_l4e = _mm_shuffle_epi8(l3_l4e_flags, l3_l4e);
+
vlan0 = _mm_or_si128(vlan0, rss);
- vol.dword = _mm_cvtsi128_si64(vlan0);
+ vlan0 = _mm_or_si128(vlan0, l3_l4e);
- rx_pkts[0]->ol_flags = vol.e[0];
- rx_pkts[1]->ol_flags = vol.e[1];
- rx_pkts[2]->ol_flags = vol.e[2];
- rx_pkts[3]->ol_flags = vol.e[3];
+ rx_pkts[0]->ol_flags = _mm_extract_epi16(vlan0, 0);
+ rx_pkts[1]->ol_flags = _mm_extract_epi16(vlan0, 2);
+ rx_pkts[2]->ol_flags = _mm_extract_epi16(vlan0, 4);
+ rx_pkts[3]->ol_flags = _mm_extract_epi16(vlan0, 6);
}
#else
#define desc_to_olflags_v(desc, rx_pkts) do {} while (0)
#endif
-#define PKTLEN_SHIFT (6)
-#define PKTLEN_MASK (0x3FFF)
-/* Handling the pkt len field is not aligned with 1byte, so shift is
- * needed to let it align
- */
+#define PKTLEN_SHIFT 10
+
static inline void
-desc_pktlen_align(__m128i descs[4])
+desc_to_ptype_v(__m128i descs[4], struct rte_mbuf **rx_pkts)
{
- __m128i pktlen0, pktlen1, zero;
- union {
- uint16_t e[4];
- uint64_t dword;
- } vol;
-
- /* mask everything except pktlen field*/
- const __m128i pktlen_msk = _mm_set_epi32(PKTLEN_MASK, PKTLEN_MASK,
- PKTLEN_MASK, PKTLEN_MASK);
-
- pktlen0 = _mm_unpackhi_epi32(descs[0], descs[2]);
- pktlen1 = _mm_unpackhi_epi32(descs[1], descs[3]);
- pktlen0 = _mm_unpackhi_epi32(pktlen0, pktlen1);
-
- zero = _mm_xor_si128(pktlen0, pktlen0);
-
- pktlen0 = _mm_srli_epi32(pktlen0, PKTLEN_SHIFT);
- pktlen0 = _mm_and_si128(pktlen0, pktlen_msk);
-
- pktlen0 = _mm_packs_epi32(pktlen0, zero);
- vol.dword = _mm_cvtsi128_si64(pktlen0);
- /* let the descriptor byte 15-14 store the pkt len */
- *((uint16_t *)&descs[0]+7) = vol.e[0];
- *((uint16_t *)&descs[1]+7) = vol.e[1];
- *((uint16_t *)&descs[2]+7) = vol.e[2];
- *((uint16_t *)&descs[3]+7) = vol.e[3];
+ __m128i ptype0 = _mm_unpackhi_epi64(descs[0], descs[1]);
+ __m128i ptype1 = _mm_unpackhi_epi64(descs[2], descs[3]);
+
+ ptype0 = _mm_srli_epi64(ptype0, 30);
+ ptype1 = _mm_srli_epi64(ptype1, 30);
+
+ rx_pkts[0]->packet_type = i40e_rxd_pkt_type_mapping(_mm_extract_epi8(ptype0, 0));
+ rx_pkts[1]->packet_type = i40e_rxd_pkt_type_mapping(_mm_extract_epi8(ptype0, 8));
+ rx_pkts[2]->packet_type = i40e_rxd_pkt_type_mapping(_mm_extract_epi8(ptype1, 0));
+ rx_pkts[3]->packet_type = i40e_rxd_pkt_type_mapping(_mm_extract_epi8(ptype1, 8));
}
/*
*/
rxdp = rxq->rx_ring + rxq->rx_tail;
- _mm_prefetch((const void *)rxdp, _MM_HINT_T0);
+ rte_prefetch0(rxdp);
/* See if we need to rearm the RX queue - gives the prefetch a bit
* of time to act
* D. fill info. from desc to mbuf
*/
- for (pos = 0, nb_pkts_recd = 0; pos < RTE_I40E_VPMD_RX_BURST;
+ for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
pos += RTE_I40E_DESCS_PER_LOOP,
rxdp += RTE_I40E_DESCS_PER_LOOP) {
__m128i descs[RTE_I40E_DESCS_PER_LOOP];
_mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2);
if (split_packet) {
- rte_prefetch0(&rx_pkts[pos]->cacheline1);
- rte_prefetch0(&rx_pkts[pos + 1]->cacheline1);
- rte_prefetch0(&rx_pkts[pos + 2]->cacheline1);
- rte_prefetch0(&rx_pkts[pos + 3]->cacheline1);
+ rte_mbuf_prefetch_part2(rx_pkts[pos]);
+ rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
+ rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
+ rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
}
- /*shift the pktlen field*/
- desc_pktlen_align(descs);
-
/* avoid compiler reorder optimization */
rte_compiler_barrier();
+ /* pkt 3,4 shift the pktlen field to be 16-bit aligned*/
+ const __m128i len3 = _mm_slli_epi32(descs[3], PKTLEN_SHIFT);
+ const __m128i len2 = _mm_slli_epi32(descs[2], PKTLEN_SHIFT);
+
+ /* merge the now-aligned packet length fields back in */
+ descs[3] = _mm_blend_epi16(descs[3], len3, 0x80);
+ descs[2] = _mm_blend_epi16(descs[2], len2, 0x80);
+
/* D.1 pkt 3,4 convert format from desc to pktmbuf */
pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk);
pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk);
pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
+ /* pkt 1,2 shift the pktlen field to be 16-bit aligned*/
+ const __m128i len1 = _mm_slli_epi32(descs[1], PKTLEN_SHIFT);
+ const __m128i len0 = _mm_slli_epi32(descs[0], PKTLEN_SHIFT);
+
+ /* merge the now-aligned packet length fields back in */
+ descs[1] = _mm_blend_epi16(descs[1], len1, 0x80);
+ descs[0] = _mm_blend_epi16(descs[0], len0, 0x80);
+
/* D.1 pkt 1,2 convert format from desc to pktmbuf */
pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);
pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk);
pkt_mb2);
_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
pkt_mb1);
+ desc_to_ptype_v(descs, &rx_pkts[pos]);
/* C.4 calc avaialbe number of desc */
var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
nb_pkts_recd += var;
return;
/* free all mbufs that are valid in the ring */
- for (i = rxq->rx_tail; i != rxq->rxrearm_start; i = (i + 1) & mask)
- rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+ if (rxq->rxrearm_nb == 0) {
+ for (i = 0; i < rxq->nb_rx_desc; i++) {
+ if (rxq->sw_ring[i].mbuf != NULL)
+ rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+ }
+ } else {
+ for (i = rxq->rx_tail;
+ i != rxq->rxrearm_start;
+ i = (i + 1) & mask) {
+ if (rxq->sw_ring[i].mbuf != NULL)
+ rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+ }
+ }
+
rxq->rxrearm_nb = rxq->nb_rx_desc;
/* set all entries to NULL */
#ifndef RTE_LIBRTE_I40E_RX_OLFLAGS_ENABLE
/* whithout rx ol_flags, no VP flag report */
if (rxmode->hw_vlan_strip != 0 ||
- rxmode->hw_vlan_extend != 0)
+ rxmode->hw_vlan_extend != 0 ||
+ rxmode->hw_ip_checksum != 0)
return -1;
#endif
/* - no csum error report support
* - no header split support
*/
- if (rxmode->hw_ip_checksum == 1 ||
- rxmode->header_split == 1)
+ if (rxmode->header_split == 1)
return -1;
return 0;
git.droids-corp.org / dpdk.git / blobdiff