app/crypto-perf: support AES-CCM

[dpdk.git] / drivers / net / i40e / i40e_rxtx_vec_sse.c

diff --git a/drivers/net/i40e/i40e_rxtx_vec_sse.c b/drivers/net/i40e/i40e_rxtx_vec_sse.c
index 8bc3399..779f14e 100644 (file)
--- a/drivers/net/i40e/i40e_rxtx_vec_sse.c
+++ b/drivers/net/i40e/i40e_rxtx_vec_sse.c
@@ -87,6 +87,8 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq)
                mb1 = rxep[1].mbuf;
 
                /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */
+               RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_physaddr) !=
+                               offsetof(struct rte_mbuf, buf_addr) + 8);
                vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
                vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
 
@@ -117,7 +119,7 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq)
 }
 
 static inline void
-desc_to_olflags_v(struct i40e_rx_queue *rxq, __m128i descs[4] __rte_unused,
+desc_to_olflags_v(struct i40e_rx_queue *rxq, __m128i descs[4],
        struct rte_mbuf **rx_pkts)
 {
        const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer);
@@ -203,6 +205,12 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, __m128i descs[4] __rte_unused,
        rearm1 = _mm_blend_epi16(mbuf_init, _mm_slli_si128(vlan0, 4), 0x10);
        rearm2 = _mm_blend_epi16(mbuf_init, vlan0, 0x10);
        rearm3 = _mm_blend_epi16(mbuf_init, _mm_srli_si128(vlan0, 4), 0x10);
+
+       /* write the rearm data and the olflags in one write */
+       RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
+                       offsetof(struct rte_mbuf, rearm_data) + 8);
+       RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
+                       RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));
        _mm_store_si128((__m128i *)&rx_pkts[0]->rearm_data, rearm0);
        _mm_store_si128((__m128i *)&rx_pkts[1]->rearm_data, rearm1);
        _mm_store_si128((__m128i *)&rx_pkts[2]->rearm_data, rearm2);
@@ -252,6 +260,15 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                                -rxq->crc_len, /* sub crc on pkt_len */
                                0, 0            /* ignore pkt_type field */
                        );
+       /*
+        * compile-time check the above crc_adjust layout is correct.
+        * NOTE: the first field (lowest address) is given last in set_epi16
+        * call above.
+        */
+       RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+                       offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+       RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+                       offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
        __m128i dd_check, eop_check;
 
        /* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */
@@ -296,6 +313,19 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                0xFF, 0xFF,  /* pkt_type set as unknown */
                0xFF, 0xFF  /*pkt_type set as unknown */
                );
+       /*
+        * Compile-time verify the shuffle mask
+        * NOTE: some field positions already verified above, but duplicated
+        * here for completeness in case of future modifications.
+        */
+       RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
+                       offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
+       RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
+                       offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
+       RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
+                       offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
+       RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
+                       offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);
 
        /* Cache is empty -> need to scan the buffer rings, but first move
         * the next 'n' mbufs into the cache
@@ -316,20 +346,26 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                __m128i descs[RTE_I40E_DESCS_PER_LOOP];
                __m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
                __m128i zero, staterr, sterr_tmp1, sterr_tmp2;
-               __m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */
+               /* 2 64 bit or 4 32 bit mbuf pointers in one XMM reg. */
+               __m128i mbp1;
+#if defined(RTE_ARCH_X86_64)
+               __m128i mbp2;
+#endif
 
-               /* B.1 load 1 mbuf point */
+               /* B.1 load 2 (64 bit) or 4 (32 bit) mbuf points */
                mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]);
                /* Read desc statuses backwards to avoid race condition */
                /* A.1 load 4 pkts desc */
                descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
                rte_compiler_barrier();
 
-               /* B.2 copy 2 mbuf point into rx_pkts  */
+               /* B.2 copy 2 64 bit or 4 32 bit mbuf point into rx_pkts */
                _mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
 
-               /* B.1 load 1 mbuf point */
+#if defined(RTE_ARCH_X86_64)
+               /* B.1 load 2 64 bit mbuf points */
                mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]);
+#endif
 
                descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
                rte_compiler_barrier();
@@ -338,8 +374,10 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
                rte_compiler_barrier();
                descs[0] = _mm_loadu_si128((__m128i *)(rxdp));
 
+#if defined(RTE_ARCH_X86_64)
                /* B.2 copy 2 mbuf point into rx_pkts  */
                _mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2);
+#endif
 
                if (split_packet) {
                        rte_mbuf_prefetch_part2(rx_pkts[pos]);
@@ -613,11 +651,5 @@ i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq)
 int __attribute__((cold))
 i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)
 {
-#ifndef RTE_LIBRTE_IEEE1588
-       /* need SSE4.1 support */
-       if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1))
-               return -1;
-#endif
-
        return i40e_rx_vec_dev_conf_condition_check_default(dev);
 }