git.droids-corp.org
/
dpdk.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
net/mlx5: add pattern template management
[dpdk.git]
/
drivers
/
net
/
mlx5
/
mlx5_rxtx_vec_sse.h
diff --git
a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
index
661fa72
..
185d269
100644
(file)
--- a/
drivers/net/mlx5/mlx5_rxtx_vec_sse.h
+++ b/
drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@
-204,12
+204,12
@@
rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
const __m128i ft_mask =
_mm_set1_epi32(0xffffff00);
const __m128i fdir_flags =
const __m128i ft_mask =
_mm_set1_epi32(0xffffff00);
const __m128i fdir_flags =
- _mm_set1_epi32(
PKT
_RX_FDIR);
+ _mm_set1_epi32(
RTE_MBUF_F
_RX_FDIR);
const __m128i fdir_all_flags =
const __m128i fdir_all_flags =
- _mm_set1_epi32(
PKT
_RX_FDIR |
-
PKT
_RX_FDIR_ID);
+ _mm_set1_epi32(
RTE_MBUF_F
_RX_FDIR |
+
RTE_MBUF_F
_RX_FDIR_ID);
__m128i fdir_id_flags =
__m128i fdir_id_flags =
- _mm_set1_epi32(
PKT
_RX_FDIR_ID);
+ _mm_set1_epi32(
RTE_MBUF_F
_RX_FDIR_ID);
/* Extract flow_tag field. */
__m128i ftag0 =
/* Extract flow_tag field. */
__m128i ftag0 =
@@
-223,7
+223,7
@@
rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
ol_flags_mask = _mm_or_si128(ol_flags_mask,
fdir_all_flags);
ol_flags_mask = _mm_or_si128(ol_flags_mask,
fdir_all_flags);
- /* Set
PKT
_RX_FDIR if flow tag is non-zero. */
+ /* Set
RTE_MBUF_F
_RX_FDIR if flow tag is non-zero. */
ol_flags = _mm_or_si128(ol_flags,
_mm_andnot_si128(invalid_mask,
fdir_flags));
ol_flags = _mm_or_si128(ol_flags,
_mm_andnot_si128(invalid_mask,
fdir_flags));
@@
-260,8
+260,8
@@
rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
const uint8_t pkt_hdr3 =
_mm_extract_epi8(mcqe2, 8);
const __m128i vlan_mask =
const uint8_t pkt_hdr3 =
_mm_extract_epi8(mcqe2, 8);
const __m128i vlan_mask =
- _mm_set1_epi32(
PKT
_RX_VLAN |
-
PKT
_RX_VLAN_STRIPPED);
+ _mm_set1_epi32(
RTE_MBUF_F
_RX_VLAN |
+
RTE_MBUF_F
_RX_VLAN_STRIPPED);
const __m128i cv_mask =
_mm_set1_epi32(MLX5_CQE_VLAN_STRIPPED);
const __m128i pkt_cv =
const __m128i cv_mask =
_mm_set1_epi32(MLX5_CQE_VLAN_STRIPPED);
const __m128i pkt_cv =
@@
-303,7
+303,7
@@
rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
}
}
const __m128i hash_flags =
}
}
const __m128i hash_flags =
- _mm_set1_epi32(
PKT
_RX_RSS_HASH);
+ _mm_set1_epi32(
RTE_MBUF_F
_RX_RSS_HASH);
const __m128i rearm_flags =
_mm_set1_epi32((uint32_t)t_pkt->ol_flags);
const __m128i rearm_flags =
_mm_set1_epi32((uint32_t)t_pkt->ol_flags);
@@
-381,7
+381,7
@@
rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, __m128i cqes[4],
{
__m128i pinfo0, pinfo1;
__m128i pinfo, ptype;
{
__m128i pinfo0, pinfo1;
__m128i pinfo, ptype;
- __m128i ol_flags = _mm_set1_epi32(rxq->rss_hash *
PKT
_RX_RSS_HASH |
+ __m128i ol_flags = _mm_set1_epi32(rxq->rss_hash *
RTE_MBUF_F
_RX_RSS_HASH |
rxq->hw_timestamp * rxq->timestamp_rx_flag);
__m128i cv_flags;
const __m128i zero = _mm_setzero_si128();
rxq->hw_timestamp * rxq->timestamp_rx_flag);
__m128i cv_flags;
const __m128i zero = _mm_setzero_si128();
@@
-390,17
+390,17
@@
rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, __m128i cqes[4],
const __m128i pinfo_mask = _mm_set1_epi32(0x3);
const __m128i cv_flag_sel =
_mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0,
const __m128i pinfo_mask = _mm_set1_epi32(0x3);
const __m128i cv_flag_sel =
_mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0,
- (uint8_t)((
PKT
_RX_IP_CKSUM_GOOD |
-
PKT
_RX_L4_CKSUM_GOOD) >> 1),
+ (uint8_t)((
RTE_MBUF_F
_RX_IP_CKSUM_GOOD |
+
RTE_MBUF_F
_RX_L4_CKSUM_GOOD) >> 1),
0,
0,
- (uint8_t)(
PKT
_RX_L4_CKSUM_GOOD >> 1),
+ (uint8_t)(
RTE_MBUF_F
_RX_L4_CKSUM_GOOD >> 1),
0,
0,
- (uint8_t)(
PKT
_RX_IP_CKSUM_GOOD >> 1),
- (uint8_t)(
PKT_RX_VLAN | PKT
_RX_VLAN_STRIPPED),
+ (uint8_t)(
RTE_MBUF_F
_RX_IP_CKSUM_GOOD >> 1),
+ (uint8_t)(
RTE_MBUF_F_RX_VLAN | RTE_MBUF_F
_RX_VLAN_STRIPPED),
0);
const __m128i cv_mask =
0);
const __m128i cv_mask =
- _mm_set1_epi32(
PKT_RX_IP_CKSUM_GOOD | PKT
_RX_L4_CKSUM_GOOD |
-
PKT_RX_VLAN | PKT
_RX_VLAN_STRIPPED);
+ _mm_set1_epi32(
RTE_MBUF_F_RX_IP_CKSUM_GOOD | RTE_MBUF_F
_RX_L4_CKSUM_GOOD |
+
RTE_MBUF_F_RX_VLAN | RTE_MBUF_F
_RX_VLAN_STRIPPED);
const __m128i mbuf_init =
_mm_load_si128((__m128i *)&rxq->mbuf_initializer);
__m128i rearm0, rearm1, rearm2, rearm3;
const __m128i mbuf_init =
_mm_load_si128((__m128i *)&rxq->mbuf_initializer);
__m128i rearm0, rearm1, rearm2, rearm3;
@@
-416,12
+416,12
@@
rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, __m128i cqes[4],
ptype = _mm_unpacklo_epi64(pinfo0, pinfo1);
if (rxq->mark) {
const __m128i pinfo_ft_mask = _mm_set1_epi32(0xffffff00);
ptype = _mm_unpacklo_epi64(pinfo0, pinfo1);
if (rxq->mark) {
const __m128i pinfo_ft_mask = _mm_set1_epi32(0xffffff00);
- const __m128i fdir_flags = _mm_set1_epi32(
PKT
_RX_FDIR);
- __m128i fdir_id_flags = _mm_set1_epi32(
PKT
_RX_FDIR_ID);
+ const __m128i fdir_flags = _mm_set1_epi32(
RTE_MBUF_F
_RX_FDIR);
+ __m128i fdir_id_flags = _mm_set1_epi32(
RTE_MBUF_F
_RX_FDIR_ID);
__m128i flow_tag, invalid_mask;
flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask);
__m128i flow_tag, invalid_mask;
flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask);
- /* Check if flow tag is non-zero then set
PKT
_RX_FDIR. */
+ /* Check if flow tag is non-zero then set
RTE_MBUF_F
_RX_FDIR. */
invalid_mask = _mm_cmpeq_epi32(flow_tag, zero);
ol_flags = _mm_or_si128(ol_flags,
_mm_andnot_si128(invalid_mask,
invalid_mask = _mm_cmpeq_epi32(flow_tag, zero);
ol_flags = _mm_or_si128(ol_flags,
_mm_andnot_si128(invalid_mask,
@@
-565,7
+565,7
@@
rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0);
/*
* A. load first Qword (8bytes) in one loop.
const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0);
/*
* A. load first Qword (8bytes) in one loop.
- * B. copy 4 mbuf pointers from elts ring to returing pkts.
+ * B. copy 4 mbuf pointers from elts ring to retur
n
ing pkts.
* C. load remained CQE data and extract necessary fields.
* Final 16bytes cqes[] extracted from original 64bytes CQE has the
* following structure:
* C. load remained CQE data and extract necessary fields.
* Final 16bytes cqes[] extracted from original 64bytes CQE has the
* following structure:
@@
-736,7
+736,13
@@
rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
*err |= _mm_cvtsi128_si64(opcode);
/* D.5 fill in mbuf - rearm_data and packet_type. */
rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
*err |= _mm_cvtsi128_si64(opcode);
/* D.5 fill in mbuf - rearm_data and packet_type. */
rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
- if (rxq->hw_timestamp) {
+ if (unlikely(rxq->shared)) {
+ pkts[pos]->port = cq[pos].user_index_low;
+ pkts[pos + p1]->port = cq[pos + p1].user_index_low;
+ pkts[pos + p2]->port = cq[pos + p2].user_index_low;
+ pkts[pos + p3]->port = cq[pos + p3].user_index_low;
+ }
+ if (unlikely(rxq->hw_timestamp)) {
int offset = rxq->timestamp_offset;
if (rxq->rt_timestamp) {
struct mlx5_dev_ctx_shared *sh = rxq->sh;
int offset = rxq->timestamp_offset;
if (rxq->rt_timestamp) {
struct mlx5_dev_ctx_shared *sh = rxq->sh;
@@
-766,28
+772,22
@@
rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
}
}
if (rxq->dynf_meta) {
}
}
if (rxq->dynf_meta) {
- /* This code is subject for futher optimization. */
+ /* This code is subject for fu
r
ther optimization. */
int32_t offs = rxq->flow_meta_offset;
uint32_t mask = rxq->flow_meta_port_mask;
int32_t offs = rxq->flow_meta_offset;
uint32_t mask = rxq->flow_meta_port_mask;
- uint32_t shift =
- __builtin_popcount(rxq->flow_meta_port_mask);
*RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *) =
*RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *) =
- (rte_be_to_cpu_32
- (cq[pos].flow_table_metadata) >> shift) &
- mask;
+ rte_be_to_cpu_32
+ (cq[pos].flow_table_metadata) & mask;
*RTE_MBUF_DYNFIELD(pkts[pos + 1], offs, uint32_t *) =
*RTE_MBUF_DYNFIELD(pkts[pos + 1], offs, uint32_t *) =
- (rte_be_to_cpu_32
- (cq[pos + p1].flow_table_metadata) >> shift) &
- mask;
+ rte_be_to_cpu_32
+ (cq[pos + p1].flow_table_metadata) & mask;
*RTE_MBUF_DYNFIELD(pkts[pos + 2], offs, uint32_t *) =
*RTE_MBUF_DYNFIELD(pkts[pos + 2], offs, uint32_t *) =
- (rte_be_to_cpu_32
- (cq[pos + p2].flow_table_metadata) >> shift) &
- mask;
+ rte_be_to_cpu_32
+ (cq[pos + p2].flow_table_metadata) & mask;
*RTE_MBUF_DYNFIELD(pkts[pos + 3], offs, uint32_t *) =
*RTE_MBUF_DYNFIELD(pkts[pos + 3], offs, uint32_t *) =
- (rte_be_to_cpu_32
- (cq[pos + p3].flow_table_metadata) >> shift) &
- mask;
+ rte_be_to_cpu_32
+ (cq[pos + p3].flow_table_metadata) & mask;
if (*RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *))
pkts[pos]->ol_flags |= rxq->flow_meta_mask;
if (*RTE_MBUF_DYNFIELD(pkts[pos + 1], offs, uint32_t *))
if (*RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *))
pkts[pos]->ol_flags |= rxq->flow_meta_mask;
if (*RTE_MBUF_DYNFIELD(pkts[pos + 1], offs, uint32_t *))