From 04840ecbcf41bdad01a8ec3ad3e74ff927a54ac7 Mon Sep 17 00:00:00 2001 From: Thomas Monjalon Date: Thu, 29 Oct 2020 04:53:38 +0100 Subject: [PATCH] net/mlx5: switch Rx timestamp to dynamic mbuf field The mbuf timestamp is moved to a dynamic field in order to allow removal of the deprecated static field. The related mbuf flag is also replaced. The dynamic offset and flag are stored in struct mlx5_rxq_data to favor cache locality. Signed-off-by: Thomas Monjalon Reviewed-by: Ruifeng Wang Reviewed-by: David Christensen Acked-by: Viacheslav Ovsiienko Acked-by: David Marchand Acked-by: Olivier Matz --- drivers/net/mlx5/mlx5_rxq.c | 8 +++++ drivers/net/mlx5/mlx5_rxtx.c | 4 +-- drivers/net/mlx5/mlx5_rxtx.h | 19 +++++++++++ drivers/net/mlx5/mlx5_rxtx_vec_altivec.h | 41 +++++++++++----------- drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 43 ++++++++++++------------ drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 35 +++++++++---------- 6 files changed, 90 insertions(+), 60 deletions(-) diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index f1d8373079..52519910ee 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -1492,7 +1492,15 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, mlx5_max_lro_msg_size_adjust(dev, idx, max_lro_size); /* Toggle RX checksum offload if hardware supports it. */ tmpl->rxq.csum = !!(offloads & DEV_RX_OFFLOAD_CHECKSUM); + /* Configure Rx timestamp. */ tmpl->rxq.hw_timestamp = !!(offloads & DEV_RX_OFFLOAD_TIMESTAMP); + tmpl->rxq.timestamp_rx_flag = 0; + if (tmpl->rxq.hw_timestamp && rte_mbuf_dyn_rx_timestamp_register( + &tmpl->rxq.timestamp_offset, + &tmpl->rxq.timestamp_rx_flag) != 0) { + DRV_LOG(ERR, "Cannot register Rx timestamp field/flag"); + goto error; + } /* Configure VLAN stripping. */ tmpl->rxq.vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP); /* By default, FCS (CRC) is stripped by hardware. */ diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index e86468b67a..b577aab00b 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -1287,8 +1287,8 @@ rxq_cq_to_mbuf(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, if (rxq->rt_timestamp) ts = mlx5_txpp_convert_rx_ts(rxq->sh, ts); - pkt->timestamp = ts; - pkt->ol_flags |= PKT_RX_TIMESTAMP; + mlx5_timestamp_set(pkt, rxq->timestamp_offset, ts); + pkt->ol_flags |= rxq->timestamp_rx_flag; } } diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index 674296ee98..e9eca36b40 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -151,6 +151,8 @@ struct mlx5_rxq_data { /* CQ (UAR) access lock required for 32bit implementations */ #endif uint32_t tunnel; /* Tunnel information. */ + int timestamp_offset; /* Dynamic mbuf field for timestamp. */ + uint64_t timestamp_rx_flag; /* Dynamic mbuf flag for timestamp. */ uint64_t flow_meta_mask; int32_t flow_meta_offset; } __rte_cache_aligned; @@ -681,4 +683,21 @@ mlx5_txpp_convert_tx_ts(struct mlx5_dev_ctx_shared *sh, uint64_t mts) return ci; } +/** + * Set timestamp in mbuf dynamic field. + * + * @param mbuf + * Structure to write into. + * @param offset + * Dynamic field offset in mbuf structure. + * @param timestamp + * Value to write. + */ +static __rte_always_inline void +mlx5_timestamp_set(struct rte_mbuf *mbuf, int offset, + rte_mbuf_timestamp_t timestamp) +{ + *RTE_MBUF_DYNFIELD(mbuf, offset, rte_mbuf_timestamp_t *) = timestamp; +} + #endif /* RTE_PMD_MLX5_RXTX_H_ */ diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h index 6bf0c9b540..171d7bb0f8 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h @@ -330,13 +330,13 @@ rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, vector unsigned char ol_flags = (vector unsigned char) (vector unsigned int){ rxq->rss_hash * PKT_RX_RSS_HASH | - rxq->hw_timestamp * PKT_RX_TIMESTAMP, + rxq->hw_timestamp * rxq->timestamp_rx_flag, rxq->rss_hash * PKT_RX_RSS_HASH | - rxq->hw_timestamp * PKT_RX_TIMESTAMP, + rxq->hw_timestamp * rxq->timestamp_rx_flag, rxq->rss_hash * PKT_RX_RSS_HASH | - rxq->hw_timestamp * PKT_RX_TIMESTAMP, + rxq->hw_timestamp * rxq->timestamp_rx_flag, rxq->rss_hash * PKT_RX_RSS_HASH | - rxq->hw_timestamp * PKT_RX_TIMESTAMP}; + rxq->hw_timestamp * rxq->timestamp_rx_flag}; vector unsigned char cv_flags; const vector unsigned char zero = (vector unsigned char){0}; const vector unsigned char ptype_mask = @@ -1025,31 +1025,32 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, /* D.5 fill in mbuf - rearm_data and packet_type. */ rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]); if (rxq->hw_timestamp) { + int offset = rxq->timestamp_offset; if (rxq->rt_timestamp) { struct mlx5_dev_ctx_shared *sh = rxq->sh; uint64_t ts; ts = rte_be_to_cpu_64(cq[pos].timestamp); - pkts[pos]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(pkts[pos], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); ts = rte_be_to_cpu_64(cq[pos + p1].timestamp); - pkts[pos + 1]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(pkts[pos + 1], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); ts = rte_be_to_cpu_64(cq[pos + p2].timestamp); - pkts[pos + 2]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(pkts[pos + 2], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); ts = rte_be_to_cpu_64(cq[pos + p3].timestamp); - pkts[pos + 3]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(pkts[pos + 3], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); } else { - pkts[pos]->timestamp = rte_be_to_cpu_64 - (cq[pos].timestamp); - pkts[pos + 1]->timestamp = rte_be_to_cpu_64 - (cq[pos + p1].timestamp); - pkts[pos + 2]->timestamp = rte_be_to_cpu_64 - (cq[pos + p2].timestamp); - pkts[pos + 3]->timestamp = rte_be_to_cpu_64 - (cq[pos + p3].timestamp); + mlx5_timestamp_set(pkts[pos], offset, + rte_be_to_cpu_64(cq[pos].timestamp)); + mlx5_timestamp_set(pkts[pos + 1], offset, + rte_be_to_cpu_64(cq[pos + p1].timestamp)); + mlx5_timestamp_set(pkts[pos + 2], offset, + rte_be_to_cpu_64(cq[pos + p2].timestamp)); + mlx5_timestamp_set(pkts[pos + 3], offset, + rte_be_to_cpu_64(cq[pos + p3].timestamp)); } } if (rxq->dynf_meta) { diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h index d122dad4fe..436b247ade 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h @@ -271,7 +271,7 @@ rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, uint32x4_t pinfo, cv_flags; uint32x4_t ol_flags = vdupq_n_u32(rxq->rss_hash * PKT_RX_RSS_HASH | - rxq->hw_timestamp * PKT_RX_TIMESTAMP); + rxq->hw_timestamp * rxq->timestamp_rx_flag); const uint32x4_t ptype_ol_mask = { 0x106, 0x106, 0x106, 0x106 }; const uint8x16_t cv_flag_sel = { 0, @@ -697,6 +697,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, rxq_cq_to_ptype_oflags_v(rxq, ptype_info, flow_tag, opcode, &elts[pos]); if (rxq->hw_timestamp) { + int offset = rxq->timestamp_offset; if (rxq->rt_timestamp) { struct mlx5_dev_ctx_shared *sh = rxq->sh; uint64_t ts; @@ -704,36 +705,36 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, ts = rte_be_to_cpu_64 (container_of(p0, struct mlx5_cqe, pkt_info)->timestamp); - elts[pos]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(elts[pos], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); ts = rte_be_to_cpu_64 (container_of(p1, struct mlx5_cqe, pkt_info)->timestamp); - elts[pos + 1]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(elts[pos + 1], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); ts = rte_be_to_cpu_64 (container_of(p2, struct mlx5_cqe, pkt_info)->timestamp); - elts[pos + 2]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(elts[pos + 2], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); ts = rte_be_to_cpu_64 (container_of(p3, struct mlx5_cqe, pkt_info)->timestamp); - elts[pos + 3]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(elts[pos + 3], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); } else { - elts[pos]->timestamp = rte_be_to_cpu_64 - (container_of(p0, struct mlx5_cqe, - pkt_info)->timestamp); - elts[pos + 1]->timestamp = rte_be_to_cpu_64 - (container_of(p1, struct mlx5_cqe, - pkt_info)->timestamp); - elts[pos + 2]->timestamp = rte_be_to_cpu_64 - (container_of(p2, struct mlx5_cqe, - pkt_info)->timestamp); - elts[pos + 3]->timestamp = rte_be_to_cpu_64 - (container_of(p3, struct mlx5_cqe, - pkt_info)->timestamp); + mlx5_timestamp_set(elts[pos], offset, + rte_be_to_cpu_64(container_of(p0, + struct mlx5_cqe, pkt_info)->timestamp)); + mlx5_timestamp_set(elts[pos + 1], offset, + rte_be_to_cpu_64(container_of(p1, + struct mlx5_cqe, pkt_info)->timestamp)); + mlx5_timestamp_set(elts[pos + 2], offset, + rte_be_to_cpu_64(container_of(p2, + struct mlx5_cqe, pkt_info)->timestamp)); + mlx5_timestamp_set(elts[pos + 3], offset, + rte_be_to_cpu_64(container_of(p3, + struct mlx5_cqe, pkt_info)->timestamp)); } } if (rxq->dynf_meta) { diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h index 0bbcbeefff..ae4439efc7 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h +++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h @@ -251,7 +251,7 @@ rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, __m128i cqes[4], __m128i pinfo0, pinfo1; __m128i pinfo, ptype; __m128i ol_flags = _mm_set1_epi32(rxq->rss_hash * PKT_RX_RSS_HASH | - rxq->hw_timestamp * PKT_RX_TIMESTAMP); + rxq->hw_timestamp * rxq->timestamp_rx_flag); __m128i cv_flags; const __m128i zero = _mm_setzero_si128(); const __m128i ptype_mask = @@ -656,31 +656,32 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, /* D.5 fill in mbuf - rearm_data and packet_type. */ rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]); if (rxq->hw_timestamp) { + int offset = rxq->timestamp_offset; if (rxq->rt_timestamp) { struct mlx5_dev_ctx_shared *sh = rxq->sh; uint64_t ts; ts = rte_be_to_cpu_64(cq[pos].timestamp); - pkts[pos]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(pkts[pos], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); ts = rte_be_to_cpu_64(cq[pos + p1].timestamp); - pkts[pos + 1]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(pkts[pos + 1], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); ts = rte_be_to_cpu_64(cq[pos + p2].timestamp); - pkts[pos + 2]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(pkts[pos + 2], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); ts = rte_be_to_cpu_64(cq[pos + p3].timestamp); - pkts[pos + 3]->timestamp = - mlx5_txpp_convert_rx_ts(sh, ts); + mlx5_timestamp_set(pkts[pos + 3], offset, + mlx5_txpp_convert_rx_ts(sh, ts)); } else { - pkts[pos]->timestamp = rte_be_to_cpu_64 - (cq[pos].timestamp); - pkts[pos + 1]->timestamp = rte_be_to_cpu_64 - (cq[pos + p1].timestamp); - pkts[pos + 2]->timestamp = rte_be_to_cpu_64 - (cq[pos + p2].timestamp); - pkts[pos + 3]->timestamp = rte_be_to_cpu_64 - (cq[pos + p3].timestamp); + mlx5_timestamp_set(pkts[pos], offset, + rte_be_to_cpu_64(cq[pos].timestamp)); + mlx5_timestamp_set(pkts[pos + 1], offset, + rte_be_to_cpu_64(cq[pos + p1].timestamp)); + mlx5_timestamp_set(pkts[pos + 2], offset, + rte_be_to_cpu_64(cq[pos + p2].timestamp)); + mlx5_timestamp_set(pkts[pos + 3], offset, + rte_be_to_cpu_64(cq[pos + p3].timestamp)); } } if (rxq->dynf_meta) { -- 2.20.1