1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Hisilicon Limited.
5 #ifndef _HNS3_RXTX_VEC_NEON_H_
6 #define _HNS3_RXTX_VEC_NEON_H_
10 #pragma GCC diagnostic ignored "-Wcast-qual"
13 hns3_vec_tx(volatile struct hns3_desc *desc, struct rte_mbuf *pkt)
16 pkt->buf_iova + pkt->data_off,
17 ((uint64_t)pkt->data_len) << HNS3_TXD_SEND_SIZE_SHIFT
21 ((uint64_t)HNS3_TXD_DEFAULT_VLD_FE_BDTYPE) << HNS3_UINT32_BIT
23 vst1q_u64((uint64_t *)&desc->addr, val1);
24 vst1q_u64((uint64_t *)&desc->tx.outer_vlan_tag, val2);
28 hns3_xmit_fixed_burst_vec(void *__restrict tx_queue,
29 struct rte_mbuf **__restrict tx_pkts,
32 struct hns3_tx_queue *txq = (struct hns3_tx_queue *)tx_queue;
33 volatile struct hns3_desc *tx_desc;
34 struct hns3_entry *tx_entry;
40 if (txq->tx_bd_ready < txq->tx_free_thresh)
41 hns3_tx_free_buffers(txq);
43 nb_commit = RTE_MIN(txq->tx_bd_ready, nb_pkts);
44 if (unlikely(nb_commit == 0)) {
45 txq->queue_full_cnt++;
50 next_to_use = txq->next_to_use;
51 tx_desc = &txq->tx_ring[next_to_use];
52 tx_entry = &txq->sw_ring[next_to_use];
55 * We need to deal with n descriptors first for better performance,
56 * if nb_commit is greater than the difference between txq->nb_tx_desc
57 * and next_to_use in sw_ring and tx_ring.
59 n = txq->nb_tx_desc - next_to_use;
61 for (i = 0; i < n; i++, tx_pkts++, tx_desc++) {
62 hns3_vec_tx(tx_desc, *tx_pkts);
63 tx_entry[i].mbuf = *tx_pkts;
68 tx_desc = &txq->tx_ring[next_to_use];
69 tx_entry = &txq->sw_ring[next_to_use];
72 for (i = 0; i < nb_commit; i++, tx_pkts++, tx_desc++) {
73 hns3_vec_tx(tx_desc, *tx_pkts);
74 tx_entry[i].mbuf = *tx_pkts;
77 next_to_use += nb_commit;
78 txq->next_to_use = next_to_use;
79 txq->tx_bd_ready -= nb_tx;
81 hns3_write_reg_opt(txq->io_tail_reg, nb_tx);
86 static inline uint32_t
87 hns3_desc_parse_field(struct hns3_rx_queue *rxq,
88 struct hns3_entry *sw_ring,
89 struct hns3_desc *rxdp,
92 uint32_t l234_info, ol_info, bd_base_info;
98 for (i = 0; i < (int)bd_vld_num; i++) {
99 pkt = sw_ring[i].mbuf;
101 /* init rte_mbuf.rearm_data last 64-bit */
102 pkt->ol_flags = PKT_RX_RSS_HASH;
104 l234_info = rxdp[i].rx.l234_info;
105 ol_info = rxdp[i].rx.ol_info;
106 bd_base_info = rxdp[i].rx.bd_base_info;
107 ret = hns3_handle_bdinfo(rxq, pkt, bd_base_info,
108 l234_info, &cksum_err);
114 pkt->packet_type = hns3_rx_calc_ptype(rxq, l234_info, ol_info);
115 if (likely(bd_base_info & BIT(HNS3_RXD_L3L4P_B)))
116 hns3_rx_set_cksum_flag(pkt, pkt->packet_type,
123 static inline uint16_t
124 hns3_recv_burst_vec(struct hns3_rx_queue *__restrict rxq,
125 struct rte_mbuf **__restrict rx_pkts,
127 uint64_t *bd_err_mask)
129 uint16_t rx_id = rxq->next_to_use;
130 struct hns3_entry *sw_ring = &rxq->sw_ring[rx_id];
131 struct hns3_desc *rxdp = &rxq->rx_ring[rx_id];
132 uint32_t bd_valid_num, parse_retcode;
136 /* mask to shuffle from desc to mbuf's rx_descriptor_fields1 */
137 uint8x16_t shuf_desc_fields_msk = {
138 0xff, 0xff, 0xff, 0xff, /* packet type init zero */
139 22, 23, 0xff, 0xff, /* rx.pkt_len to rte_mbuf.pkt_len */
140 20, 21, /* size to rte_mbuf.data_len */
141 0xff, 0xff, /* rte_mbuf.vlan_tci init zero */
142 8, 9, 10, 11, /* rx.rss_hash to rte_mbuf.hash.rss */
145 uint16x8_t crc_adjust = {
146 0, 0, /* ignore pkt_type field */
147 rxq->crc_len, /* sub crc on pkt_len */
148 0, /* ignore high-16bits of pkt_len */
149 rxq->crc_len, /* sub crc on data_len */
150 0, 0, 0, /* ignore non-length fields */
153 for (pos = 0; pos < nb_pkts; pos += HNS3_DEFAULT_DESCS_PER_LOOP,
154 rxdp += HNS3_DEFAULT_DESCS_PER_LOOP) {
155 uint64x2x2_t descs[HNS3_DEFAULT_DESCS_PER_LOOP];
156 uint8x16x2_t pkt_mbuf1, pkt_mbuf2, pkt_mbuf3, pkt_mbuf4;
157 uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
158 uint64x2_t mbp1, mbp2;
159 uint16x4_t bd_vld = {0};
163 /* calc how many bd valid */
164 bd_vld = vset_lane_u16(rxdp[0].rx.bdtype_vld_udp0, bd_vld, 0);
165 bd_vld = vset_lane_u16(rxdp[1].rx.bdtype_vld_udp0, bd_vld, 1);
166 bd_vld = vset_lane_u16(rxdp[2].rx.bdtype_vld_udp0, bd_vld, 2);
167 bd_vld = vset_lane_u16(rxdp[3].rx.bdtype_vld_udp0, bd_vld, 3);
169 /* load 2 mbuf pointer */
170 mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]);
172 bd_vld = vshl_n_u16(bd_vld,
173 HNS3_UINT16_BIT - 1 - HNS3_RXD_VLD_B);
174 bd_vld = vreinterpret_u16_s16(
175 vshr_n_s16(vreinterpret_s16_u16(bd_vld),
176 HNS3_UINT16_BIT - 1));
177 stat = ~vget_lane_u64(vreinterpret_u64_u16(bd_vld), 0);
179 /* load 2 mbuf pointer again */
180 mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]);
182 if (likely(stat == 0))
183 bd_valid_num = HNS3_DEFAULT_DESCS_PER_LOOP;
185 bd_valid_num = __builtin_ctzl(stat) / HNS3_UINT16_BIT;
186 if (bd_valid_num == 0)
189 /* use offset to control below data load oper ordering */
190 offset = rxq->offset_table[bd_valid_num];
192 /* store 2 mbuf pointer into rx_pkts */
193 vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1);
195 /* read first two descs */
196 descs[0] = vld2q_u64((uint64_t *)(rxdp + offset));
197 descs[1] = vld2q_u64((uint64_t *)(rxdp + offset + 1));
199 /* store 2 mbuf pointer into rx_pkts again */
200 vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2);
202 /* read remains two descs */
203 descs[2] = vld2q_u64((uint64_t *)(rxdp + offset + 2));
204 descs[3] = vld2q_u64((uint64_t *)(rxdp + offset + 3));
206 pkt_mbuf1.val[0] = vreinterpretq_u8_u64(descs[0].val[0]);
207 pkt_mbuf1.val[1] = vreinterpretq_u8_u64(descs[0].val[1]);
208 pkt_mbuf2.val[0] = vreinterpretq_u8_u64(descs[1].val[0]);
209 pkt_mbuf2.val[1] = vreinterpretq_u8_u64(descs[1].val[1]);
211 /* pkt 1,2 convert format from desc to pktmbuf */
212 pkt_mb1 = vqtbl2q_u8(pkt_mbuf1, shuf_desc_fields_msk);
213 pkt_mb2 = vqtbl2q_u8(pkt_mbuf2, shuf_desc_fields_msk);
215 /* store the first 8 bytes of pkt 1,2 mbuf's rearm_data */
216 *(uint64_t *)&sw_ring[pos + 0].mbuf->rearm_data =
217 rxq->mbuf_initializer;
218 *(uint64_t *)&sw_ring[pos + 1].mbuf->rearm_data =
219 rxq->mbuf_initializer;
221 /* pkt 1,2 remove crc */
222 tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust);
223 pkt_mb1 = vreinterpretq_u8_u16(tmp);
224 tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust);
225 pkt_mb2 = vreinterpretq_u8_u16(tmp);
227 pkt_mbuf3.val[0] = vreinterpretq_u8_u64(descs[2].val[0]);
228 pkt_mbuf3.val[1] = vreinterpretq_u8_u64(descs[2].val[1]);
229 pkt_mbuf4.val[0] = vreinterpretq_u8_u64(descs[3].val[0]);
230 pkt_mbuf4.val[1] = vreinterpretq_u8_u64(descs[3].val[1]);
232 /* pkt 3,4 convert format from desc to pktmbuf */
233 pkt_mb3 = vqtbl2q_u8(pkt_mbuf3, shuf_desc_fields_msk);
234 pkt_mb4 = vqtbl2q_u8(pkt_mbuf4, shuf_desc_fields_msk);
236 /* pkt 1,2 save to rx_pkts mbuf */
237 vst1q_u8((void *)&sw_ring[pos + 0].mbuf->rx_descriptor_fields1,
239 vst1q_u8((void *)&sw_ring[pos + 1].mbuf->rx_descriptor_fields1,
242 /* pkt 3,4 remove crc */
243 tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust);
244 pkt_mb3 = vreinterpretq_u8_u16(tmp);
245 tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust);
246 pkt_mb4 = vreinterpretq_u8_u16(tmp);
248 /* store the first 8 bytes of pkt 3,4 mbuf's rearm_data */
249 *(uint64_t *)&sw_ring[pos + 2].mbuf->rearm_data =
250 rxq->mbuf_initializer;
251 *(uint64_t *)&sw_ring[pos + 3].mbuf->rearm_data =
252 rxq->mbuf_initializer;
254 /* pkt 3,4 save to rx_pkts mbuf */
255 vst1q_u8((void *)&sw_ring[pos + 2].mbuf->rx_descriptor_fields1,
257 vst1q_u8((void *)&sw_ring[pos + 3].mbuf->rx_descriptor_fields1,
260 rte_prefetch_non_temporal(rxdp + HNS3_DEFAULT_DESCS_PER_LOOP);
262 parse_retcode = hns3_desc_parse_field(rxq, &sw_ring[pos],
263 &rxdp[offset], bd_valid_num);
264 if (unlikely(parse_retcode))
265 (*bd_err_mask) |= ((uint64_t)parse_retcode) << pos;
267 rte_prefetch0(sw_ring[pos +
268 HNS3_DEFAULT_DESCS_PER_LOOP + 0].mbuf);
269 rte_prefetch0(sw_ring[pos +
270 HNS3_DEFAULT_DESCS_PER_LOOP + 1].mbuf);
271 rte_prefetch0(sw_ring[pos +
272 HNS3_DEFAULT_DESCS_PER_LOOP + 2].mbuf);
273 rte_prefetch0(sw_ring[pos +
274 HNS3_DEFAULT_DESCS_PER_LOOP + 3].mbuf);
276 nb_rx += bd_valid_num;
277 if (bd_valid_num < HNS3_DEFAULT_DESCS_PER_LOOP)
281 rxq->rx_rearm_nb += nb_rx;
282 rxq->next_to_use += nb_rx;
283 if (rxq->next_to_use >= rxq->nb_rx_desc)
284 rxq->next_to_use = 0;
288 #endif /* _HNS3_RXTX_VEC_NEON_H_ */