1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
6 #ifndef RTE_PMD_MLX5_RXTX_VEC_ALTIVEC_H_
7 #define RTE_PMD_MLX5_RXTX_VEC_ALTIVEC_H_
13 #include <rte_altivec.h>
16 #include <rte_mempool.h>
17 #include <rte_prefetch.h>
21 #include "mlx5_defs.h"
23 #include "mlx5_utils.h"
24 #include "mlx5_rxtx.h"
25 #include "mlx5_rxtx_vec.h"
26 #include "mlx5_autoconf.h"
28 #ifndef __INTEL_COMPILER
29 #pragma GCC diagnostic ignored "-Wcast-qual"
30 #pragma GCC diagnostic ignored "-Wstrict-aliasing"
34 * Store free buffers to RX SW ring.
37 * Pointer to SW ring to be filled.
39 * Pointer to array of packets to be stored.
41 * Number of packets to be stored.
44 rxq_copy_mbuf_v(struct rte_mbuf **elts, struct rte_mbuf **pkts, uint16_t n)
49 for (pos = 0; pos < p; pos += 2) {
50 vector unsigned char mbp;
52 mbp = (vector unsigned char)vec_vsx_ld(0,
53 (signed int const *)&elts[pos]);
54 *(vector unsigned char *)&pkts[pos] = mbp;
57 pkts[pos] = elts[pos];
61 * Decompress a compressed completion and fill in mbufs in RX SW ring with data
62 * extracted from the title completion descriptor.
65 * Pointer to RX queue structure.
67 * Pointer to completion array having a compressed completion at first.
69 * Pointer to SW ring to be filled. The first mbuf has to be pre-built from
70 * the title completion descriptor to be copied to the rest of mbufs.
73 * Number of mini-CQEs successfully decompressed.
75 static inline uint16_t
76 rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
77 struct rte_mbuf **elts)
79 volatile struct mlx5_mini_cqe8 *mcq = (void *)&(cq + 1)->pkt_info;
80 struct rte_mbuf *t_pkt = elts[0]; /* Title packet is pre-built. */
81 const vector unsigned char zero = (vector unsigned char){0};
82 /* Mask to shuffle from extracted mini CQE to mbuf. */
83 const vector unsigned char shuf_mask1 = (vector unsigned char){
84 -1, -1, -1, -1, /* skip packet_type */
85 7, 6, -1, -1, /* bswap16, pkt_len */
86 7, 6, /* bswap16, data_len */
87 -1, -1, /* skip vlan_tci */
88 3, 2, 1, 0}; /* bswap32, rss */
89 const vector unsigned char shuf_mask2 = (vector unsigned char){
90 -1, -1, -1, -1, /* skip packet_type */
91 15, 14, -1, -1, /* bswap16, pkt_len */
92 15, 14, /* data_len, bswap16 */
93 -1, -1, /* skip vlan_tci */
94 11, 10, 9, 8}; /* bswap32, rss */
95 /* Restore the compressed count. Must be 16 bits. */
96 const uint16_t mcqe_n = t_pkt->data_len +
97 (rxq->crc_present * RTE_ETHER_CRC_LEN);
98 const vector unsigned char rearm =
99 (vector unsigned char)vec_vsx_ld(0,
100 (signed int const *)&t_pkt->rearm_data);
101 const vector unsigned char rxdf =
102 (vector unsigned char)vec_vsx_ld(0,
103 (signed int const *)&t_pkt->rx_descriptor_fields1);
104 const vector unsigned char crc_adj =
105 (vector unsigned char)(vector unsigned short){
106 0, 0, rxq->crc_present * RTE_ETHER_CRC_LEN, 0,
107 rxq->crc_present * RTE_ETHER_CRC_LEN, 0, 0, 0};
108 const vector unsigned short rxdf_sel_mask =
109 (vector unsigned short){
110 0xffff, 0xffff, 0, 0, 0, 0xffff, 0, 0};
111 const uint32_t flow_tag = t_pkt->hash.fdir.hi;
114 unsigned int inv = 0;
116 #ifdef MLX5_PMD_SOFT_COUNTERS
117 const vector unsigned char ones = vec_splat_u8(-1);
118 uint32_t rcvd_byte = 0;
119 /* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
120 const vector unsigned char len_shuf_mask = (vector unsigned char){
128 * A. load mCQEs into a 128bit register.
129 * B. store rearm data to mbuf.
130 * C. combine data from mCQEs with rx_descriptor_fields1.
131 * D. store rx_descriptor_fields1.
132 * E. store flow tag (rte_flow mark).
134 for (pos = 0; pos < mcqe_n; ) {
135 vector unsigned char mcqe1, mcqe2;
136 vector unsigned char rxdf1, rxdf2;
137 #ifdef MLX5_PMD_SOFT_COUNTERS
138 const vector unsigned short mcqe_sel_mask =
139 (vector unsigned short){0, 0, 0xffff, 0xffff,
140 0, 0, 0xfff, 0xffff};
141 const vector unsigned char lower_half = {
142 0, 1, 4, 5, 8, 9, 12, 13, 16,
143 17, 20, 21, 24, 25, 28, 29};
144 const vector unsigned char upper_half = {
145 2, 3, 6, 7, 10, 11, 14, 15,
146 18, 19, 22, 23, 26, 27, 30, 31};
147 vector unsigned short left, right;
148 vector unsigned char byte_cnt, invalid_mask;
149 vector unsigned long lshift;
150 __attribute__((altivec(vector__)))
151 __attribute__((altivec(bool__)))
152 unsigned long long shmask;
153 const vector unsigned long shmax = {64, 64};
156 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i)
157 if (likely(pos + i < mcqe_n))
158 rte_prefetch0((void *)(cq + pos + i));
159 /* A.1 load mCQEs into a 128bit register. */
160 mcqe1 = (vector unsigned char)vec_vsx_ld(0,
161 (signed int const *)&mcq[pos % 8]);
162 mcqe2 = (vector unsigned char)vec_vsx_ld(0,
163 (signed int const *)&mcq[pos % 8 + 2]);
165 /* B.1 store rearm data to mbuf. */
166 *(vector unsigned char *)
167 &elts[pos]->rearm_data = rearm;
168 *(vector unsigned char *)
169 &elts[pos + 1]->rearm_data = rearm;
171 /* C.1 combine data from mCQEs with rx_descriptor_fields1. */
172 rxdf1 = vec_perm(mcqe1, zero, shuf_mask1);
173 rxdf2 = vec_perm(mcqe1, zero, shuf_mask2);
174 rxdf1 = (vector unsigned char)
175 ((vector unsigned short)rxdf1 -
176 (vector unsigned short)crc_adj);
177 rxdf2 = (vector unsigned char)
178 ((vector unsigned short)rxdf2 -
179 (vector unsigned short)crc_adj);
180 rxdf1 = (vector unsigned char)
181 vec_sel((vector unsigned short)rxdf1,
182 (vector unsigned short)rxdf, rxdf_sel_mask);
183 rxdf2 = (vector unsigned char)
184 vec_sel((vector unsigned short)rxdf2,
185 (vector unsigned short)rxdf, rxdf_sel_mask);
187 /* D.1 store rx_descriptor_fields1. */
188 *(vector unsigned char *)
189 &elts[pos]->rx_descriptor_fields1 = rxdf1;
190 *(vector unsigned char *)
191 &elts[pos + 1]->rx_descriptor_fields1 = rxdf2;
193 /* B.1 store rearm data to mbuf. */
194 *(vector unsigned char *)
195 &elts[pos + 2]->rearm_data = rearm;
196 *(vector unsigned char *)
197 &elts[pos + 3]->rearm_data = rearm;
199 /* C.1 combine data from mCQEs with rx_descriptor_fields1. */
200 rxdf1 = vec_perm(mcqe2, zero, shuf_mask1);
201 rxdf2 = vec_perm(mcqe2, zero, shuf_mask2);
202 rxdf1 = (vector unsigned char)
203 ((vector unsigned short)rxdf1 -
204 (vector unsigned short)crc_adj);
205 rxdf2 = (vector unsigned char)
206 ((vector unsigned short)rxdf2 -
207 (vector unsigned short)crc_adj);
208 rxdf1 = (vector unsigned char)
209 vec_sel((vector unsigned short)rxdf1,
210 (vector unsigned short)rxdf, rxdf_sel_mask);
211 rxdf2 = (vector unsigned char)
212 vec_sel((vector unsigned short)rxdf2,
213 (vector unsigned short)rxdf, rxdf_sel_mask);
215 /* D.1 store rx_descriptor_fields1. */
216 *(vector unsigned char *)
217 &elts[pos + 2]->rx_descriptor_fields1 = rxdf1;
218 *(vector unsigned char *)
219 &elts[pos + 3]->rx_descriptor_fields1 = rxdf2;
221 #ifdef MLX5_PMD_SOFT_COUNTERS
222 invalid_mask = (vector unsigned char)(vector unsigned long){
223 (mcqe_n - pos) * sizeof(uint16_t) * 8, 0};
226 vec_splat((vector unsigned long)invalid_mask, 0);
227 shmask = vec_cmpgt(shmax, lshift);
228 invalid_mask = (vector unsigned char)
229 vec_sl((vector unsigned long)ones, lshift);
230 invalid_mask = (vector unsigned char)
231 vec_sel((vector unsigned long)shmask,
232 (vector unsigned long)invalid_mask, shmask);
234 mcqe1 = (vector unsigned char)
235 vec_sro((vector unsigned short)mcqe1,
236 (vector unsigned char){32}),
237 byte_cnt = (vector unsigned char)
238 vec_sel((vector unsigned short)mcqe1,
239 (vector unsigned short)mcqe2, mcqe_sel_mask);
240 byte_cnt = vec_perm(byte_cnt, zero, len_shuf_mask);
241 byte_cnt = (vector unsigned char)
242 vec_andc((vector unsigned long)byte_cnt,
243 (vector unsigned long)invalid_mask);
244 left = vec_perm((vector unsigned short)byte_cnt,
245 (vector unsigned short)zero, lower_half);
246 right = vec_perm((vector unsigned short)byte_cnt,
247 (vector unsigned short)zero, upper_half);
248 byte_cnt = (vector unsigned char)vec_add(left, right);
249 left = vec_perm((vector unsigned short)byte_cnt,
250 (vector unsigned short)zero, lower_half);
251 right = vec_perm((vector unsigned short)byte_cnt,
252 (vector unsigned short)zero, upper_half);
253 byte_cnt = (vector unsigned char)vec_add(left, right);
254 rcvd_byte += ((vector unsigned long)byte_cnt)[0];
258 /* E.1 store flow tag (rte_flow mark). */
259 elts[pos]->hash.fdir.hi = flow_tag;
260 elts[pos + 1]->hash.fdir.hi = flow_tag;
261 elts[pos + 2]->hash.fdir.hi = flow_tag;
262 elts[pos + 3]->hash.fdir.hi = flow_tag;
264 if (rxq->dynf_meta) {
265 int32_t offs = rxq->flow_meta_offset;
266 const uint32_t meta =
267 *RTE_MBUF_DYNFIELD(t_pkt, offs, uint32_t *);
269 /* Check if title packet has valid metadata. */
271 MLX5_ASSERT(t_pkt->ol_flags &
272 rxq->flow_meta_mask);
273 *RTE_MBUF_DYNFIELD(elts[pos], offs,
275 *RTE_MBUF_DYNFIELD(elts[pos + 1], offs,
277 *RTE_MBUF_DYNFIELD(elts[pos + 2], offs,
279 *RTE_MBUF_DYNFIELD(elts[pos + 3], offs,
284 pos += MLX5_VPMD_DESCS_PER_LOOP;
285 /* Move to next CQE and invalidate consumed CQEs. */
286 if (!(pos & 0x7) && pos < mcqe_n) {
287 if (pos + 8 < mcqe_n)
288 rte_prefetch0((void *)(cq + pos + 8));
289 mcq = (void *)&(cq + pos)->pkt_info;
290 for (i = 0; i < 8; ++i)
291 cq[inv++].op_own = MLX5_CQE_INVALIDATE;
295 /* Invalidate the rest of CQEs. */
296 for (; inv < mcqe_n; ++inv)
297 cq[inv].op_own = MLX5_CQE_INVALIDATE;
299 #ifdef MLX5_PMD_SOFT_COUNTERS
300 rxq->stats.ipackets += mcqe_n;
301 rxq->stats.ibytes += rcvd_byte;
304 rxq->cq_ci += mcqe_n;
309 * Calculate packet type and offload flag for mbuf and store it.
312 * Pointer to RX queue structure.
314 * Array of four 16bytes completions extracted from the original completion
317 * Opcode vector having responder error status. Each field is 4B.
319 * Pointer to array of packets to be filled.
322 rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq,
323 vector unsigned char cqes[4], vector unsigned char op_err,
324 struct rte_mbuf **pkts)
326 vector unsigned char pinfo0, pinfo1;
327 vector unsigned char pinfo, ptype;
328 vector unsigned char ol_flags = (vector unsigned char)
329 (vector unsigned int){
330 rxq->rss_hash * PKT_RX_RSS_HASH |
331 rxq->hw_timestamp * rxq->timestamp_rx_flag,
332 rxq->rss_hash * PKT_RX_RSS_HASH |
333 rxq->hw_timestamp * rxq->timestamp_rx_flag,
334 rxq->rss_hash * PKT_RX_RSS_HASH |
335 rxq->hw_timestamp * rxq->timestamp_rx_flag,
336 rxq->rss_hash * PKT_RX_RSS_HASH |
337 rxq->hw_timestamp * rxq->timestamp_rx_flag};
338 vector unsigned char cv_flags;
339 const vector unsigned char zero = (vector unsigned char){0};
340 const vector unsigned char ptype_mask =
341 (vector unsigned char)(vector unsigned int){
342 0x0000fd06, 0x0000fd06, 0x0000fd06, 0x0000fd06};
343 const vector unsigned char ptype_ol_mask =
344 (vector unsigned char)(vector unsigned int){
345 0x00000106, 0x00000106, 0x00000106, 0x00000106};
346 const vector unsigned char pinfo_mask =
347 (vector unsigned char)(vector unsigned int){
348 0x00000003, 0x00000003, 0x00000003, 0x00000003};
349 const vector unsigned char cv_flag_sel = (vector unsigned char){
350 0, (uint8_t)(PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED),
351 (uint8_t)(PKT_RX_IP_CKSUM_GOOD >> 1), 0,
352 (uint8_t)(PKT_RX_L4_CKSUM_GOOD >> 1), 0,
353 (uint8_t)((PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1),
354 0, 0, 0, 0, 0, 0, 0, 0, 0};
355 const vector unsigned char cv_mask =
356 (vector unsigned char)(vector unsigned int){
357 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
358 PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
359 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
360 PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
361 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
362 PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
363 PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD |
364 PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED};
365 const vector unsigned char mbuf_init =
366 (vector unsigned char)vec_vsx_ld
367 (0, (vector unsigned char *)&rxq->mbuf_initializer);
368 const vector unsigned short rearm_sel_mask =
369 (vector unsigned short){0, 0, 0, 0, 0xffff, 0xffff, 0, 0};
370 vector unsigned char rearm0, rearm1, rearm2, rearm3;
371 uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3;
373 /* Extract pkt_info field. */
374 pinfo0 = (vector unsigned char)
375 vec_mergeh((vector unsigned int)cqes[0],
376 (vector unsigned int)cqes[1]);
377 pinfo1 = (vector unsigned char)
378 vec_mergeh((vector unsigned int)cqes[2],
379 (vector unsigned int)cqes[3]);
380 pinfo = (vector unsigned char)
381 vec_mergeh((vector unsigned long)pinfo0,
382 (vector unsigned long)pinfo1);
384 /* Extract hdr_type_etc field. */
385 pinfo0 = (vector unsigned char)
386 vec_mergel((vector unsigned int)cqes[0],
387 (vector unsigned int)cqes[1]);
388 pinfo1 = (vector unsigned char)
389 vec_mergel((vector unsigned int)cqes[2],
390 (vector unsigned int)cqes[3]);
391 ptype = (vector unsigned char)
392 vec_mergeh((vector unsigned long)pinfo0,
393 (vector unsigned long)pinfo1);
396 const vector unsigned char pinfo_ft_mask =
397 (vector unsigned char)(vector unsigned int){
398 0xffffff00, 0xffffff00, 0xffffff00, 0xffffff00};
399 const vector unsigned char fdir_flags =
400 (vector unsigned char)(vector unsigned int){
401 PKT_RX_FDIR, PKT_RX_FDIR,
402 PKT_RX_FDIR, PKT_RX_FDIR};
403 vector unsigned char fdir_id_flags =
404 (vector unsigned char)(vector unsigned int){
405 PKT_RX_FDIR_ID, PKT_RX_FDIR_ID,
406 PKT_RX_FDIR_ID, PKT_RX_FDIR_ID};
407 vector unsigned char flow_tag, invalid_mask;
409 flow_tag = (vector unsigned char)
410 vec_and((vector unsigned long)pinfo,
411 (vector unsigned long)pinfo_ft_mask);
413 /* Check if flow tag is non-zero then set PKT_RX_FDIR. */
414 invalid_mask = (vector unsigned char)
415 vec_cmpeq((vector unsigned int)flow_tag,
416 (vector unsigned int)zero);
417 ol_flags = (vector unsigned char)
418 vec_or((vector unsigned long)ol_flags,
419 (vector unsigned long)
420 vec_andc((vector unsigned long)fdir_flags,
421 (vector unsigned long)invalid_mask));
423 /* Mask out invalid entries. */
424 fdir_id_flags = (vector unsigned char)
425 vec_andc((vector unsigned long)fdir_id_flags,
426 (vector unsigned long)invalid_mask);
428 /* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */
429 ol_flags = (vector unsigned char)
430 vec_or((vector unsigned long)ol_flags,
431 (vector unsigned long)
432 vec_andc((vector unsigned long)fdir_id_flags,
433 (vector unsigned long)
434 vec_cmpeq((vector unsigned int)flow_tag,
435 (vector unsigned int)pinfo_ft_mask)));
438 * Merge the two fields to generate the following:
442 * bit[11:10] = l3_hdr_type
443 * bit[14:12] = l4_hdr_type
446 * bit[17] = outer_l3_type
448 ptype = (vector unsigned char)
449 vec_and((vector unsigned long)ptype,
450 (vector unsigned long)ptype_mask);
451 pinfo = (vector unsigned char)
452 vec_and((vector unsigned long)pinfo,
453 (vector unsigned long)pinfo_mask);
454 pinfo = (vector unsigned char)
455 vec_sl((vector unsigned int)pinfo,
456 (vector unsigned int){16, 16, 16, 16});
458 /* Make pinfo has merged fields for ol_flags calculation. */
459 pinfo = (vector unsigned char)
460 vec_or((vector unsigned long)ptype,
461 (vector unsigned long)pinfo);
462 ptype = (vector unsigned char)
463 vec_sr((vector unsigned int)pinfo,
464 (vector unsigned int){10, 10, 10, 10});
465 ptype = (vector unsigned char)
466 vec_packs((vector unsigned int)ptype,
467 (vector unsigned int)zero);
469 /* Errored packets will have RTE_PTYPE_ALL_MASK. */
470 op_err = (vector unsigned char)
471 vec_sr((vector unsigned short)op_err,
472 (vector unsigned short){8, 8, 8, 8, 8, 8, 8, 8});
473 ptype = (vector unsigned char)
474 vec_or((vector unsigned long)ptype,
475 (vector unsigned long)op_err);
477 pt_idx0 = (uint8_t)((vector unsigned char)ptype)[0];
478 pt_idx1 = (uint8_t)((vector unsigned char)ptype)[2];
479 pt_idx2 = (uint8_t)((vector unsigned char)ptype)[4];
480 pt_idx3 = (uint8_t)((vector unsigned char)ptype)[6];
482 pkts[0]->packet_type = mlx5_ptype_table[pt_idx0] |
483 !!(pt_idx0 & (1 << 6)) * rxq->tunnel;
484 pkts[1]->packet_type = mlx5_ptype_table[pt_idx1] |
485 !!(pt_idx1 & (1 << 6)) * rxq->tunnel;
486 pkts[2]->packet_type = mlx5_ptype_table[pt_idx2] |
487 !!(pt_idx2 & (1 << 6)) * rxq->tunnel;
488 pkts[3]->packet_type = mlx5_ptype_table[pt_idx3] |
489 !!(pt_idx3 & (1 << 6)) * rxq->tunnel;
491 /* Fill flags for checksum and VLAN. */
492 pinfo = (vector unsigned char)
493 vec_and((vector unsigned long)pinfo,
494 (vector unsigned long)ptype_ol_mask);
495 pinfo = vec_perm(cv_flag_sel, zero, pinfo);
497 /* Locate checksum flags at byte[2:1] and merge with VLAN flags. */
498 cv_flags = (vector unsigned char)
499 vec_sl((vector unsigned int)pinfo,
500 (vector unsigned int){9, 9, 9, 9});
501 cv_flags = (vector unsigned char)
502 vec_or((vector unsigned long)pinfo,
503 (vector unsigned long)cv_flags);
505 /* Move back flags to start from byte[0]. */
506 cv_flags = (vector unsigned char)
507 vec_sr((vector unsigned int)cv_flags,
508 (vector unsigned int){8, 8, 8, 8});
510 /* Mask out garbage bits. */
511 cv_flags = (vector unsigned char)
512 vec_and((vector unsigned long)cv_flags,
513 (vector unsigned long)cv_mask);
515 /* Merge to ol_flags. */
516 ol_flags = (vector unsigned char)
517 vec_or((vector unsigned long)ol_flags,
518 (vector unsigned long)cv_flags);
520 /* Merge mbuf_init and ol_flags. */
521 rearm0 = (vector unsigned char)
522 vec_sel((vector unsigned short)mbuf_init,
523 (vector unsigned short)
524 vec_slo((vector unsigned short)ol_flags,
525 (vector unsigned char){64}), rearm_sel_mask);
526 rearm1 = (vector unsigned char)
527 vec_sel((vector unsigned short)mbuf_init,
528 (vector unsigned short)
529 vec_slo((vector unsigned short)ol_flags,
530 (vector unsigned char){32}), rearm_sel_mask);
531 rearm2 = (vector unsigned char)
532 vec_sel((vector unsigned short)mbuf_init,
533 (vector unsigned short)ol_flags, rearm_sel_mask);
534 rearm3 = (vector unsigned char)
535 vec_sel((vector unsigned short)mbuf_init,
536 (vector unsigned short)
537 vec_sro((vector unsigned short)ol_flags,
538 (vector unsigned char){32}), rearm_sel_mask);
540 /* Write 8B rearm_data and 8B ol_flags. */
541 vec_vsx_st(rearm0, 0,
542 (vector unsigned char *)&pkts[0]->rearm_data);
543 vec_vsx_st(rearm1, 0,
544 (vector unsigned char *)&pkts[1]->rearm_data);
545 vec_vsx_st(rearm2, 0,
546 (vector unsigned char *)&pkts[2]->rearm_data);
547 vec_vsx_st(rearm3, 0,
548 (vector unsigned char *)&pkts[3]->rearm_data);
552 * Process a non-compressed completion and fill in mbufs in RX SW ring
553 * with data extracted from the title completion descriptor.
556 * Pointer to RX queue structure.
558 * Pointer to completion array having a non-compressed completion at first.
560 * Pointer to SW ring to be filled. The first mbuf has to be pre-built from
561 * the title completion descriptor to be copied to the rest of mbufs.
563 * Array to store received packets.
565 * Maximum number of packets in array.
567 * Pointer to a flag. Set non-zero value if pkts array has at least one error
570 * Pointer to a index. Set it to the first compressed completion if any.
573 * Number of CQEs successfully processed.
575 static inline uint16_t
576 rxq_cq_process_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
577 struct rte_mbuf **elts, struct rte_mbuf **pkts,
578 uint16_t pkts_n, uint64_t *err, uint64_t *comp)
580 const uint16_t q_n = 1 << rxq->cqe_n;
581 const uint16_t q_mask = q_n - 1;
584 uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
585 uint16_t nocmp_n = 0;
586 unsigned int ownership = !!(rxq->cq_ci & (q_mask + 1));
587 const vector unsigned char zero = (vector unsigned char){0};
588 const vector unsigned char ones = vec_splat_u8(-1);
589 const vector unsigned char owner_check =
590 (vector unsigned char)(vector unsigned long){
591 0x0100000001000000LL, 0x0100000001000000LL};
592 const vector unsigned char opcode_check =
593 (vector unsigned char)(vector unsigned long){
594 0xf0000000f0000000LL, 0xf0000000f0000000LL};
595 const vector unsigned char format_check =
596 (vector unsigned char)(vector unsigned long){
597 0x0c0000000c000000LL, 0x0c0000000c000000LL};
598 const vector unsigned char resp_err_check =
599 (vector unsigned char)(vector unsigned long){
600 0xe0000000e0000000LL, 0xe0000000e0000000LL};
601 #ifdef MLX5_PMD_SOFT_COUNTERS
602 uint32_t rcvd_byte = 0;
603 /* Mask to shuffle byte_cnt to add up stats. Do bswap16 for all. */
604 const vector unsigned char len_shuf_mask = (vector unsigned char){
610 /* Mask to shuffle from extracted CQE to mbuf. */
611 const vector unsigned char shuf_mask = (vector unsigned char){
612 5, 4, /* bswap16, pkt_len */
613 -1, -1, /* zero out 2nd half of pkt_len */
614 5, 4, /* bswap16, data_len */
615 11, 10, /* bswap16, vlan+tci */
616 15, 14, 13, 12, /* bswap32, rss */
617 1, 2, 3, -1}; /* fdir.hi */
618 /* Mask to blend from the last Qword to the first DQword. */
619 /* Mask to blend from the last Qword to the first DQword. */
620 const vector unsigned char blend_mask = (vector unsigned char){
625 const vector unsigned char crc_adj =
626 (vector unsigned char)(vector unsigned short){
627 rxq->crc_present * RTE_ETHER_CRC_LEN, 0,
628 rxq->crc_present * RTE_ETHER_CRC_LEN, 0, 0, 0, 0, 0};
629 const vector unsigned char flow_mark_adj =
630 (vector unsigned char)(vector unsigned int){
631 0, 0, 0, rxq->mark * (-1)};
632 const vector unsigned short cqe_sel_mask1 =
633 (vector unsigned short){0, 0, 0, 0, 0xffff, 0xffff, 0, 0};
634 const vector unsigned short cqe_sel_mask2 =
635 (vector unsigned short){0, 0, 0xffff, 0, 0, 0, 0, 0};
638 * A. load first Qword (8bytes) in one loop.
639 * B. copy 4 mbuf pointers from elts ring to returing pkts.
640 * C. load remaining CQE data and extract necessary fields.
641 * Final 16bytes cqes[] extracted from original 64bytes CQE has the
642 * following structure:
645 * uint8_t flow_tag[3];
649 * uint16_t hdr_type_etc;
650 * uint16_t vlan_info;
651 * uint32_t rx_has_res;
655 * F. find compressed CQE.
659 pos += MLX5_VPMD_DESCS_PER_LOOP) {
660 vector unsigned char cqes[MLX5_VPMD_DESCS_PER_LOOP];
661 vector unsigned char cqe_tmp1, cqe_tmp2;
662 vector unsigned char pkt_mb0, pkt_mb1, pkt_mb2, pkt_mb3;
663 vector unsigned char op_own, op_own_tmp1, op_own_tmp2;
664 vector unsigned char opcode, owner_mask, invalid_mask;
665 vector unsigned char comp_mask;
666 vector unsigned char mask;
667 #ifdef MLX5_PMD_SOFT_COUNTERS
668 const vector unsigned char lower_half = {
669 0, 1, 4, 5, 8, 9, 12, 13,
670 16, 17, 20, 21, 24, 25, 28, 29};
671 const vector unsigned char upper_half = {
672 2, 3, 6, 7, 10, 11, 14, 15,
673 18, 19, 22, 23, 26, 27, 30, 31};
674 const vector unsigned long shmax = {64, 64};
675 vector unsigned char byte_cnt;
676 vector unsigned short left, right;
677 vector unsigned long lshift;
678 vector __attribute__((altivec(bool__)))
679 unsigned long shmask;
681 vector unsigned char mbp1, mbp2;
682 vector unsigned char p =
683 (vector unsigned char)(vector unsigned short){
684 0, 1, 2, 3, 0, 0, 0, 0};
685 unsigned int p1, p2, p3;
687 /* Prefetch next 4 CQEs. */
688 if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) {
689 rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP]);
690 rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 1]);
691 rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 2]);
692 rte_prefetch0(&cq[pos + MLX5_VPMD_DESCS_PER_LOOP + 3]);
695 /* A.0 do not cross the end of CQ. */
696 mask = (vector unsigned char)(vector unsigned long){
697 (pkts_n - pos) * sizeof(uint16_t) * 8, 0};
700 vector unsigned long lshift;
701 vector __attribute__((altivec(bool__)))
702 unsigned long shmask;
703 const vector unsigned long shmax = {64, 64};
705 lshift = vec_splat((vector unsigned long)mask, 0);
706 shmask = vec_cmpgt(shmax, lshift);
707 mask = (vector unsigned char)
708 vec_sl((vector unsigned long)ones, lshift);
709 mask = (vector unsigned char)
710 vec_sel((vector unsigned long)shmask,
711 (vector unsigned long)mask, shmask);
714 p = (vector unsigned char)
715 vec_andc((vector unsigned long)p,
716 (vector unsigned long)mask);
719 p3 = (unsigned int)((vector unsigned short)p)[3];
720 cqes[3] = (vector unsigned char)(vector unsigned long){
721 *(__rte_aligned(8) unsigned long *)
722 &cq[pos + p3].sop_drop_qpn, 0LL};
723 rte_compiler_barrier();
725 p2 = (unsigned int)((vector unsigned short)p)[2];
726 cqes[2] = (vector unsigned char)(vector unsigned long){
727 *(__rte_aligned(8) unsigned long *)
728 &cq[pos + p2].sop_drop_qpn, 0LL};
729 rte_compiler_barrier();
731 /* B.1 load mbuf pointers. */
732 mbp1 = (vector unsigned char)vec_vsx_ld(0,
733 (signed int const *)&elts[pos]);
734 mbp2 = (vector unsigned char)vec_vsx_ld(0,
735 (signed int const *)&elts[pos + 2]);
737 /* A.1 load a block having op_own. */
738 p1 = (unsigned int)((vector unsigned short)p)[1];
739 cqes[1] = (vector unsigned char)(vector unsigned long){
740 *(__rte_aligned(8) unsigned long *)
741 &cq[pos + p1].sop_drop_qpn, 0LL};
742 rte_compiler_barrier();
744 cqes[0] = (vector unsigned char)(vector unsigned long){
745 *(__rte_aligned(8) unsigned long *)
746 &cq[pos].sop_drop_qpn, 0LL};
747 rte_compiler_barrier();
749 /* B.2 copy mbuf pointers. */
750 *(vector unsigned char *)&pkts[pos] = mbp1;
751 *(vector unsigned char *)&pkts[pos + 2] = mbp2;
754 /* C.1 load remaining CQE data and extract necessary fields. */
755 cqe_tmp2 = *(vector unsigned char *)
756 &cq[pos + p3].pkt_info;
757 cqe_tmp1 = *(vector unsigned char *)
758 &cq[pos + p2].pkt_info;
759 cqes[3] = vec_sel(cqes[3], cqe_tmp2, blend_mask);
760 cqes[2] = vec_sel(cqes[2], cqe_tmp1, blend_mask);
761 cqe_tmp2 = (vector unsigned char)vec_vsx_ld(0,
762 (signed int const *)&cq[pos + p3].csum);
763 cqe_tmp1 = (vector unsigned char)vec_vsx_ld(0,
764 (signed int const *)&cq[pos + p2].csum);
765 cqes[3] = (vector unsigned char)
766 vec_sel((vector unsigned short)cqes[3],
767 (vector unsigned short)cqe_tmp2, cqe_sel_mask1);
768 cqes[2] = (vector unsigned char)
769 vec_sel((vector unsigned short)cqes[2],
770 (vector unsigned short)cqe_tmp1, cqe_sel_mask1);
771 cqe_tmp2 = (vector unsigned char)(vector unsigned long){
772 *(__rte_aligned(8) unsigned long *)
773 &cq[pos + p3].rsvd3[9], 0LL};
774 cqe_tmp1 = (vector unsigned char)(vector unsigned long){
775 *(__rte_aligned(8) unsigned long *)
776 &cq[pos + p2].rsvd3[9], 0LL};
777 cqes[3] = (vector unsigned char)
778 vec_sel((vector unsigned short)cqes[3],
779 (vector unsigned short)cqe_tmp2,
780 (vector unsigned short)cqe_sel_mask2);
781 cqes[2] = (vector unsigned char)
782 vec_sel((vector unsigned short)cqes[2],
783 (vector unsigned short)cqe_tmp1,
784 (vector unsigned short)cqe_sel_mask2);
786 /* C.2 generate final structure for mbuf with swapping bytes. */
787 pkt_mb3 = vec_perm(cqes[3], zero, shuf_mask);
788 pkt_mb2 = vec_perm(cqes[2], zero, shuf_mask);
790 /* C.3 adjust CRC length. */
791 pkt_mb3 = (vector unsigned char)
792 ((vector unsigned short)pkt_mb3 -
793 (vector unsigned short)crc_adj);
794 pkt_mb2 = (vector unsigned char)
795 ((vector unsigned short)pkt_mb2 -
796 (vector unsigned short)crc_adj);
798 /* C.4 adjust flow mark. */
799 pkt_mb3 = (vector unsigned char)
800 ((vector unsigned int)pkt_mb3 +
801 (vector unsigned int)flow_mark_adj);
802 pkt_mb2 = (vector unsigned char)
803 ((vector unsigned int)pkt_mb2 +
804 (vector unsigned int)flow_mark_adj);
806 /* D.1 fill in mbuf - rx_descriptor_fields1. */
807 *(vector unsigned char *)
808 &pkts[pos + 3]->pkt_len = pkt_mb3;
809 *(vector unsigned char *)
810 &pkts[pos + 2]->pkt_len = pkt_mb2;
812 /* E.1 extract op_own field. */
813 op_own_tmp2 = (vector unsigned char)
814 vec_mergeh((vector unsigned int)cqes[2],
815 (vector unsigned int)cqes[3]);
817 /* C.1 load remaining CQE data and extract necessary fields. */
818 cqe_tmp2 = *(vector unsigned char *)
819 &cq[pos + p1].pkt_info;
820 cqe_tmp1 = *(vector unsigned char *)
822 cqes[1] = vec_sel(cqes[1], cqe_tmp2, blend_mask);
823 cqes[0] = vec_sel(cqes[0], cqe_tmp2, blend_mask);
824 cqe_tmp2 = (vector unsigned char)vec_vsx_ld(0,
825 (signed int const *)&cq[pos + p1].csum);
826 cqe_tmp1 = (vector unsigned char)vec_vsx_ld(0,
827 (signed int const *)&cq[pos].csum);
828 cqes[1] = (vector unsigned char)
829 vec_sel((vector unsigned short)cqes[1],
830 (vector unsigned short)cqe_tmp2, cqe_sel_mask1);
831 cqes[0] = (vector unsigned char)
832 vec_sel((vector unsigned short)cqes[0],
833 (vector unsigned short)cqe_tmp1, cqe_sel_mask1);
834 cqe_tmp2 = (vector unsigned char)(vector unsigned long){
835 *(__rte_aligned(8) unsigned long *)
836 &cq[pos + p1].rsvd3[9], 0LL};
837 cqe_tmp1 = (vector unsigned char)(vector unsigned long){
838 *(__rte_aligned(8) unsigned long *)
839 &cq[pos].rsvd3[9], 0LL};
840 cqes[1] = (vector unsigned char)
841 vec_sel((vector unsigned short)cqes[1],
842 (vector unsigned short)cqe_tmp2, cqe_sel_mask2);
843 cqes[0] = (vector unsigned char)
844 vec_sel((vector unsigned short)cqes[0],
845 (vector unsigned short)cqe_tmp1, cqe_sel_mask2);
847 /* C.2 generate final structure for mbuf with swapping bytes. */
848 pkt_mb1 = vec_perm(cqes[1], zero, shuf_mask);
849 pkt_mb0 = vec_perm(cqes[0], zero, shuf_mask);
851 /* C.3 adjust CRC length. */
852 pkt_mb1 = (vector unsigned char)
853 ((vector unsigned short)pkt_mb1 -
854 (vector unsigned short)crc_adj);
855 pkt_mb0 = (vector unsigned char)
856 ((vector unsigned short)pkt_mb0 -
857 (vector unsigned short)crc_adj);
859 /* C.4 adjust flow mark. */
860 pkt_mb1 = (vector unsigned char)
861 ((vector unsigned int)pkt_mb1 +
862 (vector unsigned int)flow_mark_adj);
863 pkt_mb0 = (vector unsigned char)
864 ((vector unsigned int)pkt_mb0 +
865 (vector unsigned int)flow_mark_adj);
867 /* E.1 extract op_own byte. */
868 op_own_tmp1 = (vector unsigned char)
869 vec_mergeh((vector unsigned int)cqes[0],
870 (vector unsigned int)cqes[1]);
871 op_own = (vector unsigned char)
872 vec_mergel((vector unsigned long)op_own_tmp1,
873 (vector unsigned long)op_own_tmp2);
875 /* D.1 fill in mbuf - rx_descriptor_fields1. */
876 *(vector unsigned char *)
877 &pkts[pos + 1]->pkt_len = pkt_mb1;
878 *(vector unsigned char *)
879 &pkts[pos]->pkt_len = pkt_mb0;
881 /* E.2 flip owner bit to mark CQEs from last round. */
882 owner_mask = (vector unsigned char)
883 vec_and((vector unsigned long)op_own,
884 (vector unsigned long)owner_check);
886 owner_mask = (vector unsigned char)
887 vec_xor((vector unsigned long)owner_mask,
888 (vector unsigned long)owner_check);
889 owner_mask = (vector unsigned char)
890 vec_cmpeq((vector unsigned int)owner_mask,
891 (vector unsigned int)owner_check);
892 owner_mask = (vector unsigned char)
893 vec_packs((vector unsigned int)owner_mask,
894 (vector unsigned int)zero);
896 /* E.3 get mask for invalidated CQEs. */
897 opcode = (vector unsigned char)
898 vec_and((vector unsigned long)op_own,
899 (vector unsigned long)opcode_check);
900 invalid_mask = (vector unsigned char)
901 vec_cmpeq((vector unsigned int)opcode_check,
902 (vector unsigned int)opcode);
903 invalid_mask = (vector unsigned char)
904 vec_packs((vector unsigned int)invalid_mask,
905 (vector unsigned int)zero);
907 /* E.4 mask out beyond boundary. */
908 invalid_mask = (vector unsigned char)
909 vec_or((vector unsigned long)invalid_mask,
910 (vector unsigned long)mask);
912 /* E.5 merge invalid_mask with invalid owner. */
913 invalid_mask = (vector unsigned char)
914 vec_or((vector unsigned long)invalid_mask,
915 (vector unsigned long)owner_mask);
917 /* F.1 find compressed CQE format. */
918 comp_mask = (vector unsigned char)
919 vec_and((vector unsigned long)op_own,
920 (vector unsigned long)format_check);
921 comp_mask = (vector unsigned char)
922 vec_cmpeq((vector unsigned int)comp_mask,
923 (vector unsigned int)format_check);
924 comp_mask = (vector unsigned char)
925 vec_packs((vector unsigned int)comp_mask,
926 (vector unsigned int)zero);
928 /* F.2 mask out invalid entries. */
929 comp_mask = (vector unsigned char)
930 vec_andc((vector unsigned long)comp_mask,
931 (vector unsigned long)invalid_mask);
932 comp_idx = ((vector unsigned long)comp_mask)[0];
934 /* F.3 get the first compressed CQE. */
935 comp_idx = comp_idx ? __builtin_ctzll(comp_idx) /
936 (sizeof(uint16_t) * 8) : MLX5_VPMD_DESCS_PER_LOOP;
938 /* E.6 mask out entries after the compressed CQE. */
939 mask = (vector unsigned char)(vector unsigned long){
940 (comp_idx * sizeof(uint16_t) * 8), 0};
941 lshift = vec_splat((vector unsigned long)mask, 0);
942 shmask = vec_cmpgt(shmax, lshift);
943 mask = (vector unsigned char)
944 vec_sl((vector unsigned long)ones, lshift);
945 mask = (vector unsigned char)
946 vec_sel((vector unsigned long)shmask,
947 (vector unsigned long)mask, shmask);
948 invalid_mask = (vector unsigned char)
949 vec_or((vector unsigned long)invalid_mask,
950 (vector unsigned long)mask);
952 /* E.7 count non-compressed valid CQEs. */
953 n = ((vector unsigned long)invalid_mask)[0];
954 n = n ? __builtin_ctzll(n) / (sizeof(uint16_t) * 8) :
955 MLX5_VPMD_DESCS_PER_LOOP;
958 /* D.2 get the final invalid mask. */
959 mask = (vector unsigned char)(vector unsigned long){
960 (n * sizeof(uint16_t) * 8), 0};
961 lshift = vec_splat((vector unsigned long)mask, 0);
962 shmask = vec_cmpgt(shmax, lshift);
963 mask = (vector unsigned char)
964 vec_sl((vector unsigned long)ones, lshift);
965 mask = (vector unsigned char)
966 vec_sel((vector unsigned long)shmask,
967 (vector unsigned long)mask, shmask);
968 invalid_mask = (vector unsigned char)
969 vec_or((vector unsigned long)invalid_mask,
970 (vector unsigned long)mask);
972 /* D.3 check error in opcode. */
973 opcode = (vector unsigned char)
974 vec_cmpeq((vector unsigned int)resp_err_check,
975 (vector unsigned int)opcode);
976 opcode = (vector unsigned char)
977 vec_packs((vector unsigned int)opcode,
978 (vector unsigned int)zero);
979 opcode = (vector unsigned char)
980 vec_andc((vector unsigned long)opcode,
981 (vector unsigned long)invalid_mask);
983 /* D.4 mark if any error is set */
984 *err |= ((vector unsigned long)opcode)[0];
986 /* D.5 fill in mbuf - rearm_data and packet_type. */
987 rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
988 if (rxq->hw_timestamp) {
989 int offset = rxq->timestamp_offset;
990 if (rxq->rt_timestamp) {
991 struct mlx5_dev_ctx_shared *sh = rxq->sh;
994 ts = rte_be_to_cpu_64(cq[pos].timestamp);
995 mlx5_timestamp_set(pkts[pos], offset,
996 mlx5_txpp_convert_rx_ts(sh, ts));
997 ts = rte_be_to_cpu_64(cq[pos + p1].timestamp);
998 mlx5_timestamp_set(pkts[pos + 1], offset,
999 mlx5_txpp_convert_rx_ts(sh, ts));
1000 ts = rte_be_to_cpu_64(cq[pos + p2].timestamp);
1001 mlx5_timestamp_set(pkts[pos + 2], offset,
1002 mlx5_txpp_convert_rx_ts(sh, ts));
1003 ts = rte_be_to_cpu_64(cq[pos + p3].timestamp);
1004 mlx5_timestamp_set(pkts[pos + 3], offset,
1005 mlx5_txpp_convert_rx_ts(sh, ts));
1007 mlx5_timestamp_set(pkts[pos], offset,
1008 rte_be_to_cpu_64(cq[pos].timestamp));
1009 mlx5_timestamp_set(pkts[pos + 1], offset,
1010 rte_be_to_cpu_64(cq[pos + p1].timestamp));
1011 mlx5_timestamp_set(pkts[pos + 2], offset,
1012 rte_be_to_cpu_64(cq[pos + p2].timestamp));
1013 mlx5_timestamp_set(pkts[pos + 3], offset,
1014 rte_be_to_cpu_64(cq[pos + p3].timestamp));
1017 if (rxq->dynf_meta) {
1018 uint64_t flag = rxq->flow_meta_mask;
1019 int32_t offs = rxq->flow_meta_offset;
1022 /* This code is subject for futher optimization. */
1023 metadata = cq[pos].flow_table_metadata;
1024 *RTE_MBUF_DYNFIELD(pkts[pos], offs, uint32_t *) =
1026 pkts[pos]->ol_flags |= metadata ? flag : 0ULL;
1027 metadata = cq[pos + 1].flow_table_metadata;
1028 *RTE_MBUF_DYNFIELD(pkts[pos + 1], offs, uint32_t *) =
1030 pkts[pos + 1]->ol_flags |= metadata ? flag : 0ULL;
1031 metadata = cq[pos + 2].flow_table_metadata;
1032 *RTE_MBUF_DYNFIELD(pkts[pos + 2], offs, uint32_t *) =
1034 pkts[pos + 2]->ol_flags |= metadata ? flag : 0ULL;
1035 metadata = cq[pos + 3].flow_table_metadata;
1036 *RTE_MBUF_DYNFIELD(pkts[pos + 3], offs, uint32_t *) =
1038 pkts[pos + 3]->ol_flags |= metadata ? flag : 0ULL;
1040 #ifdef MLX5_PMD_SOFT_COUNTERS
1041 /* Add up received bytes count. */
1042 byte_cnt = vec_perm(op_own, zero, len_shuf_mask);
1043 byte_cnt = (vector unsigned char)
1044 vec_andc((vector unsigned long)byte_cnt,
1045 (vector unsigned long)invalid_mask);
1046 left = vec_perm((vector unsigned short)byte_cnt,
1047 (vector unsigned short)zero, lower_half);
1048 right = vec_perm((vector unsigned short)byte_cnt,
1049 (vector unsigned short)zero, upper_half);
1050 byte_cnt = (vector unsigned char)vec_add(left, right);
1051 left = vec_perm((vector unsigned short)byte_cnt,
1052 (vector unsigned short)zero, lower_half);
1053 right = vec_perm((vector unsigned short)byte_cnt,
1054 (vector unsigned short)zero, upper_half);
1055 byte_cnt = (vector unsigned char)vec_add(left, right);
1056 rcvd_byte += ((vector unsigned long)byte_cnt)[0];
1060 * Break the loop unless more valid CQE is expected, or if
1061 * there's a compressed CQE.
1063 if (n != MLX5_VPMD_DESCS_PER_LOOP)
1066 #ifdef MLX5_PMD_SOFT_COUNTERS
1067 rxq->stats.ipackets += nocmp_n;
1068 rxq->stats.ibytes += rcvd_byte;
1075 #endif /* RTE_PMD_MLX5_RXTX_VEC_ALTIVEC_H_ */