net/i40e: fix Rx packet statistics
[dpdk.git] / drivers / net / ice / ice_rxtx_vec_common.h
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2019 Intel Corporation
3  */
4
5 #ifndef _ICE_RXTX_VEC_COMMON_H_
6 #define _ICE_RXTX_VEC_COMMON_H_
7
8 #include "ice_rxtx.h"
9
10 #ifndef __INTEL_COMPILER
11 #pragma GCC diagnostic ignored "-Wcast-qual"
12 #endif
13
14 static inline uint16_t
15 ice_rx_reassemble_packets(struct ice_rx_queue *rxq, struct rte_mbuf **rx_bufs,
16                           uint16_t nb_bufs, uint8_t *split_flags)
17 {
18         struct rte_mbuf *pkts[ICE_VPMD_RX_BURST] = {0}; /*finished pkts*/
19         struct rte_mbuf *start = rxq->pkt_first_seg;
20         struct rte_mbuf *end =  rxq->pkt_last_seg;
21         unsigned int pkt_idx, buf_idx;
22
23         for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
24                 if (end) {
25                         /* processing a split packet */
26                         end->next = rx_bufs[buf_idx];
27                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
28
29                         start->nb_segs++;
30                         start->pkt_len += rx_bufs[buf_idx]->data_len;
31                         end = end->next;
32
33                         if (!split_flags[buf_idx]) {
34                                 /* it's the last packet of the set */
35                                 start->hash = end->hash;
36                                 start->vlan_tci = end->vlan_tci;
37                                 start->ol_flags = end->ol_flags;
38                                 /* we need to strip crc for the whole packet */
39                                 start->pkt_len -= rxq->crc_len;
40                                 if (end->data_len > rxq->crc_len) {
41                                         end->data_len -= rxq->crc_len;
42                                 } else {
43                                         /* free up last mbuf */
44                                         struct rte_mbuf *secondlast = start;
45
46                                         start->nb_segs--;
47                                         while (secondlast->next != end)
48                                                 secondlast = secondlast->next;
49                                         secondlast->data_len -= (rxq->crc_len -
50                                                         end->data_len);
51                                         secondlast->next = NULL;
52                                         rte_pktmbuf_free_seg(end);
53                                 }
54                                 pkts[pkt_idx++] = start;
55                                 start = NULL;
56                                 end = NULL;
57                         }
58                 } else {
59                         /* not processing a split packet */
60                         if (!split_flags[buf_idx]) {
61                                 /* not a split packet, save and skip */
62                                 pkts[pkt_idx++] = rx_bufs[buf_idx];
63                                 continue;
64                         }
65                         start = rx_bufs[buf_idx];
66                         end = start;
67                         rx_bufs[buf_idx]->data_len += rxq->crc_len;
68                         rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
69                 }
70         }
71
72         /* save the partial packet for next time */
73         rxq->pkt_first_seg = start;
74         rxq->pkt_last_seg = end;
75         rte_memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
76         return pkt_idx;
77 }
78
79 static __rte_always_inline int
80 ice_tx_free_bufs_vec(struct ice_tx_queue *txq)
81 {
82         struct ice_tx_entry *txep;
83         uint32_t n;
84         uint32_t i;
85         int nb_free = 0;
86         struct rte_mbuf *m, *free[ICE_TX_MAX_FREE_BUF_SZ];
87
88         /* check DD bits on threshold descriptor */
89         if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
90                         rte_cpu_to_le_64(ICE_TXD_QW1_DTYPE_M)) !=
91                         rte_cpu_to_le_64(ICE_TX_DESC_DTYPE_DESC_DONE))
92                 return 0;
93
94         n = txq->tx_rs_thresh;
95
96          /* first buffer to free from S/W ring is at index
97           * tx_next_dd - (tx_rs_thresh-1)
98           */
99         txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
100         m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
101         if (likely(m)) {
102                 free[0] = m;
103                 nb_free = 1;
104                 for (i = 1; i < n; i++) {
105                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
106                         if (likely(m)) {
107                                 if (likely(m->pool == free[0]->pool)) {
108                                         free[nb_free++] = m;
109                                 } else {
110                                         rte_mempool_put_bulk(free[0]->pool,
111                                                              (void *)free,
112                                                              nb_free);
113                                         free[0] = m;
114                                         nb_free = 1;
115                                 }
116                         }
117                 }
118                 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
119         } else {
120                 for (i = 1; i < n; i++) {
121                         m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
122                         if (m)
123                                 rte_mempool_put(m->pool, m);
124                 }
125         }
126
127         /* buffers were freed, update counters */
128         txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
129         txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
130         if (txq->tx_next_dd >= txq->nb_tx_desc)
131                 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
132
133         return txq->tx_rs_thresh;
134 }
135
136 static __rte_always_inline void
137 ice_tx_backlog_entry(struct ice_tx_entry *txep,
138                      struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
139 {
140         int i;
141
142         for (i = 0; i < (int)nb_pkts; ++i)
143                 txep[i].mbuf = tx_pkts[i];
144 }
145
146 static inline void
147 _ice_rx_queue_release_mbufs_vec(struct ice_rx_queue *rxq)
148 {
149         const unsigned int mask = rxq->nb_rx_desc - 1;
150         unsigned int i;
151
152         if (unlikely(!rxq->sw_ring)) {
153                 PMD_DRV_LOG(DEBUG, "sw_ring is NULL");
154                 return;
155         }
156
157         if (rxq->rxrearm_nb >= rxq->nb_rx_desc)
158                 return;
159
160         /* free all mbufs that are valid in the ring */
161         if (rxq->rxrearm_nb == 0) {
162                 for (i = 0; i < rxq->nb_rx_desc; i++) {
163                         if (rxq->sw_ring[i].mbuf)
164                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
165                 }
166         } else {
167                 for (i = rxq->rx_tail;
168                      i != rxq->rxrearm_start;
169                      i = (i + 1) & mask) {
170                         if (rxq->sw_ring[i].mbuf)
171                                 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
172                 }
173         }
174
175         rxq->rxrearm_nb = rxq->nb_rx_desc;
176
177         /* set all entries to NULL */
178         memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
179 }
180
181 static inline void
182 _ice_tx_queue_release_mbufs_vec(struct ice_tx_queue *txq)
183 {
184         uint16_t i;
185
186         if (unlikely(!txq || !txq->sw_ring)) {
187                 PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
188                 return;
189         }
190
191         /**
192          *  vPMD tx will not set sw_ring's mbuf to NULL after free,
193          *  so need to free remains more carefully.
194          */
195         i = txq->tx_next_dd - txq->tx_rs_thresh + 1;
196
197 #ifdef CC_AVX512_SUPPORT
198         struct rte_eth_dev *dev = &rte_eth_devices[txq->vsi->adapter->pf.dev_data->port_id];
199
200         if (dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512 ||
201             dev->tx_pkt_burst == ice_xmit_pkts_vec_avx512_offload) {
202                 struct ice_vec_tx_entry *swr = (void *)txq->sw_ring;
203
204                 if (txq->tx_tail < i) {
205                         for (; i < txq->nb_tx_desc; i++) {
206                                 rte_pktmbuf_free_seg(swr[i].mbuf);
207                                 swr[i].mbuf = NULL;
208                         }
209                         i = 0;
210                 }
211                 for (; i < txq->tx_tail; i++) {
212                         rte_pktmbuf_free_seg(swr[i].mbuf);
213                         swr[i].mbuf = NULL;
214                 }
215         } else
216 #endif
217         {
218                 if (txq->tx_tail < i) {
219                         for (; i < txq->nb_tx_desc; i++) {
220                                 rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
221                                 txq->sw_ring[i].mbuf = NULL;
222                         }
223                         i = 0;
224                 }
225                 for (; i < txq->tx_tail; i++) {
226                         rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
227                         txq->sw_ring[i].mbuf = NULL;
228                 }
229         }
230 }
231
232 static inline int
233 ice_rxq_vec_setup_default(struct ice_rx_queue *rxq)
234 {
235         uintptr_t p;
236         struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
237
238         mb_def.nb_segs = 1;
239         mb_def.data_off = RTE_PKTMBUF_HEADROOM;
240         mb_def.port = rxq->port_id;
241         rte_mbuf_refcnt_set(&mb_def, 1);
242
243         /* prevent compiler reordering: rearm_data covers previous fields */
244         rte_compiler_barrier();
245         p = (uintptr_t)&mb_def.rearm_data;
246         rxq->mbuf_initializer = *(uint64_t *)p;
247         return 0;
248 }
249
250 #define ICE_TX_NO_VECTOR_FLAGS (                        \
251                 DEV_TX_OFFLOAD_MULTI_SEGS |             \
252                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |       \
253                 DEV_TX_OFFLOAD_TCP_TSO)
254
255 #define ICE_TX_VECTOR_OFFLOAD (                         \
256                 DEV_TX_OFFLOAD_VLAN_INSERT |            \
257                 DEV_TX_OFFLOAD_QINQ_INSERT |            \
258                 DEV_TX_OFFLOAD_IPV4_CKSUM |             \
259                 DEV_TX_OFFLOAD_SCTP_CKSUM |             \
260                 DEV_TX_OFFLOAD_UDP_CKSUM |              \
261                 DEV_TX_OFFLOAD_TCP_CKSUM)
262
263 #define ICE_RX_VECTOR_OFFLOAD (                         \
264                 DEV_RX_OFFLOAD_CHECKSUM |               \
265                 DEV_RX_OFFLOAD_SCTP_CKSUM |             \
266                 DEV_RX_OFFLOAD_VLAN |                   \
267                 DEV_RX_OFFLOAD_RSS_HASH)
268
269 #define ICE_VECTOR_PATH         0
270 #define ICE_VECTOR_OFFLOAD_PATH 1
271
272 static inline int
273 ice_rx_vec_queue_default(struct ice_rx_queue *rxq)
274 {
275         if (!rxq)
276                 return -1;
277
278         if (!rte_is_power_of_2(rxq->nb_rx_desc))
279                 return -1;
280
281         if (rxq->rx_free_thresh < ICE_VPMD_RX_BURST)
282                 return -1;
283
284         if (rxq->nb_rx_desc % rxq->rx_free_thresh)
285                 return -1;
286
287         if (rxq->proto_xtr != PROTO_XTR_NONE)
288                 return -1;
289
290         if (rxq->offloads & DEV_RX_OFFLOAD_TIMESTAMP)
291                 return -1;
292
293         if (rxq->offloads & ICE_RX_VECTOR_OFFLOAD)
294                 return ICE_VECTOR_OFFLOAD_PATH;
295
296         return ICE_VECTOR_PATH;
297 }
298
299 static inline int
300 ice_tx_vec_queue_default(struct ice_tx_queue *txq)
301 {
302         if (!txq)
303                 return -1;
304
305         if (txq->tx_rs_thresh < ICE_VPMD_TX_BURST ||
306             txq->tx_rs_thresh > ICE_TX_MAX_FREE_BUF_SZ)
307                 return -1;
308
309         if (txq->offloads & ICE_TX_NO_VECTOR_FLAGS)
310                 return -1;
311
312         if (txq->offloads & ICE_TX_VECTOR_OFFLOAD)
313                 return ICE_VECTOR_OFFLOAD_PATH;
314
315         return ICE_VECTOR_PATH;
316 }
317
318 static inline int
319 ice_rx_vec_dev_check_default(struct rte_eth_dev *dev)
320 {
321         int i;
322         struct ice_rx_queue *rxq;
323         int ret = 0;
324         int result = 0;
325
326         for (i = 0; i < dev->data->nb_rx_queues; i++) {
327                 rxq = dev->data->rx_queues[i];
328                 ret = (ice_rx_vec_queue_default(rxq));
329                 if (ret < 0)
330                         return -1;
331                 if (ret == ICE_VECTOR_OFFLOAD_PATH)
332                         result = ret;
333         }
334
335         return result;
336 }
337
338 static inline int
339 ice_tx_vec_dev_check_default(struct rte_eth_dev *dev)
340 {
341         int i;
342         struct ice_tx_queue *txq;
343         int ret = 0;
344         int result = 0;
345
346         for (i = 0; i < dev->data->nb_tx_queues; i++) {
347                 txq = dev->data->tx_queues[i];
348                 ret = ice_tx_vec_queue_default(txq);
349                 if (ret < 0)
350                         return -1;
351                 if (ret == ICE_VECTOR_OFFLOAD_PATH)
352                         result = ret;
353         }
354
355         return result;
356 }
357
358 #ifdef CC_AVX2_SUPPORT
359 static __rte_always_inline void
360 ice_rxq_rearm_common(struct ice_rx_queue *rxq, __rte_unused bool avx512)
361 {
362         int i;
363         uint16_t rx_id;
364         volatile union ice_rx_flex_desc *rxdp;
365         struct ice_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
366
367         rxdp = rxq->rx_ring + rxq->rxrearm_start;
368
369         /* Pull 'n' more MBUFs into the software ring */
370         if (rte_mempool_get_bulk(rxq->mp,
371                                  (void *)rxep,
372                                  ICE_RXQ_REARM_THRESH) < 0) {
373                 if (rxq->rxrearm_nb + ICE_RXQ_REARM_THRESH >=
374                     rxq->nb_rx_desc) {
375                         __m128i dma_addr0;
376
377                         dma_addr0 = _mm_setzero_si128();
378                         for (i = 0; i < ICE_DESCS_PER_LOOP; i++) {
379                                 rxep[i].mbuf = &rxq->fake_mbuf;
380                                 _mm_store_si128((__m128i *)&rxdp[i].read,
381                                                 dma_addr0);
382                         }
383                 }
384                 rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
385                         ICE_RXQ_REARM_THRESH;
386                 return;
387         }
388
389 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
390         struct rte_mbuf *mb0, *mb1;
391         __m128i dma_addr0, dma_addr1;
392         __m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
393                         RTE_PKTMBUF_HEADROOM);
394         /* Initialize the mbufs in vector, process 2 mbufs in one loop */
395         for (i = 0; i < ICE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
396                 __m128i vaddr0, vaddr1;
397
398                 mb0 = rxep[0].mbuf;
399                 mb1 = rxep[1].mbuf;
400
401                 /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
402                 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
403                                 offsetof(struct rte_mbuf, buf_addr) + 8);
404                 vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
405                 vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
406
407                 /* convert pa to dma_addr hdr/data */
408                 dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
409                 dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
410
411                 /* add headroom to pa values */
412                 dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
413                 dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);
414
415                 /* flush desc with pa dma_addr */
416                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr0);
417                 _mm_store_si128((__m128i *)&rxdp++->read, dma_addr1);
418         }
419 #else
420 #ifdef CC_AVX512_SUPPORT
421         if (avx512) {
422                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
423                 struct rte_mbuf *mb4, *mb5, *mb6, *mb7;
424                 __m512i dma_addr0_3, dma_addr4_7;
425                 __m512i hdr_room = _mm512_set1_epi64(RTE_PKTMBUF_HEADROOM);
426                 /* Initialize the mbufs in vector, process 8 mbufs in one loop */
427                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
428                                 i += 8, rxep += 8, rxdp += 8) {
429                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
430                         __m128i vaddr4, vaddr5, vaddr6, vaddr7;
431                         __m256i vaddr0_1, vaddr2_3;
432                         __m256i vaddr4_5, vaddr6_7;
433                         __m512i vaddr0_3, vaddr4_7;
434
435                         mb0 = rxep[0].mbuf;
436                         mb1 = rxep[1].mbuf;
437                         mb2 = rxep[2].mbuf;
438                         mb3 = rxep[3].mbuf;
439                         mb4 = rxep[4].mbuf;
440                         mb5 = rxep[5].mbuf;
441                         mb6 = rxep[6].mbuf;
442                         mb7 = rxep[7].mbuf;
443
444                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
445                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
446                                         offsetof(struct rte_mbuf, buf_addr) + 8);
447                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
448                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
449                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
450                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
451                         vaddr4 = _mm_loadu_si128((__m128i *)&mb4->buf_addr);
452                         vaddr5 = _mm_loadu_si128((__m128i *)&mb5->buf_addr);
453                         vaddr6 = _mm_loadu_si128((__m128i *)&mb6->buf_addr);
454                         vaddr7 = _mm_loadu_si128((__m128i *)&mb7->buf_addr);
455
456                         /**
457                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
458                          * into the high lanes. Similarly for 2 & 3, and so on.
459                          */
460                         vaddr0_1 =
461                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
462                                                         vaddr1, 1);
463                         vaddr2_3 =
464                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
465                                                         vaddr3, 1);
466                         vaddr4_5 =
467                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr4),
468                                                         vaddr5, 1);
469                         vaddr6_7 =
470                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr6),
471                                                         vaddr7, 1);
472                         vaddr0_3 =
473                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr0_1),
474                                                         vaddr2_3, 1);
475                         vaddr4_7 =
476                                 _mm512_inserti64x4(_mm512_castsi256_si512(vaddr4_5),
477                                                         vaddr6_7, 1);
478
479                         /* convert pa to dma_addr hdr/data */
480                         dma_addr0_3 = _mm512_unpackhi_epi64(vaddr0_3, vaddr0_3);
481                         dma_addr4_7 = _mm512_unpackhi_epi64(vaddr4_7, vaddr4_7);
482
483                         /* add headroom to pa values */
484                         dma_addr0_3 = _mm512_add_epi64(dma_addr0_3, hdr_room);
485                         dma_addr4_7 = _mm512_add_epi64(dma_addr4_7, hdr_room);
486
487                         /* flush desc with pa dma_addr */
488                         _mm512_store_si512((__m512i *)&rxdp->read, dma_addr0_3);
489                         _mm512_store_si512((__m512i *)&(rxdp + 4)->read, dma_addr4_7);
490                 }
491         } else
492 #endif
493         {
494                 struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
495                 __m256i dma_addr0_1, dma_addr2_3;
496                 __m256i hdr_room = _mm256_set1_epi64x(RTE_PKTMBUF_HEADROOM);
497                 /* Initialize the mbufs in vector, process 4 mbufs in one loop */
498                 for (i = 0; i < ICE_RXQ_REARM_THRESH;
499                                 i += 4, rxep += 4, rxdp += 4) {
500                         __m128i vaddr0, vaddr1, vaddr2, vaddr3;
501                         __m256i vaddr0_1, vaddr2_3;
502
503                         mb0 = rxep[0].mbuf;
504                         mb1 = rxep[1].mbuf;
505                         mb2 = rxep[2].mbuf;
506                         mb3 = rxep[3].mbuf;
507
508                         /* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
509                         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
510                                         offsetof(struct rte_mbuf, buf_addr) + 8);
511                         vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
512                         vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
513                         vaddr2 = _mm_loadu_si128((__m128i *)&mb2->buf_addr);
514                         vaddr3 = _mm_loadu_si128((__m128i *)&mb3->buf_addr);
515
516                         /**
517                          * merge 0 & 1, by casting 0 to 256-bit and inserting 1
518                          * into the high lanes. Similarly for 2 & 3
519                          */
520                         vaddr0_1 =
521                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr0),
522                                                         vaddr1, 1);
523                         vaddr2_3 =
524                                 _mm256_inserti128_si256(_mm256_castsi128_si256(vaddr2),
525                                                         vaddr3, 1);
526
527                         /* convert pa to dma_addr hdr/data */
528                         dma_addr0_1 = _mm256_unpackhi_epi64(vaddr0_1, vaddr0_1);
529                         dma_addr2_3 = _mm256_unpackhi_epi64(vaddr2_3, vaddr2_3);
530
531                         /* add headroom to pa values */
532                         dma_addr0_1 = _mm256_add_epi64(dma_addr0_1, hdr_room);
533                         dma_addr2_3 = _mm256_add_epi64(dma_addr2_3, hdr_room);
534
535                         /* flush desc with pa dma_addr */
536                         _mm256_store_si256((__m256i *)&rxdp->read, dma_addr0_1);
537                         _mm256_store_si256((__m256i *)&(rxdp + 2)->read, dma_addr2_3);
538                 }
539         }
540
541 #endif
542
543         rxq->rxrearm_start += ICE_RXQ_REARM_THRESH;
544         if (rxq->rxrearm_start >= rxq->nb_rx_desc)
545                 rxq->rxrearm_start = 0;
546
547         rxq->rxrearm_nb -= ICE_RXQ_REARM_THRESH;
548
549         rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
550                              (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
551
552         /* Update the tail pointer on the NIC */
553         ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
554 }
555 #endif
556
557 static inline void
558 ice_txd_enable_offload(struct rte_mbuf *tx_pkt,
559                        uint64_t *txd_hi)
560 {
561         uint64_t ol_flags = tx_pkt->ol_flags;
562         uint32_t td_cmd = 0;
563         uint32_t td_offset = 0;
564
565         /* Tx Checksum Offload */
566         /* SET MACLEN */
567         td_offset |= (tx_pkt->l2_len >> 1) <<
568                         ICE_TX_DESC_LEN_MACLEN_S;
569
570         /* Enable L3 checksum offload */
571         if (ol_flags & PKT_TX_IP_CKSUM) {
572                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM;
573                 td_offset |= (tx_pkt->l3_len >> 2) <<
574                         ICE_TX_DESC_LEN_IPLEN_S;
575         } else if (ol_flags & PKT_TX_IPV4) {
576                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV4;
577                 td_offset |= (tx_pkt->l3_len >> 2) <<
578                         ICE_TX_DESC_LEN_IPLEN_S;
579         } else if (ol_flags & PKT_TX_IPV6) {
580                 td_cmd |= ICE_TX_DESC_CMD_IIPT_IPV6;
581                 td_offset |= (tx_pkt->l3_len >> 2) <<
582                         ICE_TX_DESC_LEN_IPLEN_S;
583         }
584
585         /* Enable L4 checksum offloads */
586         switch (ol_flags & PKT_TX_L4_MASK) {
587         case PKT_TX_TCP_CKSUM:
588                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
589                 td_offset |= (sizeof(struct rte_tcp_hdr) >> 2) <<
590                         ICE_TX_DESC_LEN_L4_LEN_S;
591                 break;
592         case PKT_TX_SCTP_CKSUM:
593                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP;
594                 td_offset |= (sizeof(struct rte_sctp_hdr) >> 2) <<
595                         ICE_TX_DESC_LEN_L4_LEN_S;
596                 break;
597         case PKT_TX_UDP_CKSUM:
598                 td_cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP;
599                 td_offset |= (sizeof(struct rte_udp_hdr) >> 2) <<
600                         ICE_TX_DESC_LEN_L4_LEN_S;
601                 break;
602         default:
603                 break;
604         }
605
606         *txd_hi |= ((uint64_t)td_offset) << ICE_TXD_QW1_OFFSET_S;
607
608         /* Tx VLAN/QINQ insertion Offload */
609         if (ol_flags & (PKT_TX_VLAN | PKT_TX_QINQ)) {
610                 td_cmd |= ICE_TX_DESC_CMD_IL2TAG1;
611                 *txd_hi |= ((uint64_t)tx_pkt->vlan_tci <<
612                                 ICE_TXD_QW1_L2TAG1_S);
613         }
614
615         *txd_hi |= ((uint64_t)td_cmd) << ICE_TXD_QW1_CMD_S;
616 }
617 #endif