9bc85731278848a8e17afb91472da7c9ffba26f6
[dpdk.git] / drivers / net / iavf / iavf_rxtx.h
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Intel Corporation
3  */
4
5 #ifndef _IAVF_RXTX_H_
6 #define _IAVF_RXTX_H_
7
8 /* In QLEN must be whole number of 32 descriptors. */
9 #define IAVF_ALIGN_RING_DESC      32
10 #define IAVF_MIN_RING_DESC        64
11 #define IAVF_MAX_RING_DESC        4096
12 #define IAVF_DMA_MEM_ALIGN        4096
13 /* Base address of the HW descriptor ring should be 128B aligned. */
14 #define IAVF_RING_BASE_ALIGN      128
15
16 /* used for Rx Bulk Allocate */
17 #define IAVF_RX_MAX_BURST         32
18
19 /* used for Vector PMD */
20 #define IAVF_VPMD_RX_MAX_BURST    32
21 #define IAVF_VPMD_TX_MAX_BURST    32
22 #define IAVF_RXQ_REARM_THRESH     32
23 #define IAVF_VPMD_DESCS_PER_LOOP  4
24 #define IAVF_VPMD_TX_MAX_FREE_BUF 64
25
26 #define IAVF_NO_VECTOR_FLAGS (                           \
27                 DEV_TX_OFFLOAD_MULTI_SEGS |              \
28                 DEV_TX_OFFLOAD_VLAN_INSERT |             \
29                 DEV_TX_OFFLOAD_SCTP_CKSUM |              \
30                 DEV_TX_OFFLOAD_UDP_CKSUM |               \
31                 DEV_TX_OFFLOAD_TCP_TSO |                 \
32                 DEV_TX_OFFLOAD_TCP_CKSUM)
33
34 #define DEFAULT_TX_RS_THRESH     32
35 #define DEFAULT_TX_FREE_THRESH   32
36
37 #define IAVF_MIN_TSO_MSS          256
38 #define IAVF_MAX_TSO_MSS          9668
39 #define IAVF_TSO_MAX_SEG          UINT8_MAX
40 #define IAVF_TX_MAX_MTU_SEG       8
41
42 #define IAVF_TX_CKSUM_OFFLOAD_MASK (             \
43                 PKT_TX_IP_CKSUM |                \
44                 PKT_TX_L4_MASK |                 \
45                 PKT_TX_TCP_SEG)
46
47 #define IAVF_TX_OFFLOAD_MASK (  \
48                 PKT_TX_OUTER_IPV6 |              \
49                 PKT_TX_OUTER_IPV4 |              \
50                 PKT_TX_IPV6 |                    \
51                 PKT_TX_IPV4 |                    \
52                 PKT_TX_VLAN_PKT |                \
53                 PKT_TX_IP_CKSUM |                \
54                 PKT_TX_L4_MASK |                 \
55                 PKT_TX_TCP_SEG)
56
57 #define IAVF_TX_OFFLOAD_NOTSUP_MASK \
58                 (PKT_TX_OFFLOAD_MASK ^ IAVF_TX_OFFLOAD_MASK)
59
60 /* HW desc structure, both 16-byte and 32-byte types are supported */
61 #ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
62 #define iavf_rx_desc iavf_16byte_rx_desc
63 #define iavf_rx_flex_desc iavf_16b_rx_flex_desc
64 #else
65 #define iavf_rx_desc iavf_32byte_rx_desc
66 #define iavf_rx_flex_desc iavf_32b_rx_flex_desc
67 #endif
68
69 struct iavf_rxq_ops {
70         void (*release_mbufs)(struct iavf_rx_queue *rxq);
71 };
72
73 struct iavf_txq_ops {
74         void (*release_mbufs)(struct iavf_tx_queue *txq);
75 };
76
77 /* Structure associated with each Rx queue. */
78 struct iavf_rx_queue {
79         struct rte_mempool *mp;       /* mbuf pool to populate Rx ring */
80         const struct rte_memzone *mz; /* memzone for Rx ring */
81         volatile union iavf_rx_desc *rx_ring; /* Rx ring virtual address */
82         uint64_t rx_ring_phys_addr;   /* Rx ring DMA address */
83         struct rte_mbuf **sw_ring;     /* address of SW ring */
84         uint16_t nb_rx_desc;          /* ring length */
85         uint16_t rx_tail;             /* current value of tail */
86         volatile uint8_t *qrx_tail;   /* register address of tail */
87         uint16_t rx_free_thresh;      /* max free RX desc to hold */
88         uint16_t nb_rx_hold;          /* number of held free RX desc */
89         struct rte_mbuf *pkt_first_seg; /* first segment of current packet */
90         struct rte_mbuf *pkt_last_seg;  /* last segment of current packet */
91         struct rte_mbuf fake_mbuf;      /* dummy mbuf */
92         uint8_t rxdid;
93
94         /* used for VPMD */
95         uint16_t rxrearm_nb;       /* number of remaining to be re-armed */
96         uint16_t rxrearm_start;    /* the idx we start the re-arming from */
97         uint64_t mbuf_initializer; /* value to init mbufs */
98
99         /* for rx bulk */
100         uint16_t rx_nb_avail;      /* number of staged packets ready */
101         uint16_t rx_next_avail;    /* index of next staged packets */
102         uint16_t rx_free_trigger;  /* triggers rx buffer allocation */
103         struct rte_mbuf *rx_stage[IAVF_RX_MAX_BURST * 2]; /* store mbuf */
104
105         uint16_t port_id;        /* device port ID */
106         uint8_t crc_len;        /* 0 if CRC stripped, 4 otherwise */
107         uint16_t queue_id;      /* Rx queue index */
108         uint16_t rx_buf_len;    /* The packet buffer size */
109         uint16_t rx_hdr_len;    /* The header buffer size */
110         uint16_t max_pkt_len;   /* Maximum packet length */
111         struct iavf_vsi *vsi; /**< the VSI this queue belongs to */
112
113         bool q_set;             /* if rx queue has been configured */
114         bool rx_deferred_start; /* don't start this queue in dev start */
115         const struct iavf_rxq_ops *ops;
116 };
117
118 struct iavf_tx_entry {
119         struct rte_mbuf *mbuf;
120         uint16_t next_id;
121         uint16_t last_id;
122 };
123
124 /* Structure associated with each TX queue. */
125 struct iavf_tx_queue {
126         const struct rte_memzone *mz;  /* memzone for Tx ring */
127         volatile struct iavf_tx_desc *tx_ring; /* Tx ring virtual address */
128         uint64_t tx_ring_phys_addr;    /* Tx ring DMA address */
129         struct iavf_tx_entry *sw_ring;  /* address array of SW ring */
130         uint16_t nb_tx_desc;           /* ring length */
131         uint16_t tx_tail;              /* current value of tail */
132         volatile uint8_t *qtx_tail;    /* register address of tail */
133         /* number of used desc since RS bit set */
134         uint16_t nb_used;
135         uint16_t nb_free;
136         uint16_t last_desc_cleaned;    /* last desc have been cleaned*/
137         uint16_t free_thresh;
138         uint16_t rs_thresh;
139
140         uint16_t port_id;
141         uint16_t queue_id;
142         uint64_t offloads;
143         uint16_t next_dd;              /* next to set RS, for VPMD */
144         uint16_t next_rs;              /* next to check DD,  for VPMD */
145
146         bool q_set;                    /* if rx queue has been configured */
147         bool tx_deferred_start;        /* don't start this queue in dev start */
148         const struct iavf_txq_ops *ops;
149 };
150
151 /* Offload features */
152 union iavf_tx_offload {
153         uint64_t data;
154         struct {
155                 uint64_t l2_len:7; /* L2 (MAC) Header Length. */
156                 uint64_t l3_len:9; /* L3 (IP) Header Length. */
157                 uint64_t l4_len:8; /* L4 Header Length. */
158                 uint64_t tso_segsz:16; /* TCP TSO segment size */
159                 /* uint64_t unused : 24; */
160         };
161 };
162
163 /* Rx Flex Descriptors
164  * These descriptors are used instead of the legacy version descriptors
165  */
166 union iavf_16b_rx_flex_desc {
167         struct {
168                 __le64 pkt_addr; /* Packet buffer address */
169                 __le64 hdr_addr; /* Header buffer address */
170                                  /* bit 0 of hdr_addr is DD bit */
171         } read;
172         struct {
173                 /* Qword 0 */
174                 u8 rxdid; /* descriptor builder profile ID */
175                 u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
176                 __le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
177                 __le16 pkt_len; /* [15:14] are reserved */
178                 __le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
179                                                 /* sph=[11:11] */
180                                                 /* ff1/ext=[15:12] */
181
182                 /* Qword 1 */
183                 __le16 status_error0;
184                 __le16 l2tag1;
185                 __le16 flex_meta0;
186                 __le16 flex_meta1;
187         } wb; /* writeback */
188 };
189
190 union iavf_32b_rx_flex_desc {
191         struct {
192                 __le64 pkt_addr; /* Packet buffer address */
193                 __le64 hdr_addr; /* Header buffer address */
194                                  /* bit 0 of hdr_addr is DD bit */
195                 __le64 rsvd1;
196                 __le64 rsvd2;
197         } read;
198         struct {
199                 /* Qword 0 */
200                 u8 rxdid; /* descriptor builder profile ID */
201                 u8 mir_id_umb_cast; /* mirror=[5:0], umb=[7:6] */
202                 __le16 ptype_flex_flags0; /* ptype=[9:0], ff0=[15:10] */
203                 __le16 pkt_len; /* [15:14] are reserved */
204                 __le16 hdr_len_sph_flex_flags1; /* header=[10:0] */
205                                                 /* sph=[11:11] */
206                                                 /* ff1/ext=[15:12] */
207
208                 /* Qword 1 */
209                 __le16 status_error0;
210                 __le16 l2tag1;
211                 __le16 flex_meta0;
212                 __le16 flex_meta1;
213
214                 /* Qword 2 */
215                 __le16 status_error1;
216                 u8 flex_flags2;
217                 u8 time_stamp_low;
218                 __le16 l2tag2_1st;
219                 __le16 l2tag2_2nd;
220
221                 /* Qword 3 */
222                 __le16 flex_meta2;
223                 __le16 flex_meta3;
224                 union {
225                         struct {
226                                 __le16 flex_meta4;
227                                 __le16 flex_meta5;
228                         } flex;
229                         __le32 ts_high;
230                 } flex_ts;
231         } wb; /* writeback */
232 };
233
234 /* Rx Flex Descriptor
235  * RxDID Profile ID 16-21
236  * Flex-field 0: RSS hash lower 16-bits
237  * Flex-field 1: RSS hash upper 16-bits
238  * Flex-field 2: Flow ID lower 16-bits
239  * Flex-field 3: Flow ID upper 16-bits
240  * Flex-field 4: AUX0
241  * Flex-field 5: AUX1
242  */
243 struct iavf_32b_rx_flex_desc_comms {
244         /* Qword 0 */
245         u8 rxdid;
246         u8 mir_id_umb_cast;
247         __le16 ptype_flexi_flags0;
248         __le16 pkt_len;
249         __le16 hdr_len_sph_flex_flags1;
250
251         /* Qword 1 */
252         __le16 status_error0;
253         __le16 l2tag1;
254         __le32 rss_hash;
255
256         /* Qword 2 */
257         __le16 status_error1;
258         u8 flexi_flags2;
259         u8 ts_low;
260         __le16 l2tag2_1st;
261         __le16 l2tag2_2nd;
262
263         /* Qword 3 */
264         __le32 flow_id;
265         union {
266                 struct {
267                         __le16 aux0;
268                         __le16 aux1;
269                 } flex;
270                 __le32 ts_high;
271         } flex_ts;
272 };
273
274 /* Rx Flex Descriptor
275  * RxDID Profile ID 22-23 (swap Hash and FlowID)
276  * Flex-field 0: Flow ID lower 16-bits
277  * Flex-field 1: Flow ID upper 16-bits
278  * Flex-field 2: RSS hash lower 16-bits
279  * Flex-field 3: RSS hash upper 16-bits
280  * Flex-field 4: AUX0
281  * Flex-field 5: AUX1
282  */
283 struct iavf_32b_rx_flex_desc_comms_ovs {
284         /* Qword 0 */
285         u8 rxdid;
286         u8 mir_id_umb_cast;
287         __le16 ptype_flexi_flags0;
288         __le16 pkt_len;
289         __le16 hdr_len_sph_flex_flags1;
290
291         /* Qword 1 */
292         __le16 status_error0;
293         __le16 l2tag1;
294         __le32 flow_id;
295
296         /* Qword 2 */
297         __le16 status_error1;
298         u8 flexi_flags2;
299         u8 ts_low;
300         __le16 l2tag2_1st;
301         __le16 l2tag2_2nd;
302
303         /* Qword 3 */
304         __le32 rss_hash;
305         union {
306                 struct {
307                         __le16 aux0;
308                         __le16 aux1;
309                 } flex;
310                 __le32 ts_high;
311         } flex_ts;
312 };
313
314 /* Receive Flex Descriptor profile IDs: There are a total
315  * of 64 profiles where profile IDs 0/1 are for legacy; and
316  * profiles 2-63 are flex profiles that can be programmed
317  * with a specific metadata (profile 7 reserved for HW)
318  */
319 enum iavf_rxdid {
320         IAVF_RXDID_LEGACY_0             = 0,
321         IAVF_RXDID_LEGACY_1             = 1,
322         IAVF_RXDID_FLEX_NIC             = 2,
323         IAVF_RXDID_FLEX_NIC_2           = 6,
324         IAVF_RXDID_HW                   = 7,
325         IAVF_RXDID_COMMS_GENERIC        = 16,
326         IAVF_RXDID_COMMS_AUX_VLAN       = 17,
327         IAVF_RXDID_COMMS_AUX_IPV4       = 18,
328         IAVF_RXDID_COMMS_AUX_IPV6       = 19,
329         IAVF_RXDID_COMMS_AUX_IPV6_FLOW  = 20,
330         IAVF_RXDID_COMMS_AUX_TCP        = 21,
331         IAVF_RXDID_COMMS_OVS_1          = 22,
332         IAVF_RXDID_COMMS_OVS_2          = 23,
333         IAVF_RXDID_LAST                 = 63,
334 };
335
336 enum iavf_rx_flex_desc_status_error_0_bits {
337         /* Note: These are predefined bit offsets */
338         IAVF_RX_FLEX_DESC_STATUS0_DD_S = 0,
339         IAVF_RX_FLEX_DESC_STATUS0_EOF_S,
340         IAVF_RX_FLEX_DESC_STATUS0_HBO_S,
341         IAVF_RX_FLEX_DESC_STATUS0_L3L4P_S,
342         IAVF_RX_FLEX_DESC_STATUS0_XSUM_IPE_S,
343         IAVF_RX_FLEX_DESC_STATUS0_XSUM_L4E_S,
344         IAVF_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S,
345         IAVF_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_S,
346         IAVF_RX_FLEX_DESC_STATUS0_LPBK_S,
347         IAVF_RX_FLEX_DESC_STATUS0_IPV6EXADD_S,
348         IAVF_RX_FLEX_DESC_STATUS0_RXE_S,
349         IAVF_RX_FLEX_DESC_STATUS0_CRCP_S,
350         IAVF_RX_FLEX_DESC_STATUS0_RSS_VALID_S,
351         IAVF_RX_FLEX_DESC_STATUS0_L2TAG1P_S,
352         IAVF_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_S,
353         IAVF_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_S,
354         IAVF_RX_FLEX_DESC_STATUS0_LAST /* this entry must be last!!! */
355 };
356
357 /* for iavf_32b_rx_flex_desc.ptype_flex_flags0 member */
358 #define IAVF_RX_FLEX_DESC_PTYPE_M       (0x3FF) /* 10-bits */
359
360 /* for iavf_32b_rx_flex_desc.pkt_len member */
361 #define IAVF_RX_FLX_DESC_PKT_LEN_M      (0x3FFF) /* 14-bits */
362
363 int iavf_dev_rx_queue_setup(struct rte_eth_dev *dev,
364                            uint16_t queue_idx,
365                            uint16_t nb_desc,
366                            unsigned int socket_id,
367                            const struct rte_eth_rxconf *rx_conf,
368                            struct rte_mempool *mp);
369
370 int iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id);
371 int iavf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id);
372 void iavf_dev_rx_queue_release(void *rxq);
373
374 int iavf_dev_tx_queue_setup(struct rte_eth_dev *dev,
375                            uint16_t queue_idx,
376                            uint16_t nb_desc,
377                            unsigned int socket_id,
378                            const struct rte_eth_txconf *tx_conf);
379 int iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id);
380 int iavf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id);
381 void iavf_dev_tx_queue_release(void *txq);
382 void iavf_stop_queues(struct rte_eth_dev *dev);
383 uint16_t iavf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
384                        uint16_t nb_pkts);
385 uint16_t iavf_recv_pkts_flex_rxd(void *rx_queue,
386                                  struct rte_mbuf **rx_pkts,
387                                  uint16_t nb_pkts);
388 uint16_t iavf_recv_scattered_pkts(void *rx_queue,
389                                  struct rte_mbuf **rx_pkts,
390                                  uint16_t nb_pkts);
391 uint16_t iavf_recv_scattered_pkts_flex_rxd(void *rx_queue,
392                                            struct rte_mbuf **rx_pkts,
393                                            uint16_t nb_pkts);
394 uint16_t iavf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
395                        uint16_t nb_pkts);
396 uint16_t iavf_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
397                        uint16_t nb_pkts);
398 void iavf_set_rx_function(struct rte_eth_dev *dev);
399 void iavf_set_tx_function(struct rte_eth_dev *dev);
400 void iavf_dev_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
401                           struct rte_eth_rxq_info *qinfo);
402 void iavf_dev_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
403                           struct rte_eth_txq_info *qinfo);
404 uint32_t iavf_dev_rxq_count(struct rte_eth_dev *dev, uint16_t queue_id);
405 int iavf_dev_rx_desc_status(void *rx_queue, uint16_t offset);
406 int iavf_dev_tx_desc_status(void *tx_queue, uint16_t offset);
407
408 uint16_t iavf_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
409                            uint16_t nb_pkts);
410 uint16_t iavf_recv_pkts_vec_flex_rxd(void *rx_queue, struct rte_mbuf **rx_pkts,
411                                      uint16_t nb_pkts);
412 uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
413                                      struct rte_mbuf **rx_pkts,
414                                      uint16_t nb_pkts);
415 uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
416                                                struct rte_mbuf **rx_pkts,
417                                                uint16_t nb_pkts);
418 uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
419                                   uint16_t nb_pkts);
420 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
421                                  uint16_t nb_pkts);
422 uint16_t iavf_recv_pkts_vec_avx2_flex_rxd(void *rx_queue,
423                                           struct rte_mbuf **rx_pkts,
424                                           uint16_t nb_pkts);
425 uint16_t iavf_recv_scattered_pkts_vec_avx2(void *rx_queue,
426                                            struct rte_mbuf **rx_pkts,
427                                            uint16_t nb_pkts);
428 uint16_t iavf_recv_scattered_pkts_vec_avx2_flex_rxd(void *rx_queue,
429                                                     struct rte_mbuf **rx_pkts,
430                                                     uint16_t nb_pkts);
431 uint16_t iavf_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
432                             uint16_t nb_pkts);
433 uint16_t iavf_xmit_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts,
434                                  uint16_t nb_pkts);
435 int iavf_rx_vec_dev_check(struct rte_eth_dev *dev);
436 int iavf_tx_vec_dev_check(struct rte_eth_dev *dev);
437 int iavf_rxq_vec_setup(struct iavf_rx_queue *rxq);
438 int iavf_txq_vec_setup(struct iavf_tx_queue *txq);
439
440 const uint32_t *iavf_get_default_ptype_table(void);
441
442 static inline
443 void iavf_dump_rx_descriptor(struct iavf_rx_queue *rxq,
444                             const volatile void *desc,
445                             uint16_t rx_id)
446 {
447 #ifdef RTE_LIBRTE_IAVF_16BYTE_RX_DESC
448         const volatile union iavf_16byte_rx_desc *rx_desc = desc;
449
450         printf("Queue %d Rx_desc %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64"\n",
451                rxq->queue_id, rx_id, rx_desc->read.pkt_addr,
452                rx_desc->read.hdr_addr);
453 #else
454         const volatile union iavf_32byte_rx_desc *rx_desc = desc;
455
456         printf("Queue %d Rx_desc %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64
457                " QW2: 0x%016"PRIx64" QW3: 0x%016"PRIx64"\n", rxq->queue_id,
458                rx_id, rx_desc->read.pkt_addr, rx_desc->read.hdr_addr,
459                rx_desc->read.rsvd1, rx_desc->read.rsvd2);
460 #endif
461 }
462
463 /* All the descriptors are 16 bytes, so just use one of them
464  * to print the qwords
465  */
466 static inline
467 void iavf_dump_tx_descriptor(const struct iavf_tx_queue *txq,
468                             const volatile void *desc, uint16_t tx_id)
469 {
470         const char *name;
471         const volatile struct iavf_tx_desc *tx_desc = desc;
472         enum iavf_tx_desc_dtype_value type;
473
474         type = (enum iavf_tx_desc_dtype_value)rte_le_to_cpu_64(
475                 tx_desc->cmd_type_offset_bsz &
476                 rte_cpu_to_le_64(IAVF_TXD_QW1_DTYPE_MASK));
477         switch (type) {
478         case IAVF_TX_DESC_DTYPE_DATA:
479                 name = "Tx_data_desc";
480                 break;
481         case IAVF_TX_DESC_DTYPE_CONTEXT:
482                 name = "Tx_context_desc";
483                 break;
484         default:
485                 name = "unknown_desc";
486                 break;
487         }
488
489         printf("Queue %d %s %d: QW0: 0x%016"PRIx64" QW1: 0x%016"PRIx64"\n",
490                txq->queue_id, name, tx_id, tx_desc->buffer_addr,
491                tx_desc->cmd_type_offset_bsz);
492 }
493
494 #ifdef RTE_LIBRTE_IAVF_DEBUG_DUMP_DESC
495 #define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) \
496         iavf_dump_rx_descriptor(rxq, desc, rx_id)
497 #define IAVF_DUMP_TX_DESC(txq, desc, tx_id) \
498         iavf_dump_tx_descriptor(txq, desc, tx_id)
499 #else
500 #define IAVF_DUMP_RX_DESC(rxq, desc, rx_id) do { } while (0)
501 #define IAVF_DUMP_TX_DESC(txq, desc, tx_id) do { } while (0)
502 #endif
503
504 #endif /* _IAVF_RXTX_H_ */