4 * Copyright (C) Cavium, Inc. 2017. All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Cavium, Inc nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include <rte_atomic.h>
39 #include <rte_common.h>
40 #include <rte_ethdev.h>
41 #include <rte_ether.h>
44 #include <rte_prefetch.h>
46 #include "octeontx_ethdev.h"
47 #include "octeontx_rxtx.h"
48 #include "octeontx_logs.h"
50 /* Packet type table */
51 #define PTYPE_SIZE OCCTX_PKI_LTYPE_LAST
53 static const uint32_t __rte_cache_aligned
54 ptype_table[PTYPE_SIZE][PTYPE_SIZE][PTYPE_SIZE] = {
55 [LC_NONE][LE_NONE][LF_NONE] = RTE_PTYPE_UNKNOWN,
56 [LC_NONE][LE_NONE][LF_IPSEC_ESP] = RTE_PTYPE_UNKNOWN,
57 [LC_NONE][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L4_FRAG,
58 [LC_NONE][LE_NONE][LF_IPCOMP] = RTE_PTYPE_UNKNOWN,
59 [LC_NONE][LE_NONE][LF_TCP] = RTE_PTYPE_L4_TCP,
60 [LC_NONE][LE_NONE][LF_UDP] = RTE_PTYPE_L4_UDP,
61 [LC_NONE][LE_NONE][LF_GRE] = RTE_PTYPE_TUNNEL_GRE,
62 [LC_NONE][LE_NONE][LF_UDP_GENEVE] = RTE_PTYPE_TUNNEL_GENEVE,
63 [LC_NONE][LE_NONE][LF_UDP_VXLAN] = RTE_PTYPE_TUNNEL_VXLAN,
64 [LC_NONE][LE_NONE][LF_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE,
66 [LC_IPV4][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN,
67 [LC_IPV4][LE_NONE][LF_IPSEC_ESP] =
68 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L3_IPV4,
69 [LC_IPV4][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_FRAG,
70 [LC_IPV4][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_UNKNOWN,
71 [LC_IPV4][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP,
72 [LC_IPV4][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP,
73 [LC_IPV4][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GRE,
74 [LC_IPV4][LE_NONE][LF_UDP_GENEVE] =
75 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_GENEVE,
76 [LC_IPV4][LE_NONE][LF_UDP_VXLAN] =
77 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_VXLAN,
78 [LC_IPV4][LE_NONE][LF_NVGRE] =
79 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE,
81 [LC_IPV4_OPT][LE_NONE][LF_NONE] =
82 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN,
83 [LC_IPV4_OPT][LE_NONE][LF_IPSEC_ESP] =
84 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L3_IPV4,
85 [LC_IPV4_OPT][LE_NONE][LF_IPFRAG] =
86 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_FRAG,
87 [LC_IPV4_OPT][LE_NONE][LF_IPCOMP] =
88 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_UNKNOWN,
89 [LC_IPV4_OPT][LE_NONE][LF_TCP] =
90 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_TCP,
91 [LC_IPV4_OPT][LE_NONE][LF_UDP] =
92 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L4_UDP,
93 [LC_IPV4_OPT][LE_NONE][LF_GRE] =
94 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GRE,
95 [LC_IPV4_OPT][LE_NONE][LF_UDP_GENEVE] =
96 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_GENEVE,
97 [LC_IPV4_OPT][LE_NONE][LF_UDP_VXLAN] =
98 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_VXLAN,
99 [LC_IPV4_OPT][LE_NONE][LF_NVGRE] =
100 RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_TUNNEL_NVGRE,
102 [LC_IPV6][LE_NONE][LF_NONE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN,
103 [LC_IPV6][LE_NONE][LF_IPSEC_ESP] =
104 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L3_IPV4,
105 [LC_IPV6][LE_NONE][LF_IPFRAG] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_FRAG,
106 [LC_IPV6][LE_NONE][LF_IPCOMP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_UNKNOWN,
107 [LC_IPV6][LE_NONE][LF_TCP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP,
108 [LC_IPV6][LE_NONE][LF_UDP] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP,
109 [LC_IPV6][LE_NONE][LF_GRE] = RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GRE,
110 [LC_IPV6][LE_NONE][LF_UDP_GENEVE] =
111 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_GENEVE,
112 [LC_IPV6][LE_NONE][LF_UDP_VXLAN] =
113 RTE_PTYPE_L3_IPV6 | RTE_PTYPE_TUNNEL_VXLAN,
114 [LC_IPV6][LE_NONE][LF_NVGRE] =
115 RTE_PTYPE_L3_IPV4 | RTE_PTYPE_TUNNEL_NVGRE,
116 [LC_IPV6_OPT][LE_NONE][LF_NONE] =
117 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN,
118 [LC_IPV6_OPT][LE_NONE][LF_IPSEC_ESP] =
119 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L3_IPV4,
120 [LC_IPV6_OPT][LE_NONE][LF_IPFRAG] =
121 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_FRAG,
122 [LC_IPV6_OPT][LE_NONE][LF_IPCOMP] =
123 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_UNKNOWN,
124 [LC_IPV6_OPT][LE_NONE][LF_TCP] =
125 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_TCP,
126 [LC_IPV6_OPT][LE_NONE][LF_UDP] =
127 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L4_UDP,
128 [LC_IPV6_OPT][LE_NONE][LF_GRE] =
129 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GRE,
130 [LC_IPV6_OPT][LE_NONE][LF_UDP_GENEVE] =
131 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_GENEVE,
132 [LC_IPV6_OPT][LE_NONE][LF_UDP_VXLAN] =
133 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_VXLAN,
134 [LC_IPV6_OPT][LE_NONE][LF_NVGRE] =
135 RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_TUNNEL_NVGRE,
139 static __rte_always_inline uint16_t __hot
140 __octeontx_xmit_pkts(void *lmtline_va, void *ioreg_va, int64_t *fc_status_va,
141 struct rte_mbuf *tx_pkt)
146 if (unlikely(*((volatile int64_t *)fc_status_va) < 0))
149 /* Get the gaura Id */
150 gaura_id = octeontx_fpa_bufpool_gpool((uintptr_t)tx_pkt->pool->pool_id);
152 /* Setup PKO_SEND_HDR_S */
153 cmd_buf[0] = tx_pkt->data_len & 0xffff;
156 /* Set don't free bit if reference count > 1 */
157 if (rte_mbuf_refcnt_read(tx_pkt) > 1)
158 cmd_buf[0] |= (1ULL << 58); /* SET DF */
160 /* Setup PKO_SEND_GATHER_S */
161 cmd_buf[(1 << 1) | 1] = rte_mbuf_data_dma_addr(tx_pkt);
162 cmd_buf[(1 << 1) | 0] = PKO_SEND_GATHER_SUBDC |
163 PKO_SEND_GATHER_LDTYPE(0x1ull) |
164 PKO_SEND_GATHER_GAUAR((long)gaura_id) |
167 octeontx_reg_lmtst(lmtline_va, ioreg_va, cmd_buf, PKO_CMD_SZ);
173 octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
176 struct octeontx_txq *txq = tx_queue;
177 octeontx_dq_t *dq = &txq->dq;
182 while (count < nb_pkts) {
183 res = __octeontx_xmit_pkts(dq->lmtline_va, dq->ioreg_va,
192 return count; /* return number of pkts transmitted */
196 octeontx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
198 struct rte_mbuf *mbuf;
199 struct octeontx_rxq *rxq;
203 uint16_t valid_event;
207 while (count < nb_pkts) {
208 valid_event = rte_event_dequeue_burst(rxq->evdev,
214 wqe = (octtx_wqe_t *)(uintptr_t)ev.u64;
215 rte_prefetch_non_temporal(wqe);
217 /* Get mbuf from wqe */
218 mbuf = (struct rte_mbuf *)((uintptr_t)wqe -
219 OCTTX_PACKET_WQE_SKIP);
221 ptype_table[wqe->s.w2.lcty][wqe->s.w2.lety][wqe->s.w2.lfty];
222 mbuf->data_off = RTE_PTR_DIFF(wqe->s.w3.addr, mbuf->buf_addr);
223 mbuf->pkt_len = wqe->s.w1.len;
224 mbuf->data_len = mbuf->pkt_len;
227 mbuf->port = rxq->port_id;
228 rte_mbuf_refcnt_set(mbuf, 1);
229 rx_pkts[count++] = mbuf;
232 return count; /* return number of pkts received */