1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (C) 2020 Marvell International Ltd.
4 #ifndef _IPSEC_WORKER_H_
5 #define _IPSEC_WORKER_H_
8 #include <rte_ethdev.h>
14 /* Configure how many packets ahead to prefetch, when reading packets */
15 #define PREFETCH_OFFSET 3
17 PKT_TYPE_PLAIN_IPV4 = 1,
27 PKT_POSTED /* for lookaside case */
31 struct rt_ctx *rt4_ctx;
32 struct rt_ctx *rt6_ctx;
36 * Conf required by event mode worker with tx internal port
38 struct lcore_conf_ev_tx_int_port_wrkr {
39 struct ipsec_ctx inbound;
40 struct ipsec_ctx outbound;
41 struct route_table rt;
42 } __rte_cache_aligned;
44 void ipsec_poll_mode_worker(void);
46 int ipsec_launch_one_lcore(void *args);
49 * helper routine for inline and cpu(synchronous) processing
50 * this is just to satisfy inbound_sa_check() and get_hop_for_offload_pkt().
51 * Should be removed in future.
54 prep_process_group(void *sa, struct rte_mbuf *mb[], uint32_t cnt)
57 struct ipsec_mbuf_metadata *priv;
59 for (j = 0; j != cnt; j++) {
60 priv = get_priv(mb[j]);
62 /* setup TSO related fields if TSO enabled*/
64 uint32_t ptype = mb[j]->packet_type;
65 /* only TCP is supported */
66 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP) {
67 mb[j]->tso_segsz = priv->sa->mss;
68 if ((IS_TUNNEL(priv->sa->flags))) {
69 mb[j]->outer_l3_len = mb[j]->l3_len;
70 mb[j]->outer_l2_len = mb[j]->l2_len;
72 RTE_MBUF_F_TX_TUNNEL_ESP;
73 if (RTE_ETH_IS_IPV4_HDR(ptype))
75 RTE_MBUF_F_TX_OUTER_IP_CKSUM;
77 mb[j]->l4_len = sizeof(struct rte_tcp_hdr);
78 mb[j]->ol_flags |= (RTE_MBUF_F_TX_TCP_SEG |
79 RTE_MBUF_F_TX_TCP_CKSUM);
80 if (RTE_ETH_IS_IPV4_HDR(ptype))
82 RTE_MBUF_F_TX_OUTER_IPV4;
85 RTE_MBUF_F_TX_OUTER_IPV6;
92 adjust_ipv4_pktlen(struct rte_mbuf *m, const struct rte_ipv4_hdr *iph,
97 plen = rte_be_to_cpu_16(iph->total_length) + l2_len;
98 if (plen < m->pkt_len) {
99 trim = m->pkt_len - plen;
100 rte_pktmbuf_trim(m, trim);
105 adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph,
110 plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len;
111 if (plen < m->pkt_len) {
112 trim = m->pkt_len - plen;
113 rte_pktmbuf_trim(m, trim);
118 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
120 const struct rte_ether_hdr *eth;
121 const struct rte_ipv4_hdr *iph4;
122 const struct rte_ipv6_hdr *iph6;
123 const struct rte_udp_hdr *udp;
124 uint16_t ip4_hdr_len;
127 eth = rte_pktmbuf_mtod(pkt, const struct rte_ether_hdr *);
128 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
130 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt,
132 adjust_ipv4_pktlen(pkt, iph4, 0);
134 switch (iph4->next_proto_id) {
136 t->ipsec.pkts[(t->ipsec.num)++] = pkt;
139 if (app_sa_prm.udp_encap == 1) {
140 ip4_hdr_len = ((iph4->version_ihl &
141 RTE_IPV4_HDR_IHL_MASK) *
142 RTE_IPV4_IHL_MULTIPLIER);
143 udp = rte_pktmbuf_mtod_offset(pkt,
144 struct rte_udp_hdr *, ip4_hdr_len);
145 nat_port = rte_cpu_to_be_16(IPSEC_NAT_T_PORT);
146 if (udp->src_port == nat_port ||
147 udp->dst_port == nat_port){
148 t->ipsec.pkts[(t->ipsec.num)++] = pkt;
150 MBUF_PTYPE_TUNNEL_ESP_IN_UDP;
156 t->ip4.data[t->ip4.num] = &iph4->next_proto_id;
157 t->ip4.pkts[(t->ip4.num)++] = pkt;
160 pkt->l3_len = sizeof(*iph4);
161 pkt->packet_type |= RTE_PTYPE_L3_IPV4;
162 if (pkt->packet_type & RTE_PTYPE_L4_TCP)
163 pkt->l4_len = sizeof(struct rte_tcp_hdr);
164 else if (pkt->packet_type & RTE_PTYPE_L4_UDP)
165 pkt->l4_len = sizeof(struct rte_udp_hdr);
166 } else if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
168 size_t l3len, ext_len;
171 /* get protocol type */
172 iph6 = (const struct rte_ipv6_hdr *)rte_pktmbuf_adj(pkt,
174 adjust_ipv6_pktlen(pkt, iph6, 0);
176 next_proto = iph6->proto;
178 /* determine l3 header size up to ESP extension */
179 l3len = sizeof(struct ip6_hdr);
180 p = rte_pktmbuf_mtod(pkt, uint8_t *);
181 while (next_proto != IPPROTO_ESP && l3len < pkt->data_len &&
182 (next_proto = rte_ipv6_get_next_ext(p + l3len,
183 next_proto, &ext_len)) >= 0)
186 /* drop packet when IPv6 header exceeds first segment length */
187 if (unlikely(l3len > pkt->data_len)) {
192 switch (next_proto) {
194 t->ipsec.pkts[(t->ipsec.num)++] = pkt;
197 if (app_sa_prm.udp_encap == 1) {
198 udp = rte_pktmbuf_mtod_offset(pkt,
199 struct rte_udp_hdr *, l3len);
200 nat_port = rte_cpu_to_be_16(IPSEC_NAT_T_PORT);
201 if (udp->src_port == nat_port ||
202 udp->dst_port == nat_port){
203 t->ipsec.pkts[(t->ipsec.num)++] = pkt;
205 MBUF_PTYPE_TUNNEL_ESP_IN_UDP;
211 t->ip6.data[t->ip6.num] = &iph6->proto;
212 t->ip6.pkts[(t->ip6.num)++] = pkt;
216 pkt->packet_type |= RTE_PTYPE_L3_IPV6;
218 /* Unknown/Unsupported type, drop the packet */
219 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
220 rte_be_to_cpu_16(eth->ether_type));
225 /* Check if the packet has been processed inline. For inline protocol
226 * processed packets, the metadata in the mbuf can be used to identify
227 * the security processing done on the packet. The metadata will be
228 * used to retrieve the application registered userdata associated
229 * with the security session.
232 if (pkt->ol_flags & RTE_MBUF_F_RX_SEC_OFFLOAD &&
233 rte_security_dynfield_is_registered()) {
235 struct ipsec_mbuf_metadata *priv;
236 struct rte_security_ctx *ctx = (struct rte_security_ctx *)
237 rte_eth_dev_get_sec_ctx(
240 /* Retrieve the userdata registered. Here, the userdata
241 * registered is the SA pointer.
243 sa = (struct ipsec_sa *)rte_security_get_userdata(ctx,
244 *rte_security_dynfield(pkt));
246 /* userdata could not be retrieved */
250 /* Save SA as priv member in mbuf. This will be used in the
251 * IPsec selector(SP-SA) check.
254 priv = get_priv(pkt);
260 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t,
269 for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) {
270 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET],
272 prepare_one_packet(pkts[i], t);
274 /* Process left packets */
275 for (; i < nb_pkts; i++)
276 prepare_one_packet(pkts[i], t);
280 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port,
281 const struct lcore_conf *qconf)
284 struct rte_ether_hdr *ethhdr;
286 ip = rte_pktmbuf_mtod(pkt, struct ip *);
288 ethhdr = (struct rte_ether_hdr *)
289 rte_pktmbuf_prepend(pkt, RTE_ETHER_HDR_LEN);
291 if (ip->ip_v == IPVERSION) {
292 pkt->ol_flags |= qconf->outbound.ipv4_offloads;
293 pkt->l3_len = sizeof(struct ip);
294 pkt->l2_len = RTE_ETHER_HDR_LEN;
298 /* calculate IPv4 cksum in SW */
299 if ((pkt->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) == 0)
300 ip->ip_sum = rte_ipv4_cksum((struct rte_ipv4_hdr *)ip);
302 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
304 pkt->ol_flags |= qconf->outbound.ipv6_offloads;
305 pkt->l3_len = sizeof(struct ip6_hdr);
306 pkt->l2_len = RTE_ETHER_HDR_LEN;
308 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
311 memcpy(ðhdr->src_addr, ðaddr_tbl[port].src,
312 sizeof(struct rte_ether_addr));
313 memcpy(ðhdr->dst_addr, ðaddr_tbl[port].dst,
314 sizeof(struct rte_ether_addr));
318 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port,
319 const struct lcore_conf *qconf)
322 const int32_t prefetch_offset = 2;
324 for (i = 0; i < (nb_pkts - prefetch_offset); i++) {
325 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]);
326 prepare_tx_pkt(pkts[i], port, qconf);
328 /* Process left packets */
329 for (; i < nb_pkts; i++)
330 prepare_tx_pkt(pkts[i], port, qconf);
333 /* Send burst of packets on an output interface */
334 static inline int32_t
335 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
337 struct rte_mbuf **m_table;
341 queueid = qconf->tx_queue_id[port];
342 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
344 prepare_tx_burst(m_table, n, port, qconf);
346 ret = rte_eth_tx_burst(port, queueid, m_table, n);
348 core_stats_update_tx(ret);
350 if (unlikely(ret < n)) {
352 free_pkts(&m_table[ret], 1);
360 * Helper function to fragment and queue for TX one packet.
362 static inline uint32_t
363 send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,
364 uint16_t port, uint8_t proto)
370 tbl = qconf->tx_mbufs + port;
373 /* free space for new fragments */
374 if (len + RTE_LIBRTE_IP_FRAG_MAX_FRAG >= RTE_DIM(tbl->m_table)) {
375 send_burst(qconf, len, port);
379 n = RTE_DIM(tbl->m_table) - len;
381 if (proto == IPPROTO_IP)
382 rc = rte_ipv4_fragment_packet(m, tbl->m_table + len,
383 n, mtu_size, m->pool, qconf->frag.pool_indir);
385 rc = rte_ipv6_fragment_packet(m, tbl->m_table + len,
386 n, mtu_size, m->pool, qconf->frag.pool_indir);
392 "%s: failed to fragment packet with size %u, "
394 __func__, m->pkt_len, rte_errno);
400 /* Enqueue a single packet, and send burst if queue is filled */
401 static inline int32_t
402 send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto)
406 struct lcore_conf *qconf;
408 lcore_id = rte_lcore_id();
410 qconf = &lcore_conf[lcore_id];
411 len = qconf->tx_mbufs[port].len;
413 if (m->pkt_len <= mtu_size) {
414 qconf->tx_mbufs[port].m_table[len] = m;
417 /* need to fragment the packet */
418 } else if (frag_tbl_sz > 0)
419 len = send_fragment_packet(qconf, m, port, proto);
423 /* enough pkts to be sent */
424 if (unlikely(len == MAX_PKT_BURST)) {
425 send_burst(qconf, MAX_PKT_BURST, port);
429 qconf->tx_mbufs[port].len = len;
434 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,
435 uint16_t lim, struct ipsec_spd_stats *stats)
438 uint32_t i, j, res, sa_idx;
440 if (ip->num == 0 || sp == NULL)
443 rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res,
444 ip->num, DEFAULT_MAX_CATEGORIES);
447 for (i = 0; i < ip->num; i++) {
455 if (res == DISCARD) {
461 /* Only check SPI match for processed IPSec packets */
462 if (i < lim && ((m->ol_flags & RTE_MBUF_F_RX_SEC_OFFLOAD) == 0)) {
469 if (!inbound_sa_check(sa, m, sa_idx)) {
480 static inline int32_t
481 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6)
483 struct ipsec_mbuf_metadata *priv;
486 priv = get_priv(pkt);
489 if (unlikely(sa == NULL)) {
490 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n");
498 return (sa->portid | RTE_LPM_LOOKUP_SUCCESS);
509 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
511 uint32_t hop[MAX_PKT_BURST * 2];
512 uint32_t dst_ip[MAX_PKT_BURST * 2];
515 uint16_t lpm_pkts = 0;
516 unsigned int lcoreid = rte_lcore_id();
521 /* Need to do an LPM lookup for non-inline packets. Inline packets will
522 * have port ID in the SA
525 for (i = 0; i < nb_pkts; i++) {
526 if (!(pkts[i]->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)) {
527 /* Security offload not enabled. So an LPM lookup is
528 * required to get the hop
530 offset = offsetof(struct ip, ip_dst);
531 dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i],
533 dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]);
538 rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts);
542 for (i = 0; i < nb_pkts; i++) {
543 if (pkts[i]->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) {
544 /* Read hop from the SA */
545 pkt_hop = get_hop_for_offload_pkt(pkts[i], 0);
547 /* Need to use hop returned by lookup */
548 pkt_hop = hop[lpm_pkts++];
551 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
552 core_statistics[lcoreid].lpm4.miss++;
553 free_pkts(&pkts[i], 1);
556 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP);
561 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
563 int32_t hop[MAX_PKT_BURST * 2];
564 uint8_t dst_ip[MAX_PKT_BURST * 2][16];
568 uint16_t lpm_pkts = 0;
569 unsigned int lcoreid = rte_lcore_id();
574 /* Need to do an LPM lookup for non-inline packets. Inline packets will
575 * have port ID in the SA
578 for (i = 0; i < nb_pkts; i++) {
579 if (!(pkts[i]->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD)) {
580 /* Security offload not enabled. So an LPM lookup is
581 * required to get the hop
583 offset = offsetof(struct ip6_hdr, ip6_dst);
584 ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *,
586 memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16);
591 rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop,
596 for (i = 0; i < nb_pkts; i++) {
597 if (pkts[i]->ol_flags & RTE_MBUF_F_TX_SEC_OFFLOAD) {
598 /* Read hop from the SA */
599 pkt_hop = get_hop_for_offload_pkt(pkts[i], 1);
601 /* Need to use hop returned by lookup */
602 pkt_hop = hop[lpm_pkts++];
606 core_statistics[lcoreid].lpm6.miss++;
607 free_pkts(&pkts[i], 1);
610 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6);
615 drain_tx_buffers(struct lcore_conf *qconf)
620 for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
621 buf = &qconf->tx_mbufs[portid];
624 send_burst(qconf, buf->len, portid);
629 #endif /* _IPSEC_WORKER_H_ */