1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
9 #include <rte_cryptodev.h>
12 #include "ipsec_sqn.h"
18 typedef uint16_t (*esp_inb_process_t)(const struct rte_ipsec_sa *sa,
19 struct rte_mbuf *mb[], uint32_t sqn[], uint32_t dr[], uint16_t num);
22 * helper function to fill crypto_sym op for cipher+auth algorithms.
23 * used by inb_cop_prepare(), see below.
26 sop_ciph_auth_prepare(struct rte_crypto_sym_op *sop,
27 const struct rte_ipsec_sa *sa, const union sym_op_data *icv,
28 uint32_t pofs, uint32_t plen)
30 sop->cipher.data.offset = pofs + sa->ctp.cipher.offset;
31 sop->cipher.data.length = plen - sa->ctp.cipher.length;
32 sop->auth.data.offset = pofs + sa->ctp.auth.offset;
33 sop->auth.data.length = plen - sa->ctp.auth.length;
34 sop->auth.digest.data = icv->va;
35 sop->auth.digest.phys_addr = icv->pa;
39 * helper function to fill crypto_sym op for aead algorithms
40 * used by inb_cop_prepare(), see below.
43 sop_aead_prepare(struct rte_crypto_sym_op *sop,
44 const struct rte_ipsec_sa *sa, const union sym_op_data *icv,
45 uint32_t pofs, uint32_t plen)
47 sop->aead.data.offset = pofs + sa->ctp.cipher.offset;
48 sop->aead.data.length = plen - sa->ctp.cipher.length;
49 sop->aead.digest.data = icv->va;
50 sop->aead.digest.phys_addr = icv->pa;
51 sop->aead.aad.data = icv->va + sa->icv_len;
52 sop->aead.aad.phys_addr = icv->pa + sa->icv_len;
56 * setup crypto op and crypto sym op for ESP inbound packet.
59 inb_cop_prepare(struct rte_crypto_op *cop,
60 const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,
61 const union sym_op_data *icv, uint32_t pofs, uint32_t plen)
63 struct rte_crypto_sym_op *sop;
64 struct aead_gcm_iv *gcm;
65 struct aesctr_cnt_blk *ctr;
70 ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *,
71 pofs + sizeof(struct rte_esp_hdr));
73 /* fill sym op fields */
77 case ALGO_TYPE_AES_GCM:
78 sop_aead_prepare(sop, sa, icv, pofs, plen);
80 /* fill AAD IV (located inside crypto op) */
81 gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *,
83 aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
85 case ALGO_TYPE_AES_CBC:
86 case ALGO_TYPE_3DES_CBC:
87 sop_ciph_auth_prepare(sop, sa, icv, pofs, plen);
89 /* copy iv from the input packet to the cop */
90 ivc = rte_crypto_op_ctod_offset(cop, uint64_t *, sa->iv_ofs);
91 copy_iv(ivc, ivp, sa->iv_len);
93 case ALGO_TYPE_AES_CTR:
94 sop_ciph_auth_prepare(sop, sa, icv, pofs, plen);
96 /* fill CTR block (located inside crypto op) */
97 ctr = rte_crypto_op_ctod_offset(cop, struct aesctr_cnt_blk *,
99 aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);
102 sop_ciph_auth_prepare(sop, sa, icv, pofs, plen);
108 * for pure cryptodev (lookaside none) depending on SA settings,
109 * we might have to write some extra data to the packet.
112 inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
113 const union sym_op_data *icv)
115 struct aead_gcm_aad *aad;
117 /* insert SQN.hi between ESP trailer and ICV */
118 if (sa->sqh_len != 0)
119 insert_sqh(sqn_hi32(sqc), icv->va, sa->icv_len);
122 * fill AAD fields, if any (aad fields are placed after icv),
123 * right now we support only one AEAD algorithm: AES-GCM.
125 if (sa->aad_len != 0) {
126 aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);
127 aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa));
132 * setup/update packet data and metadata for ESP inbound tunnel case.
134 static inline int32_t
135 inb_pkt_prepare(const struct rte_ipsec_sa *sa, const struct replay_sqn *rsn,
136 struct rte_mbuf *mb, uint32_t hlen, union sym_op_data *icv)
140 uint32_t clen, icv_ofs, plen;
142 struct rte_esp_hdr *esph;
144 esph = rte_pktmbuf_mtod_offset(mb, struct rte_esp_hdr *, hlen);
147 * retrieve and reconstruct SQN, then check it, then
148 * convert it back into network byte order.
150 sqn = rte_be_to_cpu_32(esph->seq);
152 sqn = reconstruct_esn(rsn->sqn, sqn, sa->replay.win_sz);
154 rc = esn_inb_check_sqn(rsn, sa, sqn);
158 sqn = rte_cpu_to_be_64(sqn);
160 /* start packet manipulation */
164 ml = rte_pktmbuf_lastseg(mb);
165 icv_ofs = ml->data_len - sa->icv_len + sa->sqh_len;
167 /* check that packet has a valid length */
168 clen = plen - sa->ctp.cipher.length;
169 if ((int32_t)clen < 0 || (clen & (sa->pad_align - 1)) != 0)
172 /* we have to allocate space for AAD somewhere,
173 * right now - just use free trailing space at the last segment.
174 * Would probably be more convenient to reserve space for AAD
175 * inside rte_crypto_op itself
176 * (again for IV space is already reserved inside cop).
178 if (sa->aad_len + sa->sqh_len > rte_pktmbuf_tailroom(ml))
181 icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_ofs);
182 icv->pa = rte_pktmbuf_iova_offset(ml, icv_ofs);
184 inb_pkt_xprepare(sa, sqn, icv);
189 * setup/update packets and crypto ops for ESP inbound case.
192 esp_inb_pkt_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
193 struct rte_crypto_op *cop[], uint16_t num)
197 struct rte_ipsec_sa *sa;
198 struct rte_cryptodev_sym_session *cs;
199 struct replay_sqn *rsn;
200 union sym_op_data icv;
205 rsn = rsn_acquire(sa);
208 for (i = 0; i != num; i++) {
210 hl = mb[i]->l2_len + mb[i]->l3_len;
211 rc = inb_pkt_prepare(sa, rsn, mb[i], hl, &icv);
213 lksd_none_cop_prepare(cop[k], cs, mb[i]);
214 inb_cop_prepare(cop[k], sa, mb[i], &icv, hl, rc);
220 rsn_release(sa, rsn);
222 /* copy not prepared mbufs beyond good ones */
223 if (k != num && k != 0) {
224 move_bad_mbufs(mb, dr, num, num - k);
232 * Start with processing inbound packet.
233 * This is common part for both tunnel and transport mode.
234 * Extract information that will be needed later from mbuf metadata and
235 * actual packet data:
236 * - mbuf for packet's last segment
237 * - length of the L2/L3 headers
238 * - esp tail structure
241 process_step1(struct rte_mbuf *mb, uint32_t tlen, struct rte_mbuf **ml,
242 struct esp_tail *espt, uint32_t *hlen)
244 const struct esp_tail *pt;
246 ml[0] = rte_pktmbuf_lastseg(mb);
247 hlen[0] = mb->l2_len + mb->l3_len;
248 pt = rte_pktmbuf_mtod_offset(ml[0], const struct esp_tail *,
249 ml[0]->data_len - tlen);
254 * packet checks for transport mode:
255 * - no reported IPsec related failures in ol_flags
256 * - tail length is valid
257 * - padding bytes are valid
259 static inline int32_t
260 trs_process_check(const struct rte_mbuf *mb, const struct rte_mbuf *ml,
261 struct esp_tail espt, uint32_t hlen, uint32_t tlen)
266 ofs = ml->data_len - tlen;
267 pd = rte_pktmbuf_mtod_offset(ml, const uint8_t *, ofs);
269 return ((mb->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) != 0 ||
270 ofs < 0 || tlen + hlen > mb->pkt_len ||
271 (espt.pad_len != 0 && memcmp(pd, esp_pad_bytes, espt.pad_len)));
275 * packet checks for tunnel mode:
276 * - same as for trasnport mode
277 * - esp tail next proto contains expected for that SA value
279 static inline int32_t
280 tun_process_check(const struct rte_mbuf *mb, struct rte_mbuf *ml,
281 struct esp_tail espt, uint32_t hlen, const uint32_t tlen, uint8_t proto)
283 return (trs_process_check(mb, ml, espt, hlen, tlen) ||
284 espt.next_proto != proto);
288 * step two for tunnel mode:
289 * - read SQN value (for future use)
290 * - cut of ICV, ESP tail and padding bytes
291 * - cut of ESP header and IV, also if needed - L2/L3 headers
292 * (controlled by *adj* value)
295 tun_process_step2(struct rte_mbuf *mb, struct rte_mbuf *ml, uint32_t hlen,
296 uint32_t adj, uint32_t tlen, uint32_t *sqn)
298 const struct rte_esp_hdr *ph;
301 ph = rte_pktmbuf_mtod_offset(mb, const struct rte_esp_hdr *, hlen);
304 /* cut of ICV, ESP tail and padding bytes */
305 ml->data_len -= tlen;
308 /* cut of L2/L3 headers, ESP header and IV */
309 return rte_pktmbuf_adj(mb, adj);
313 * step two for transport mode:
314 * - read SQN value (for future use)
315 * - cut of ICV, ESP tail and padding bytes
316 * - cut of ESP header and IV
317 * - move L2/L3 header to fill the gap after ESP header removal
320 trs_process_step2(struct rte_mbuf *mb, struct rte_mbuf *ml, uint32_t hlen,
321 uint32_t adj, uint32_t tlen, uint32_t *sqn)
325 /* get start of the packet before modifications */
326 op = rte_pktmbuf_mtod(mb, char *);
328 /* cut off ESP header and IV */
329 np = tun_process_step2(mb, ml, hlen, adj, tlen, sqn);
331 /* move header bytes to fill the gap after ESP header removal */
332 remove_esph(np, op, hlen);
337 * step three for transport mode:
338 * update mbuf metadata:
343 trs_process_step3(struct rte_mbuf *mb)
345 /* reset mbuf packet type */
346 mb->packet_type &= (RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK);
348 /* clear the PKT_RX_SEC_OFFLOAD flag if set */
349 mb->ol_flags &= ~PKT_RX_SEC_OFFLOAD;
353 * step three for tunnel mode:
354 * update mbuf metadata:
360 tun_process_step3(struct rte_mbuf *mb, uint64_t txof_msk, uint64_t txof_val)
362 /* reset mbuf metatdata: L2/L3 len, packet type */
363 mb->packet_type = RTE_PTYPE_UNKNOWN;
364 mb->tx_offload = (mb->tx_offload & txof_msk) | txof_val;
366 /* clear the PKT_RX_SEC_OFFLOAD flag if set */
367 mb->ol_flags &= ~PKT_RX_SEC_OFFLOAD;
372 * *process* function for tunnel packets
374 static inline uint16_t
375 tun_process(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],
376 uint32_t sqn[], uint32_t dr[], uint16_t num)
378 uint32_t adj, i, k, tl;
380 struct esp_tail espt[num];
381 struct rte_mbuf *ml[num];
383 const uint32_t tlen = sa->icv_len + sizeof(espt[0]);
384 const uint32_t cofs = sa->ctp.cipher.offset;
387 * to minimize stalls due to load latency,
388 * read mbufs metadata and esp tail first.
390 for (i = 0; i != num; i++)
391 process_step1(mb[i], tlen, &ml[i], &espt[i], &hl[i]);
394 for (i = 0; i != num; i++) {
397 tl = tlen + espt[i].pad_len;
399 /* check that packet is valid */
400 if (tun_process_check(mb[i], ml[i], espt[i], adj, tl,
403 /* modify packet's layout */
404 tun_process_step2(mb[i], ml[i], hl[i], adj,
406 /* update mbuf's metadata */
407 tun_process_step3(mb[i], sa->tx_offload.msk,
419 * *process* function for tunnel packets
421 static inline uint16_t
422 trs_process(const struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],
423 uint32_t sqn[], uint32_t dr[], uint16_t num)
426 uint32_t i, k, l2, tl;
428 struct esp_tail espt[num];
429 struct rte_mbuf *ml[num];
431 const uint32_t tlen = sa->icv_len + sizeof(espt[0]);
432 const uint32_t cofs = sa->ctp.cipher.offset;
435 * to minimize stalls due to load latency,
436 * read mbufs metadata and esp tail first.
438 for (i = 0; i != num; i++)
439 process_step1(mb[i], tlen, &ml[i], &espt[i], &hl[i]);
442 for (i = 0; i != num; i++) {
444 tl = tlen + espt[i].pad_len;
447 /* check that packet is valid */
448 if (trs_process_check(mb[i], ml[i], espt[i], hl[i] + cofs,
451 /* modify packet's layout */
452 np = trs_process_step2(mb[i], ml[i], hl[i], cofs, tl,
454 update_trs_l3hdr(sa, np + l2, mb[i]->pkt_len,
455 l2, hl[i] - l2, espt[i].next_proto);
457 /* update mbuf's metadata */
458 trs_process_step3(mb[i]);
468 * for group of ESP inbound packets perform SQN check and update.
470 static inline uint16_t
471 esp_inb_rsn_update(struct rte_ipsec_sa *sa, const uint32_t sqn[],
472 uint32_t dr[], uint16_t num)
475 struct replay_sqn *rsn;
477 /* replay not enabled */
478 if (sa->replay.win_sz == 0)
481 rsn = rsn_update_start(sa);
484 for (i = 0; i != num; i++) {
485 if (esn_inb_update_sqn(rsn, sa, rte_be_to_cpu_32(sqn[i])) == 0)
491 rsn_update_finish(sa, rsn);
496 * process group of ESP inbound packets.
498 static inline uint16_t
499 esp_inb_pkt_process(const struct rte_ipsec_session *ss,
500 struct rte_mbuf *mb[], uint16_t num, esp_inb_process_t process)
503 struct rte_ipsec_sa *sa;
509 /* process packets, extract seq numbers */
510 k = process(sa, mb, sqn, dr, num);
512 /* handle unprocessed mbufs */
513 if (k != num && k != 0)
514 move_bad_mbufs(mb, dr, num, num - k);
516 /* update SQN and replay winow */
517 n = esp_inb_rsn_update(sa, sqn, dr, k);
519 /* handle mbufs with wrong SQN */
520 if (n != k && n != 0)
521 move_bad_mbufs(mb, dr, k, k - n);
530 * process group of ESP inbound tunnel packets.
533 esp_inb_tun_pkt_process(const struct rte_ipsec_session *ss,
534 struct rte_mbuf *mb[], uint16_t num)
536 return esp_inb_pkt_process(ss, mb, num, tun_process);
540 * process group of ESP inbound transport packets.
543 esp_inb_trs_pkt_process(const struct rte_ipsec_session *ss,
544 struct rte_mbuf *mb[], uint16_t num)
546 return esp_inb_pkt_process(ss, mb, num, trs_process);