1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
9 #include <rte_cryptodev.h>
12 #include "ipsec_sqn.h"
18 #define MBUF_MAX_L2_LEN RTE_LEN2MASK(RTE_MBUF_L2_LEN_BITS, uint64_t)
19 #define MBUF_MAX_L3_LEN RTE_LEN2MASK(RTE_MBUF_L3_LEN_BITS, uint64_t)
21 /* some helper structures */
23 struct rte_crypto_auth_xform *auth;
24 struct rte_crypto_cipher_xform *cipher;
25 struct rte_crypto_aead_xform *aead;
29 * helper routine, fills internal crypto_xform structure.
32 fill_crypto_xform(struct crypto_xform *xform, uint64_t type,
33 const struct rte_ipsec_sa_prm *prm)
35 struct rte_crypto_sym_xform *xf, *xfn;
37 memset(xform, 0, sizeof(*xform));
39 xf = prm->crypto_xform;
45 /* for AEAD just one xform required */
46 if (xf->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
49 xform->aead = &xf->aead;
51 * CIPHER+AUTH xforms are expected in strict order,
52 * depending on SA direction:
53 * inbound: AUTH+CIPHER
54 * outbound: CIPHER+AUTH
56 } else if ((type & RTE_IPSEC_SATP_DIR_MASK) == RTE_IPSEC_SATP_DIR_IB) {
58 /* wrong order or no cipher */
59 if (xfn == NULL || xf->type != RTE_CRYPTO_SYM_XFORM_AUTH ||
60 xfn->type != RTE_CRYPTO_SYM_XFORM_CIPHER)
63 xform->auth = &xf->auth;
64 xform->cipher = &xfn->cipher;
68 /* wrong order or no auth */
69 if (xfn == NULL || xf->type != RTE_CRYPTO_SYM_XFORM_CIPHER ||
70 xfn->type != RTE_CRYPTO_SYM_XFORM_AUTH)
73 xform->cipher = &xf->cipher;
74 xform->auth = &xfn->auth;
81 rte_ipsec_sa_type(const struct rte_ipsec_sa *sa)
87 * Based on number of buckets calculated required size for the
88 * structure that holds replay window and sequence number (RSN) information.
91 rsn_size(uint32_t nb_bucket)
94 struct replay_sqn *rsn;
96 sz = sizeof(*rsn) + nb_bucket * sizeof(rsn->window[0]);
97 sz = RTE_ALIGN_CEIL(sz, RTE_CACHE_LINE_SIZE);
102 * for given size, calculate required number of buckets.
105 replay_num_bucket(uint32_t wsz)
109 nb = rte_align32pow2(RTE_ALIGN_MUL_CEIL(wsz, WINDOW_BUCKET_SIZE) /
111 nb = RTE_MAX(nb, (uint32_t)WINDOW_BUCKET_MIN);
117 ipsec_sa_size(uint64_t type, uint32_t *wnd_sz, uint32_t *nb_bucket)
124 if ((type & RTE_IPSEC_SATP_DIR_MASK) == RTE_IPSEC_SATP_DIR_IB) {
127 * RFC 4303 recommends 64 as minimum window size.
128 * there is no point to use ESN mode without SQN window,
129 * so make sure we have at least 64 window when ESN is enalbed.
131 wsz = ((type & RTE_IPSEC_SATP_ESN_MASK) ==
132 RTE_IPSEC_SATP_ESN_DISABLE) ?
133 wsz : RTE_MAX(wsz, (uint32_t)WINDOW_BUCKET_SIZE);
135 n = replay_num_bucket(wsz);
138 if (n > WINDOW_BUCKET_MAX)
145 if ((type & RTE_IPSEC_SATP_SQN_MASK) == RTE_IPSEC_SATP_SQN_ATOM)
146 sz *= REPLAY_SQN_NUM;
148 sz += sizeof(struct rte_ipsec_sa);
153 rte_ipsec_sa_fini(struct rte_ipsec_sa *sa)
155 memset(sa, 0, sa->size);
159 * Determine expected SA type based on input parameters.
162 fill_sa_type(const struct rte_ipsec_sa_prm *prm, uint64_t *type)
168 if (prm->ipsec_xform.proto == RTE_SECURITY_IPSEC_SA_PROTO_AH)
169 tp |= RTE_IPSEC_SATP_PROTO_AH;
170 else if (prm->ipsec_xform.proto == RTE_SECURITY_IPSEC_SA_PROTO_ESP)
171 tp |= RTE_IPSEC_SATP_PROTO_ESP;
175 if (prm->ipsec_xform.direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS)
176 tp |= RTE_IPSEC_SATP_DIR_OB;
177 else if (prm->ipsec_xform.direction ==
178 RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
179 tp |= RTE_IPSEC_SATP_DIR_IB;
183 if (prm->ipsec_xform.mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL) {
184 if (prm->ipsec_xform.tunnel.type ==
185 RTE_SECURITY_IPSEC_TUNNEL_IPV4)
186 tp |= RTE_IPSEC_SATP_MODE_TUNLV4;
187 else if (prm->ipsec_xform.tunnel.type ==
188 RTE_SECURITY_IPSEC_TUNNEL_IPV6)
189 tp |= RTE_IPSEC_SATP_MODE_TUNLV6;
193 if (prm->tun.next_proto == IPPROTO_IPIP)
194 tp |= RTE_IPSEC_SATP_IPV4;
195 else if (prm->tun.next_proto == IPPROTO_IPV6)
196 tp |= RTE_IPSEC_SATP_IPV6;
199 } else if (prm->ipsec_xform.mode ==
200 RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT) {
201 tp |= RTE_IPSEC_SATP_MODE_TRANS;
202 if (prm->trs.proto == IPPROTO_IPIP)
203 tp |= RTE_IPSEC_SATP_IPV4;
204 else if (prm->trs.proto == IPPROTO_IPV6)
205 tp |= RTE_IPSEC_SATP_IPV6;
211 /* check for ESN flag */
212 if (prm->ipsec_xform.options.esn == 0)
213 tp |= RTE_IPSEC_SATP_ESN_DISABLE;
215 tp |= RTE_IPSEC_SATP_ESN_ENABLE;
217 /* check for ECN flag */
218 if (prm->ipsec_xform.options.ecn == 0)
219 tp |= RTE_IPSEC_SATP_ECN_DISABLE;
221 tp |= RTE_IPSEC_SATP_ECN_ENABLE;
223 /* check for DSCP flag */
224 if (prm->ipsec_xform.options.copy_dscp == 0)
225 tp |= RTE_IPSEC_SATP_DSCP_DISABLE;
227 tp |= RTE_IPSEC_SATP_DSCP_ENABLE;
229 /* interpret flags */
230 if (prm->flags & RTE_IPSEC_SAFLAG_SQN_ATOM)
231 tp |= RTE_IPSEC_SATP_SQN_ATOM;
233 tp |= RTE_IPSEC_SATP_SQN_RAW;
240 * Init ESP inbound specific things.
243 esp_inb_init(struct rte_ipsec_sa *sa)
245 /* these params may differ with new algorithms support */
246 sa->ctp.auth.offset = 0;
247 sa->ctp.auth.length = sa->icv_len - sa->sqh_len;
248 sa->ctp.cipher.offset = sizeof(struct rte_esp_hdr) + sa->iv_len;
249 sa->ctp.cipher.length = sa->icv_len + sa->ctp.cipher.offset;
253 * Init ESP inbound tunnel specific things.
256 esp_inb_tun_init(struct rte_ipsec_sa *sa, const struct rte_ipsec_sa_prm *prm)
258 sa->proto = prm->tun.next_proto;
263 * Init ESP outbound specific things.
266 esp_outb_init(struct rte_ipsec_sa *sa, uint32_t hlen)
270 sa->sqn.outb.raw = 1;
272 /* these params may differ with new algorithms support */
273 sa->ctp.auth.offset = hlen;
274 sa->ctp.auth.length = sizeof(struct rte_esp_hdr) +
275 sa->iv_len + sa->sqh_len;
277 algo_type = sa->algo_type;
280 case ALGO_TYPE_AES_GCM:
281 case ALGO_TYPE_AES_CTR:
283 sa->ctp.cipher.offset = hlen + sizeof(struct rte_esp_hdr) +
285 sa->ctp.cipher.length = 0;
287 case ALGO_TYPE_AES_CBC:
288 case ALGO_TYPE_3DES_CBC:
289 sa->ctp.cipher.offset = sa->hdr_len +
290 sizeof(struct rte_esp_hdr);
291 sa->ctp.cipher.length = sa->iv_len;
297 * Init ESP outbound tunnel specific things.
300 esp_outb_tun_init(struct rte_ipsec_sa *sa, const struct rte_ipsec_sa_prm *prm)
302 sa->proto = prm->tun.next_proto;
303 sa->hdr_len = prm->tun.hdr_len;
304 sa->hdr_l3_off = prm->tun.hdr_l3_off;
306 /* update l2_len and l3_len fields for outbound mbuf */
307 sa->tx_offload.val = rte_mbuf_tx_offload(sa->hdr_l3_off,
308 sa->hdr_len - sa->hdr_l3_off, 0, 0, 0, 0, 0);
310 memcpy(sa->hdr, prm->tun.hdr, sa->hdr_len);
312 esp_outb_init(sa, sa->hdr_len);
316 * helper function, init SA structure.
319 esp_sa_init(struct rte_ipsec_sa *sa, const struct rte_ipsec_sa_prm *prm,
320 const struct crypto_xform *cxf)
322 static const uint64_t msk = RTE_IPSEC_SATP_DIR_MASK |
323 RTE_IPSEC_SATP_MODE_MASK;
325 if (prm->ipsec_xform.options.ecn)
326 sa->tos_mask |= RTE_IPV4_HDR_ECN_MASK;
328 if (prm->ipsec_xform.options.copy_dscp)
329 sa->tos_mask |= RTE_IPV4_HDR_DSCP_MASK;
331 if (cxf->aead != NULL) {
332 switch (cxf->aead->algo) {
333 case RTE_CRYPTO_AEAD_AES_GCM:
335 sa->aad_len = sizeof(struct aead_gcm_aad);
336 sa->icv_len = cxf->aead->digest_length;
337 sa->iv_ofs = cxf->aead->iv.offset;
338 sa->iv_len = sizeof(uint64_t);
339 sa->pad_align = IPSEC_PAD_AES_GCM;
340 sa->algo_type = ALGO_TYPE_AES_GCM;
346 sa->icv_len = cxf->auth->digest_length;
347 sa->iv_ofs = cxf->cipher->iv.offset;
348 sa->sqh_len = IS_ESN(sa) ? sizeof(uint32_t) : 0;
350 switch (cxf->cipher->algo) {
351 case RTE_CRYPTO_CIPHER_NULL:
352 sa->pad_align = IPSEC_PAD_NULL;
354 sa->algo_type = ALGO_TYPE_NULL;
357 case RTE_CRYPTO_CIPHER_AES_CBC:
358 sa->pad_align = IPSEC_PAD_AES_CBC;
359 sa->iv_len = IPSEC_MAX_IV_SIZE;
360 sa->algo_type = ALGO_TYPE_AES_CBC;
363 case RTE_CRYPTO_CIPHER_AES_CTR:
365 sa->pad_align = IPSEC_PAD_AES_CTR;
366 sa->iv_len = IPSEC_AES_CTR_IV_SIZE;
367 sa->algo_type = ALGO_TYPE_AES_CTR;
370 case RTE_CRYPTO_CIPHER_3DES_CBC:
372 sa->pad_align = IPSEC_PAD_3DES_CBC;
373 sa->iv_len = IPSEC_3DES_IV_SIZE;
374 sa->algo_type = ALGO_TYPE_3DES_CBC;
382 sa->udata = prm->userdata;
383 sa->spi = rte_cpu_to_be_32(prm->ipsec_xform.spi);
384 sa->salt = prm->ipsec_xform.salt;
386 /* preserve all values except l2_len and l3_len */
388 ~rte_mbuf_tx_offload(MBUF_MAX_L2_LEN, MBUF_MAX_L3_LEN,
391 switch (sa->type & msk) {
392 case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV4):
393 case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV6):
394 esp_inb_tun_init(sa, prm);
396 case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TRANS):
399 case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV4):
400 case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV6):
401 esp_outb_tun_init(sa, prm);
403 case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TRANS):
404 esp_outb_init(sa, 0);
412 * helper function, init SA replay structure.
415 fill_sa_replay(struct rte_ipsec_sa *sa, uint32_t wnd_sz, uint32_t nb_bucket)
417 sa->replay.win_sz = wnd_sz;
418 sa->replay.nb_bucket = nb_bucket;
419 sa->replay.bucket_index_mask = nb_bucket - 1;
420 sa->sqn.inb.rsn[0] = (struct replay_sqn *)(sa + 1);
421 if ((sa->type & RTE_IPSEC_SATP_SQN_MASK) == RTE_IPSEC_SATP_SQN_ATOM)
422 sa->sqn.inb.rsn[1] = (struct replay_sqn *)
423 ((uintptr_t)sa->sqn.inb.rsn[0] + rsn_size(nb_bucket));
427 rte_ipsec_sa_size(const struct rte_ipsec_sa_prm *prm)
436 /* determine SA type */
437 rc = fill_sa_type(prm, &type);
441 /* determine required size */
442 wsz = prm->replay_win_sz;
443 return ipsec_sa_size(type, &wsz, &nb);
447 rte_ipsec_sa_init(struct rte_ipsec_sa *sa, const struct rte_ipsec_sa_prm *prm,
453 struct crypto_xform cxf;
455 if (sa == NULL || prm == NULL)
458 /* determine SA type */
459 rc = fill_sa_type(prm, &type);
463 /* determine required size */
464 wsz = prm->replay_win_sz;
465 sz = ipsec_sa_size(type, &wsz, &nb);
468 else if (size < (uint32_t)sz)
471 /* only esp is supported right now */
472 if (prm->ipsec_xform.proto != RTE_SECURITY_IPSEC_SA_PROTO_ESP)
475 if (prm->ipsec_xform.mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
476 prm->tun.hdr_len > sizeof(sa->hdr))
479 rc = fill_crypto_xform(&cxf, type, prm);
489 /* check for ESN flag */
490 sa->sqn_mask = (prm->ipsec_xform.options.esn == 0) ?
491 UINT32_MAX : UINT64_MAX;
493 rc = esp_sa_init(sa, prm, &cxf);
495 rte_ipsec_sa_fini(sa);
497 /* fill replay window related fields */
499 fill_sa_replay(sa, wsz, nb);
505 * setup crypto ops for LOOKASIDE_PROTO type of devices.
508 lksd_proto_cop_prepare(const struct rte_ipsec_session *ss,
509 struct rte_mbuf *mb[], struct rte_crypto_op *cop[], uint16_t num)
512 struct rte_crypto_sym_op *sop;
514 for (i = 0; i != num; i++) {
516 cop[i]->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
517 cop[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
518 cop[i]->sess_type = RTE_CRYPTO_OP_SECURITY_SESSION;
520 __rte_security_attach_session(sop, ss->security.ses);
525 * setup packets and crypto ops for LOOKASIDE_PROTO type of devices.
526 * Note that for LOOKASIDE_PROTO all packet modifications will be
527 * performed by PMD/HW.
528 * SW has only to prepare crypto op.
531 lksd_proto_prepare(const struct rte_ipsec_session *ss,
532 struct rte_mbuf *mb[], struct rte_crypto_op *cop[], uint16_t num)
534 lksd_proto_cop_prepare(ss, mb, cop, num);
539 * simplest pkt process routine:
540 * all actual processing is already done by HW/PMD,
541 * just check mbuf ol_flags.
543 * - inbound for RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL
544 * - inbound/outbound for RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL
545 * - outbound for RTE_SECURITY_ACTION_TYPE_NONE when ESN is disabled
548 pkt_flag_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
557 for (i = 0; i != num; i++) {
558 if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0)
564 /* handle unprocessed mbufs */
568 move_bad_mbufs(mb, dr, num, num - k);
575 * Select packet processing function for session on LOOKASIDE_NONE
579 lksd_none_pkt_func_select(const struct rte_ipsec_sa *sa,
580 struct rte_ipsec_sa_pkt_func *pf)
584 static const uint64_t msk = RTE_IPSEC_SATP_DIR_MASK |
585 RTE_IPSEC_SATP_MODE_MASK;
588 switch (sa->type & msk) {
589 case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV4):
590 case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV6):
591 pf->prepare = esp_inb_pkt_prepare;
592 pf->process = esp_inb_tun_pkt_process;
594 case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TRANS):
595 pf->prepare = esp_inb_pkt_prepare;
596 pf->process = esp_inb_trs_pkt_process;
598 case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV4):
599 case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV6):
600 pf->prepare = esp_outb_tun_prepare;
601 pf->process = (sa->sqh_len != 0) ?
602 esp_outb_sqh_process : pkt_flag_process;
604 case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TRANS):
605 pf->prepare = esp_outb_trs_prepare;
606 pf->process = (sa->sqh_len != 0) ?
607 esp_outb_sqh_process : pkt_flag_process;
617 * Select packet processing function for session on INLINE_CRYPTO
621 inline_crypto_pkt_func_select(const struct rte_ipsec_sa *sa,
622 struct rte_ipsec_sa_pkt_func *pf)
626 static const uint64_t msk = RTE_IPSEC_SATP_DIR_MASK |
627 RTE_IPSEC_SATP_MODE_MASK;
630 switch (sa->type & msk) {
631 case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV4):
632 case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TUNLV6):
633 pf->process = inline_inb_tun_pkt_process;
635 case (RTE_IPSEC_SATP_DIR_IB | RTE_IPSEC_SATP_MODE_TRANS):
636 pf->process = inline_inb_trs_pkt_process;
638 case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV4):
639 case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TUNLV6):
640 pf->process = inline_outb_tun_pkt_process;
642 case (RTE_IPSEC_SATP_DIR_OB | RTE_IPSEC_SATP_MODE_TRANS):
643 pf->process = inline_outb_trs_pkt_process;
653 * Select packet processing function for given session based on SA parameters
654 * and type of associated with the session device.
657 ipsec_sa_pkt_func_select(const struct rte_ipsec_session *ss,
658 const struct rte_ipsec_sa *sa, struct rte_ipsec_sa_pkt_func *pf)
663 pf[0] = (struct rte_ipsec_sa_pkt_func) { 0 };
666 case RTE_SECURITY_ACTION_TYPE_NONE:
667 rc = lksd_none_pkt_func_select(sa, pf);
669 case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
670 rc = inline_crypto_pkt_func_select(sa, pf);
672 case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
673 if ((sa->type & RTE_IPSEC_SATP_DIR_MASK) ==
674 RTE_IPSEC_SATP_DIR_IB)
675 pf->process = pkt_flag_process;
677 pf->process = inline_proto_outb_pkt_process;
679 case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
680 pf->prepare = lksd_proto_prepare;
681 pf->process = pkt_flag_process;