-static inline void
-mbuf_bulk_copy(struct rte_mbuf *dst[], struct rte_mbuf * const src[],
- uint32_t num)
-{
- uint32_t i;
-
- for (i = 0; i != num; i++)
- dst[i] = src[i];
-}
-
-/*
- * setup crypto ops for LOOKASIDE_NONE (pure crypto) type of devices.
- */
-static inline void
-lksd_none_cop_prepare(const struct rte_ipsec_session *ss,
- struct rte_mbuf *mb[], struct rte_crypto_op *cop[], uint16_t num)
-{
- uint32_t i;
- struct rte_crypto_sym_op *sop;
-
- for (i = 0; i != num; i++) {
- sop = cop[i]->sym;
- cop[i]->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
- cop[i]->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
- cop[i]->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
- sop->m_src = mb[i];
- __rte_crypto_sym_op_attach_sym_session(sop, ss->crypto.ses);
- }
-}
-
-/*
- * setup crypto op and crypto sym op for ESP outbound packet.
- */
-static inline void
-esp_outb_cop_prepare(struct rte_crypto_op *cop,
- const struct rte_ipsec_sa *sa, const uint64_t ivp[IPSEC_MAX_IV_QWORD],
- const union sym_op_data *icv, uint32_t hlen, uint32_t plen)
-{
- struct rte_crypto_sym_op *sop;
- struct aead_gcm_iv *gcm;
- struct aesctr_cnt_blk *ctr;
- uint8_t algo_type = sa->algo_type;
-
- /* fill sym op fields */
- sop = cop->sym;
-
- switch (algo_type) {
- case ALGO_TYPE_AES_CBC:
- /* Cipher-Auth (AES-CBC *) case */
- case ALGO_TYPE_3DES_CBC:
- /* Cipher-Auth (3DES-CBC *) case */
- case ALGO_TYPE_NULL:
- /* NULL case */
- sop->cipher.data.offset = sa->ctp.cipher.offset + hlen;
- sop->cipher.data.length = sa->ctp.cipher.length + plen;
- sop->auth.data.offset = sa->ctp.auth.offset + hlen;
- sop->auth.data.length = sa->ctp.auth.length + plen;
- sop->auth.digest.data = icv->va;
- sop->auth.digest.phys_addr = icv->pa;
- break;
- case ALGO_TYPE_AES_GCM:
- /* AEAD (AES_GCM) case */
- sop->aead.data.offset = sa->ctp.cipher.offset + hlen;
- sop->aead.data.length = sa->ctp.cipher.length + plen;
- sop->aead.digest.data = icv->va;
- sop->aead.digest.phys_addr = icv->pa;
- sop->aead.aad.data = icv->va + sa->icv_len;
- sop->aead.aad.phys_addr = icv->pa + sa->icv_len;
-
- /* fill AAD IV (located inside crypto op) */
- gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *,
- sa->iv_ofs);
- aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
- break;
- case ALGO_TYPE_AES_CTR:
- /* Cipher-Auth (AES-CTR *) case */
- sop->cipher.data.offset = sa->ctp.cipher.offset + hlen;
- sop->cipher.data.length = sa->ctp.cipher.length + plen;
- sop->auth.data.offset = sa->ctp.auth.offset + hlen;
- sop->auth.data.length = sa->ctp.auth.length + plen;
- sop->auth.digest.data = icv->va;
- sop->auth.digest.phys_addr = icv->pa;
-
- ctr = rte_crypto_op_ctod_offset(cop, struct aesctr_cnt_blk *,
- sa->iv_ofs);
- aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);
- break;
- default:
- break;
- }
-}
-
-/*
- * setup/update packet data and metadata for ESP outbound tunnel case.
- */
-static inline int32_t
-esp_outb_tun_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
- const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb,
- union sym_op_data *icv)
-{
- uint32_t clen, hlen, l2len, pdlen, pdofs, plen, tlen;
- struct rte_mbuf *ml;
- struct esp_hdr *esph;
- struct esp_tail *espt;
- char *ph, *pt;
- uint64_t *iv;
-
- /* calculate extra header space required */
- hlen = sa->hdr_len + sa->iv_len + sizeof(*esph);
-
- /* size of ipsec protected data */
- l2len = mb->l2_len;
- plen = mb->pkt_len - l2len;
-
- /* number of bytes to encrypt */
- clen = plen + sizeof(*espt);
- clen = RTE_ALIGN_CEIL(clen, sa->pad_align);
-
- /* pad length + esp tail */
- pdlen = clen - plen;
- tlen = pdlen + sa->icv_len;
-
- /* do append and prepend */
- ml = rte_pktmbuf_lastseg(mb);
- if (tlen + sa->sqh_len + sa->aad_len > rte_pktmbuf_tailroom(ml))
- return -ENOSPC;
-
- /* prepend header */
- ph = rte_pktmbuf_prepend(mb, hlen - l2len);
- if (ph == NULL)
- return -ENOSPC;
-
- /* append tail */
- pdofs = ml->data_len;
- ml->data_len += tlen;
- mb->pkt_len += tlen;
- pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs);
-
- /* update pkt l2/l3 len */
- mb->tx_offload = (mb->tx_offload & sa->tx_offload.msk) |
- sa->tx_offload.val;
-
- /* copy tunnel pkt header */
- rte_memcpy(ph, sa->hdr, sa->hdr_len);
-
- /* update original and new ip header fields */
- update_tun_l3hdr(sa, ph + sa->hdr_l3_off, mb->pkt_len, sa->hdr_l3_off,
- sqn_low16(sqc));
-
- /* update spi, seqn and iv */
- esph = (struct esp_hdr *)(ph + sa->hdr_len);
- iv = (uint64_t *)(esph + 1);
- copy_iv(iv, ivp, sa->iv_len);
-
- esph->spi = sa->spi;
- esph->seq = sqn_low32(sqc);
-
- /* offset for ICV */
- pdofs += pdlen + sa->sqh_len;
-
- /* pad length */
- pdlen -= sizeof(*espt);
-
- /* copy padding data */
- rte_memcpy(pt, esp_pad_bytes, pdlen);
-
- /* update esp trailer */
- espt = (struct esp_tail *)(pt + pdlen);
- espt->pad_len = pdlen;
- espt->next_proto = sa->proto;
-
- icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs);
- icv->pa = rte_pktmbuf_iova_offset(ml, pdofs);
-
- return clen;
-}
-
-/*
- * for pure cryptodev (lookaside none) depending on SA settings,
- * we might have to write some extra data to the packet.
- */
-static inline void
-outb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
- const union sym_op_data *icv)
-{
- uint32_t *psqh;
- struct aead_gcm_aad *aad;
- uint8_t algo_type = sa->algo_type;
-
- /* insert SQN.hi between ESP trailer and ICV */
- if (sa->sqh_len != 0) {
- psqh = (uint32_t *)(icv->va - sa->sqh_len);
- psqh[0] = sqn_hi32(sqc);
- }
-
- /*
- * fill IV and AAD fields, if any (aad fields are placed after icv),
- * right now we support only one AEAD algorithm: AES-GCM .
- */
- if (algo_type == ALGO_TYPE_AES_GCM) {
- aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);
- aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa));
- }
-}
-
-/*
- * setup/update packets and crypto ops for ESP outbound tunnel case.
- */
-static uint16_t
-outb_tun_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
- struct rte_crypto_op *cop[], uint16_t num)
-{
- int32_t rc;
- uint32_t i, k, n;
- uint64_t sqn;
- rte_be64_t sqc;
- struct rte_ipsec_sa *sa;
- union sym_op_data icv;
- uint64_t iv[IPSEC_MAX_IV_QWORD];
- struct rte_mbuf *dr[num];
-
- sa = ss->sa;
-
- n = num;
- sqn = esn_outb_update_sqn(sa, &n);
- if (n != num)
- rte_errno = EOVERFLOW;
-
- k = 0;
- for (i = 0; i != n; i++) {
-
- sqc = rte_cpu_to_be_64(sqn + i);
- gen_iv(iv, sqc);
-
- /* try to update the packet itself */
- rc = esp_outb_tun_pkt_prepare(sa, sqc, iv, mb[i], &icv);
-
- /* success, setup crypto op */
- if (rc >= 0) {
- mb[k] = mb[i];
- outb_pkt_xprepare(sa, sqc, &icv);
- esp_outb_cop_prepare(cop[k], sa, iv, &icv, 0, rc);
- k++;
- /* failure, put packet into the death-row */
- } else {
- dr[i - k] = mb[i];
- rte_errno = -rc;
- }
- }
-
- /* update cops */
- lksd_none_cop_prepare(ss, mb, cop, k);
-
- /* copy not prepared mbufs beyond good ones */
- if (k != n && k != 0)
- mbuf_bulk_copy(mb + k, dr, n - k);
-
- return k;
-}
-
-/*
- * setup/update packet data and metadata for ESP outbound transport case.
- */
-static inline int32_t
-esp_outb_trs_pkt_prepare(struct rte_ipsec_sa *sa, rte_be64_t sqc,
- const uint64_t ivp[IPSEC_MAX_IV_QWORD], struct rte_mbuf *mb,
- uint32_t l2len, uint32_t l3len, union sym_op_data *icv)
-{
- uint8_t np;
- uint32_t clen, hlen, pdlen, pdofs, plen, tlen, uhlen;
- struct rte_mbuf *ml;
- struct esp_hdr *esph;
- struct esp_tail *espt;
- char *ph, *pt;
- uint64_t *iv;
-
- uhlen = l2len + l3len;
- plen = mb->pkt_len - uhlen;
-
- /* calculate extra header space required */
- hlen = sa->iv_len + sizeof(*esph);
-
- /* number of bytes to encrypt */
- clen = plen + sizeof(*espt);
- clen = RTE_ALIGN_CEIL(clen, sa->pad_align);
-
- /* pad length + esp tail */
- pdlen = clen - plen;
- tlen = pdlen + sa->icv_len;
-
- /* do append and insert */
- ml = rte_pktmbuf_lastseg(mb);
- if (tlen + sa->sqh_len + sa->aad_len > rte_pktmbuf_tailroom(ml))
- return -ENOSPC;
-
- /* prepend space for ESP header */
- ph = rte_pktmbuf_prepend(mb, hlen);
- if (ph == NULL)
- return -ENOSPC;
-
- /* append tail */
- pdofs = ml->data_len;
- ml->data_len += tlen;
- mb->pkt_len += tlen;
- pt = rte_pktmbuf_mtod_offset(ml, typeof(pt), pdofs);
-
- /* shift L2/L3 headers */
- insert_esph(ph, ph + hlen, uhlen);
-
- /* update ip header fields */
- np = update_trs_l3hdr(sa, ph + l2len, mb->pkt_len, l2len, l3len,
- IPPROTO_ESP);
-
- /* update spi, seqn and iv */
- esph = (struct esp_hdr *)(ph + uhlen);
- iv = (uint64_t *)(esph + 1);
- copy_iv(iv, ivp, sa->iv_len);
-
- esph->spi = sa->spi;
- esph->seq = sqn_low32(sqc);
-
- /* offset for ICV */
- pdofs += pdlen + sa->sqh_len;
-
- /* pad length */
- pdlen -= sizeof(*espt);
-
- /* copy padding data */
- rte_memcpy(pt, esp_pad_bytes, pdlen);
-
- /* update esp trailer */
- espt = (struct esp_tail *)(pt + pdlen);
- espt->pad_len = pdlen;
- espt->next_proto = np;
-
- icv->va = rte_pktmbuf_mtod_offset(ml, void *, pdofs);
- icv->pa = rte_pktmbuf_iova_offset(ml, pdofs);
-
- return clen;
-}
-
-/*
- * setup/update packets and crypto ops for ESP outbound transport case.
- */
-static uint16_t
-outb_trs_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
- struct rte_crypto_op *cop[], uint16_t num)
-{
- int32_t rc;
- uint32_t i, k, n, l2, l3;
- uint64_t sqn;
- rte_be64_t sqc;
- struct rte_ipsec_sa *sa;
- union sym_op_data icv;
- uint64_t iv[IPSEC_MAX_IV_QWORD];
- struct rte_mbuf *dr[num];
-
- sa = ss->sa;
-
- n = num;
- sqn = esn_outb_update_sqn(sa, &n);
- if (n != num)
- rte_errno = EOVERFLOW;
-
- k = 0;
- for (i = 0; i != n; i++) {
-
- l2 = mb[i]->l2_len;
- l3 = mb[i]->l3_len;
-
- sqc = rte_cpu_to_be_64(sqn + i);
- gen_iv(iv, sqc);
-
- /* try to update the packet itself */
- rc = esp_outb_trs_pkt_prepare(sa, sqc, iv, mb[i],
- l2, l3, &icv);
-
- /* success, setup crypto op */
- if (rc >= 0) {
- mb[k] = mb[i];
- outb_pkt_xprepare(sa, sqc, &icv);
- esp_outb_cop_prepare(cop[k], sa, iv, &icv, l2 + l3, rc);
- k++;
- /* failure, put packet into the death-row */
- } else {
- dr[i - k] = mb[i];
- rte_errno = -rc;
- }
- }
-
- /* update cops */
- lksd_none_cop_prepare(ss, mb, cop, k);
-
- /* copy not prepared mbufs beyond good ones */
- if (k != n && k != 0)
- mbuf_bulk_copy(mb + k, dr, n - k);
-
- return k;
-}
-
-/*
- * setup crypto op and crypto sym op for ESP inbound tunnel packet.
- */
-static inline int32_t
-esp_inb_tun_cop_prepare(struct rte_crypto_op *cop,
- const struct rte_ipsec_sa *sa, struct rte_mbuf *mb,
- const union sym_op_data *icv, uint32_t pofs, uint32_t plen)
-{
- struct rte_crypto_sym_op *sop;
- struct aead_gcm_iv *gcm;
- struct aesctr_cnt_blk *ctr;
- uint64_t *ivc, *ivp;
- uint32_t clen;
- uint8_t algo_type = sa->algo_type;
-
- clen = plen - sa->ctp.cipher.length;
- if ((int32_t)clen < 0 || (clen & (sa->pad_align - 1)) != 0)
- return -EINVAL;
-
- /* fill sym op fields */
- sop = cop->sym;
-
- switch (algo_type) {
- case ALGO_TYPE_AES_GCM:
- sop->aead.data.offset = pofs + sa->ctp.cipher.offset;
- sop->aead.data.length = clen;
- sop->aead.digest.data = icv->va;
- sop->aead.digest.phys_addr = icv->pa;
- sop->aead.aad.data = icv->va + sa->icv_len;
- sop->aead.aad.phys_addr = icv->pa + sa->icv_len;
-
- /* fill AAD IV (located inside crypto op) */
- gcm = rte_crypto_op_ctod_offset(cop, struct aead_gcm_iv *,
- sa->iv_ofs);
- ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *,
- pofs + sizeof(struct esp_hdr));
- aead_gcm_iv_fill(gcm, ivp[0], sa->salt);
- break;
- case ALGO_TYPE_AES_CBC:
- case ALGO_TYPE_3DES_CBC:
- sop->cipher.data.offset = pofs + sa->ctp.cipher.offset;
- sop->cipher.data.length = clen;
- sop->auth.data.offset = pofs + sa->ctp.auth.offset;
- sop->auth.data.length = plen - sa->ctp.auth.length;
- sop->auth.digest.data = icv->va;
- sop->auth.digest.phys_addr = icv->pa;
-
- /* copy iv from the input packet to the cop */
- ivc = rte_crypto_op_ctod_offset(cop, uint64_t *, sa->iv_ofs);
- ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *,
- pofs + sizeof(struct esp_hdr));
- copy_iv(ivc, ivp, sa->iv_len);
- break;
- case ALGO_TYPE_AES_CTR:
- sop->cipher.data.offset = pofs + sa->ctp.cipher.offset;
- sop->cipher.data.length = clen;
- sop->auth.data.offset = pofs + sa->ctp.auth.offset;
- sop->auth.data.length = plen - sa->ctp.auth.length;
- sop->auth.digest.data = icv->va;
- sop->auth.digest.phys_addr = icv->pa;
-
- /* copy iv from the input packet to the cop */
- ctr = rte_crypto_op_ctod_offset(cop, struct aesctr_cnt_blk *,
- sa->iv_ofs);
- ivp = rte_pktmbuf_mtod_offset(mb, uint64_t *,
- pofs + sizeof(struct esp_hdr));
- aes_ctr_cnt_blk_fill(ctr, ivp[0], sa->salt);
- break;
- case ALGO_TYPE_NULL:
- sop->cipher.data.offset = pofs + sa->ctp.cipher.offset;
- sop->cipher.data.length = clen;
- sop->auth.data.offset = pofs + sa->ctp.auth.offset;
- sop->auth.data.length = plen - sa->ctp.auth.length;
- sop->auth.digest.data = icv->va;
- sop->auth.digest.phys_addr = icv->pa;
- break;
-
- default:
- return -EINVAL;
- }
-
- return 0;
-}
-
-/*
- * for pure cryptodev (lookaside none) depending on SA settings,
- * we might have to write some extra data to the packet.
- */
-static inline void
-inb_pkt_xprepare(const struct rte_ipsec_sa *sa, rte_be64_t sqc,
- const union sym_op_data *icv)
-{
- struct aead_gcm_aad *aad;
-
- /* insert SQN.hi between ESP trailer and ICV */
- if (sa->sqh_len != 0)
- insert_sqh(sqn_hi32(sqc), icv->va, sa->icv_len);
-
- /*
- * fill AAD fields, if any (aad fields are placed after icv),
- * right now we support only one AEAD algorithm: AES-GCM.
- */
- if (sa->aad_len != 0) {
- aad = (struct aead_gcm_aad *)(icv->va + sa->icv_len);
- aead_gcm_aad_fill(aad, sa->spi, sqc, IS_ESN(sa));
- }
-}
-
-/*
- * setup/update packet data and metadata for ESP inbound tunnel case.
- */
-static inline int32_t
-esp_inb_tun_pkt_prepare(const struct rte_ipsec_sa *sa,
- const struct replay_sqn *rsn, struct rte_mbuf *mb,
- uint32_t hlen, union sym_op_data *icv)
-{
- int32_t rc;
- uint64_t sqn;
- uint32_t icv_ofs, plen;
- struct rte_mbuf *ml;
- struct esp_hdr *esph;
-
- esph = rte_pktmbuf_mtod_offset(mb, struct esp_hdr *, hlen);
-
- /*
- * retrieve and reconstruct SQN, then check it, then
- * convert it back into network byte order.
- */
- sqn = rte_be_to_cpu_32(esph->seq);
- if (IS_ESN(sa))
- sqn = reconstruct_esn(rsn->sqn, sqn, sa->replay.win_sz);
-
- rc = esn_inb_check_sqn(rsn, sa, sqn);
- if (rc != 0)
- return rc;
-
- sqn = rte_cpu_to_be_64(sqn);
-
- /* start packet manipulation */
- plen = mb->pkt_len;
- plen = plen - hlen;
-
- ml = rte_pktmbuf_lastseg(mb);
- icv_ofs = ml->data_len - sa->icv_len + sa->sqh_len;
-
- /* we have to allocate space for AAD somewhere,
- * right now - just use free trailing space at the last segment.
- * Would probably be more convenient to reserve space for AAD
- * inside rte_crypto_op itself
- * (again for IV space is already reserved inside cop).
- */
- if (sa->aad_len + sa->sqh_len > rte_pktmbuf_tailroom(ml))
- return -ENOSPC;
-
- icv->va = rte_pktmbuf_mtod_offset(ml, void *, icv_ofs);
- icv->pa = rte_pktmbuf_iova_offset(ml, icv_ofs);
-
- inb_pkt_xprepare(sa, sqn, icv);
- return plen;
-}
-
-/*
- * setup/update packets and crypto ops for ESP inbound case.
- */
-static uint16_t
-inb_pkt_prepare(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
- struct rte_crypto_op *cop[], uint16_t num)
-{
- int32_t rc;
- uint32_t i, k, hl;
- struct rte_ipsec_sa *sa;
- struct replay_sqn *rsn;
- union sym_op_data icv;
- struct rte_mbuf *dr[num];
-
- sa = ss->sa;
- rsn = rsn_acquire(sa);
-
- k = 0;
- for (i = 0; i != num; i++) {
-
- hl = mb[i]->l2_len + mb[i]->l3_len;
- rc = esp_inb_tun_pkt_prepare(sa, rsn, mb[i], hl, &icv);
- if (rc >= 0)
- rc = esp_inb_tun_cop_prepare(cop[k], sa, mb[i], &icv,
- hl, rc);
-
- if (rc == 0)
- mb[k++] = mb[i];
- else {
- dr[i - k] = mb[i];
- rte_errno = -rc;
- }
- }
-
- rsn_release(sa, rsn);
-
- /* update cops */
- lksd_none_cop_prepare(ss, mb, cop, k);
-
- /* copy not prepared mbufs beyond good ones */
- if (k != num && k != 0)
- mbuf_bulk_copy(mb + k, dr, num - k);
-
- return k;
-}
-