* Copyright(C) 2021 Marvell.
*/
-#include <rte_malloc.h>
-#include <rte_cryptodev.h>
+#include <cryptodev_pmd.h>
#include <rte_esp.h>
#include <rte_ip.h>
+#include <rte_malloc.h>
#include <rte_security.h>
#include <rte_security_driver.h>
#include <rte_udp.h>
+#include "cn10k_ipsec.h"
#include "cnxk_cryptodev.h"
+#include "cnxk_cryptodev_ops.h"
#include "cnxk_ipsec.h"
#include "cnxk_security.h"
-#include "cn10k_ipsec.h"
#include "roc_api.h"
-static int
-ipsec_xform_cipher_verify(struct rte_crypto_sym_xform *xform)
-{
- if (xform->cipher.algo == RTE_CRYPTO_CIPHER_AES_CBC) {
- switch (xform->cipher.key.length) {
- case 16:
- case 24:
- case 32:
- break;
- default:
- return -ENOTSUP;
- }
- return 0;
- }
-
- return -ENOTSUP;
-}
-
-static int
-ipsec_xform_auth_verify(struct rte_crypto_sym_xform *xform)
-{
- uint16_t keylen = xform->auth.key.length;
-
- if (xform->auth.algo == RTE_CRYPTO_AUTH_SHA1_HMAC) {
- if (keylen >= 20 && keylen <= 64)
- return 0;
- }
-
- return -ENOTSUP;
-}
-
-static int
-ipsec_xform_aead_verify(struct rte_security_ipsec_xform *ipsec_xfrm,
- struct rte_crypto_sym_xform *crypto_xfrm)
-{
- if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS &&
- crypto_xfrm->aead.op != RTE_CRYPTO_AEAD_OP_ENCRYPT)
- return -EINVAL;
-
- if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS &&
- crypto_xfrm->aead.op != RTE_CRYPTO_AEAD_OP_DECRYPT)
- return -EINVAL;
-
- if (crypto_xfrm->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
- switch (crypto_xfrm->aead.key.length) {
- case ROC_CPT_AES128_KEY_LEN:
- case ROC_CPT_AES192_KEY_LEN:
- case ROC_CPT_AES256_KEY_LEN:
- break;
- default:
- return -EINVAL;
- }
- return 0;
- }
-
- return -ENOTSUP;
-}
-
-static int
-cn10k_ipsec_xform_verify(struct rte_security_ipsec_xform *ipsec_xfrm,
- struct rte_crypto_sym_xform *crypto_xfrm)
-{
- struct rte_crypto_sym_xform *auth_xform, *cipher_xform;
- int ret;
-
- if ((ipsec_xfrm->direction != RTE_SECURITY_IPSEC_SA_DIR_INGRESS) &&
- (ipsec_xfrm->direction != RTE_SECURITY_IPSEC_SA_DIR_EGRESS))
- return -EINVAL;
-
- if ((ipsec_xfrm->proto != RTE_SECURITY_IPSEC_SA_PROTO_ESP) &&
- (ipsec_xfrm->proto != RTE_SECURITY_IPSEC_SA_PROTO_AH))
- return -EINVAL;
-
- if ((ipsec_xfrm->mode != RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT) &&
- (ipsec_xfrm->mode != RTE_SECURITY_IPSEC_SA_MODE_TUNNEL))
- return -EINVAL;
-
- if ((ipsec_xfrm->tunnel.type != RTE_SECURITY_IPSEC_TUNNEL_IPV4) &&
- (ipsec_xfrm->tunnel.type != RTE_SECURITY_IPSEC_TUNNEL_IPV6))
- return -EINVAL;
-
- if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_AEAD)
- return ipsec_xform_aead_verify(ipsec_xfrm, crypto_xfrm);
-
- if (crypto_xfrm->next == NULL)
- return -EINVAL;
-
- if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
- /* Ingress */
- if (crypto_xfrm->type != RTE_CRYPTO_SYM_XFORM_AUTH ||
- crypto_xfrm->next->type != RTE_CRYPTO_SYM_XFORM_CIPHER)
- return -EINVAL;
- auth_xform = crypto_xfrm;
- cipher_xform = crypto_xfrm->next;
- } else {
- /* Egress */
- if (crypto_xfrm->type != RTE_CRYPTO_SYM_XFORM_CIPHER ||
- crypto_xfrm->next->type != RTE_CRYPTO_SYM_XFORM_AUTH)
- return -EINVAL;
- cipher_xform = crypto_xfrm;
- auth_xform = crypto_xfrm->next;
- }
-
- ret = ipsec_xform_cipher_verify(cipher_xform);
- if (ret)
- return ret;
-
- ret = ipsec_xform_auth_verify(auth_xform);
- if (ret)
- return ret;
-
- return 0;
-}
-
static uint64_t
ipsec_cpt_inst_w7_get(struct roc_cpt *roc_cpt, void *sa)
{
}
static int
-cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt,
+cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
struct rte_security_ipsec_xform *ipsec_xfrm,
struct rte_crypto_sym_xform *crypto_xfrm,
struct rte_security_session *sec_sess)
{
- struct roc_ot_ipsec_outb_sa *out_sa;
+ union roc_ot_ipsec_outb_param1 param1;
+ struct roc_ot_ipsec_outb_sa *sa_dptr;
struct cnxk_ipsec_outb_rlens rlens;
struct cn10k_sec_session *sess;
struct cn10k_ipsec_sa *sa;
union cpt_inst_w4 inst_w4;
- int ret;
+ void *out_sa;
+ int ret = 0;
sess = get_sec_session_private_data(sec_sess);
sa = &sess->sa;
out_sa = &sa->out_sa;
- memset(out_sa, 0, sizeof(struct roc_ot_ipsec_outb_sa));
+ /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
+ sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8);
+ if (sa_dptr == NULL) {
+ plt_err("Couldn't allocate memory for SA dptr");
+ return -ENOMEM;
+ }
/* Translate security parameters to SA */
- ret = cnxk_ot_ipsec_outb_sa_fill(out_sa, ipsec_xfrm, crypto_xfrm);
- if (ret)
- return ret;
+ ret = cnxk_ot_ipsec_outb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm);
+ if (ret) {
+ plt_err("Could not fill outbound session parameters");
+ goto sa_dptr_free;
+ }
- sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, sa);
+ sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, out_sa);
+
+#ifdef LA_IPSEC_DEBUG
+ /* Use IV from application in debug mode */
+ if (ipsec_xfrm->options.iv_gen_disable == 1) {
+ sa_dptr->w2.s.iv_src = ROC_IE_OT_SA_IV_SRC_FROM_SA;
+ if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
+ sa->iv_offset = crypto_xfrm->aead.iv.offset;
+ sa->iv_length = crypto_xfrm->aead.iv.length;
+ } else if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
+ sa->iv_offset = crypto_xfrm->cipher.iv.offset;
+ sa->iv_length = crypto_xfrm->cipher.iv.length;
+ } else {
+ sa->iv_offset = crypto_xfrm->auth.iv.offset;
+ sa->iv_length = crypto_xfrm->auth.iv.length;
+ }
+ }
+#else
+ if (ipsec_xfrm->options.iv_gen_disable != 0) {
+ plt_err("Application provided IV not supported");
+ ret = -ENOTSUP;
+ goto sa_dptr_free;
+ }
+#endif
+
+ sa->is_outbound = true;
/* Get Rlen calculation data */
ret = cnxk_ipsec_outb_rlens_get(&rlens, ipsec_xfrm, crypto_xfrm);
if (ret)
- return ret;
+ goto sa_dptr_free;
- sa->partial_len = rlens.partial_len;
- sa->roundup_byte = rlens.roundup_byte;
- sa->roundup_len = rlens.roundup_len;
+ sa->max_extended_len = rlens.max_extended_len;
/* pre-populate CPT INST word 4 */
inst_w4.u64 = 0;
inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC;
- inst_w4.s.param1 = 0;
+
+ param1.u16 = 0;
+
+ param1.s.ttl_or_hop_limit = ipsec_xfrm->options.dec_ttl;
+
+ /* Disable IP checksum computation by default */
+ param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
+
+ if (ipsec_xfrm->options.ip_csum_enable) {
+ param1.s.ip_csum_disable =
+ ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
+ }
+
+ /* Disable L4 checksum computation by default */
+ param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
+
+ if (ipsec_xfrm->options.l4_csum_enable) {
+ param1.s.l4_csum_disable =
+ ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
+ }
+
+ inst_w4.s.param1 = param1.u16;
+
sa->inst.w4 = inst_w4.u64;
- return 0;
+ if (ipsec_xfrm->options.stats == 1) {
+ /* Enable mib counters */
+ sa_dptr->w0.s.count_mib_bytes = 1;
+ sa_dptr->w0.s.count_mib_pkts = 1;
+ }
+
+ memset(out_sa, 0, sizeof(struct roc_ot_ipsec_outb_sa));
+
+ /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
+ memcpy(out_sa, sa_dptr, 8);
+
+ plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
+ /* Write session using microcode opcode */
+ ret = roc_cpt_ctx_write(lf, sa_dptr, out_sa,
+ sizeof(struct roc_ot_ipsec_outb_sa));
+ if (ret) {
+ plt_err("Could not write outbound session to hardware");
+ goto sa_dptr_free;
+ }
+
+ /* Trigger CTX flush so that data is written back to DRAM */
+ roc_cpt_lf_ctx_flush(lf, out_sa, false);
+
+ plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
+sa_dptr_free:
+ plt_free(sa_dptr);
+
+ return ret;
}
static int
-cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt,
+cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
struct rte_security_ipsec_xform *ipsec_xfrm,
struct rte_crypto_sym_xform *crypto_xfrm,
struct rte_security_session *sec_sess)
{
- struct roc_ot_ipsec_inb_sa *in_sa;
+ union roc_ot_ipsec_inb_param1 param1;
+ struct roc_ot_ipsec_inb_sa *sa_dptr;
struct cn10k_sec_session *sess;
struct cn10k_ipsec_sa *sa;
union cpt_inst_w4 inst_w4;
- int ret;
+ void *in_sa;
+ int ret = 0;
sess = get_sec_session_private_data(sec_sess);
sa = &sess->sa;
in_sa = &sa->in_sa;
- /* Translate security parameters to SA */
- ret = cnxk_ot_ipsec_inb_sa_fill(in_sa, ipsec_xfrm, crypto_xfrm);
- if (ret)
- return ret;
-
- /* TODO add support for antireplay */
- sa->in_sa.w0.s.ar_win = 0;
+ /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
+ sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8);
+ if (sa_dptr == NULL) {
+ plt_err("Couldn't allocate memory for SA dptr");
+ return -ENOMEM;
+ }
- /* TODO add support for udp encap */
+ /* Translate security parameters to SA */
+ ret = cnxk_ot_ipsec_inb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm,
+ false);
+ if (ret) {
+ plt_err("Could not fill inbound session parameters");
+ goto sa_dptr_free;
+ }
- sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, sa);
+ sa->is_outbound = false;
+ sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, in_sa);
/* pre-populate CPT INST word 4 */
inst_w4.u64 = 0;
inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_INBOUND_IPSEC;
- /* Disable checksum verification for now */
- inst_w4.s.param1 = 7;
+ param1.u16 = 0;
+
+ /* Disable IP checksum verification by default */
+ param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
+
+ if (ipsec_xfrm->options.ip_csum_enable) {
+ param1.s.ip_csum_disable =
+ ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
+ }
+
+ /* Disable L4 checksum verification by default */
+ param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
+
+ if (ipsec_xfrm->options.l4_csum_enable) {
+ param1.s.l4_csum_disable =
+ ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
+ }
+
+ param1.s.esp_trailer_disable = 1;
+
+ inst_w4.s.param1 = param1.u16;
+
sa->inst.w4 = inst_w4.u64;
- return 0;
+ if (ipsec_xfrm->options.stats == 1) {
+ /* Enable mib counters */
+ sa_dptr->w0.s.count_mib_bytes = 1;
+ sa_dptr->w0.s.count_mib_pkts = 1;
+ }
+
+ memset(in_sa, 0, sizeof(struct roc_ot_ipsec_inb_sa));
+
+ /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
+ memcpy(in_sa, sa_dptr, 8);
+
+ plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
+ /* Write session using microcode opcode */
+ ret = roc_cpt_ctx_write(lf, sa_dptr, in_sa,
+ sizeof(struct roc_ot_ipsec_inb_sa));
+ if (ret) {
+ plt_err("Could not write inbound session to hardware");
+ goto sa_dptr_free;
+ }
+
+ /* Trigger CTX flush so that data is written back to DRAM */
+ roc_cpt_lf_ctx_flush(lf, in_sa, true);
+
+ plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
+sa_dptr_free:
+ plt_free(sa_dptr);
+
+ return ret;
}
static int
struct rte_cryptodev *crypto_dev = dev;
struct roc_cpt *roc_cpt;
struct cnxk_cpt_vf *vf;
+ struct cnxk_cpt_qp *qp;
int ret;
- vf = crypto_dev->data->dev_private;
- roc_cpt = &vf->cpt;
-
- if (crypto_dev->data->queue_pairs[0] == NULL) {
+ qp = crypto_dev->data->queue_pairs[0];
+ if (qp == NULL) {
plt_err("Setup cpt queue pair before creating security session");
return -EPERM;
}
- ret = cn10k_ipsec_xform_verify(ipsec_xfrm, crypto_xfrm);
+ ret = cnxk_ipsec_xform_verify(ipsec_xfrm, crypto_xfrm);
if (ret)
return ret;
+ vf = crypto_dev->data->dev_private;
+ roc_cpt = &vf->cpt;
+
if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
- return cn10k_ipsec_inb_sa_create(roc_cpt, ipsec_xfrm,
+ return cn10k_ipsec_inb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm,
crypto_xfrm, sess);
else
- return cn10k_ipsec_outb_sa_create(roc_cpt, ipsec_xfrm,
+ return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm,
crypto_xfrm, sess);
}
if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
return -EINVAL;
- if (rte_security_dynfield_register() < 0)
- return -ENOTSUP;
-
if (rte_mempool_get(mempool, (void **)&priv)) {
plt_err("Could not allocate security session private data");
return -ENOMEM;
set_sec_session_private_data(sess, priv);
- priv->userdata = conf->userdata;
-
if (conf->protocol != RTE_SECURITY_PROTOCOL_IPSEC) {
ret = -ENOTSUP;
goto mempool_put;
}
static int
-cn10k_sec_session_destroy(void *device __rte_unused,
- struct rte_security_session *sess)
+cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess)
{
- struct cn10k_sec_session *priv;
+ struct rte_cryptodev *crypto_dev = dev;
+ union roc_ot_ipsec_sa_word2 *w2;
+ struct cn10k_sec_session *sess;
struct rte_mempool *sess_mp;
+ struct cn10k_ipsec_sa *sa;
+ struct cnxk_cpt_qp *qp;
+ struct roc_cpt_lf *lf;
+ void *sa_dptr = NULL;
+ int ret;
- priv = get_sec_session_private_data(sess);
+ sess = get_sec_session_private_data(sec_sess);
+ if (sess == NULL)
+ return 0;
- if (priv == NULL)
+ qp = crypto_dev->data->queue_pairs[0];
+ if (qp == NULL)
return 0;
- sess_mp = rte_mempool_from_obj(priv);
+ lf = &qp->lf;
- set_sec_session_private_data(sess, NULL);
- rte_mempool_put(sess_mp, priv);
+ sa = &sess->sa;
+
+ /* Trigger CTX flush to write dirty data back to DRAM */
+ roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false);
+
+ ret = -1;
+
+ if (sa->is_outbound) {
+ sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8);
+ if (sa_dptr != NULL) {
+ roc_ot_ipsec_outb_sa_init(sa_dptr);
+
+ ret = roc_cpt_ctx_write(
+ lf, sa_dptr, &sa->out_sa,
+ sizeof(struct roc_ot_ipsec_outb_sa));
+ }
+ } else {
+ sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8);
+ if (sa_dptr != NULL) {
+ roc_ot_ipsec_inb_sa_init(sa_dptr, false);
+
+ ret = roc_cpt_ctx_write(
+ lf, sa_dptr, &sa->in_sa,
+ sizeof(struct roc_ot_ipsec_inb_sa));
+ }
+ }
+
+ plt_free(sa_dptr);
+
+ if (ret) {
+ /* MC write_ctx failed. Attempt reload of CTX */
+
+ /* Wait for 1 ms so that flush is complete */
+ rte_delay_ms(1);
+
+ w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
+ w2->s.valid = 0;
+
+ plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
+
+ /* Trigger CTX reload to fetch new data from DRAM */
+ roc_cpt_lf_ctx_reload(lf, &sa->in_sa);
+ }
+
+ sess_mp = rte_mempool_from_obj(sess);
+
+ set_sec_session_private_data(sec_sess, NULL);
+ rte_mempool_put(sess_mp, sess);
return 0;
}
return sizeof(struct cn10k_sec_session);
}
+static int
+cn10k_sec_session_stats_get(void *device, struct rte_security_session *sess,
+ struct rte_security_stats *stats)
+{
+ struct rte_cryptodev *crypto_dev = device;
+ struct roc_ot_ipsec_outb_sa *out_sa;
+ struct roc_ot_ipsec_inb_sa *in_sa;
+ union roc_ot_ipsec_sa_word2 *w2;
+ struct cn10k_sec_session *priv;
+ struct cn10k_ipsec_sa *sa;
+ struct cnxk_cpt_qp *qp;
+
+ priv = get_sec_session_private_data(sess);
+ if (priv == NULL)
+ return -EINVAL;
+
+ qp = crypto_dev->data->queue_pairs[0];
+ if (qp == NULL)
+ return -EINVAL;
+
+ sa = &priv->sa;
+ w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
+
+ stats->protocol = RTE_SECURITY_PROTOCOL_IPSEC;
+
+ if (w2->s.dir == ROC_IE_SA_DIR_OUTBOUND) {
+ out_sa = &sa->out_sa;
+ roc_cpt_lf_ctx_flush(&qp->lf, out_sa, false);
+ rte_delay_ms(1);
+ stats->ipsec.opackets = out_sa->ctx.mib_pkts;
+ stats->ipsec.obytes = out_sa->ctx.mib_octs;
+ } else {
+ in_sa = &sa->in_sa;
+ roc_cpt_lf_ctx_flush(&qp->lf, in_sa, false);
+ rte_delay_ms(1);
+ stats->ipsec.ipackets = in_sa->ctx.mib_pkts;
+ stats->ipsec.ibytes = in_sa->ctx.mib_octs;
+ }
+
+ return 0;
+}
+
+static int
+cn10k_sec_session_update(void *device, struct rte_security_session *sess,
+ struct rte_security_session_conf *conf)
+{
+ struct rte_cryptodev *crypto_dev = device;
+ struct cn10k_sec_session *priv;
+ struct roc_cpt *roc_cpt;
+ struct cnxk_cpt_qp *qp;
+ struct cnxk_cpt_vf *vf;
+ int ret;
+
+ priv = get_sec_session_private_data(sess);
+ if (priv == NULL)
+ return -EINVAL;
+
+ qp = crypto_dev->data->queue_pairs[0];
+ if (qp == NULL)
+ return -EINVAL;
+
+ if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
+ return -ENOTSUP;
+
+ ret = cnxk_ipsec_xform_verify(&conf->ipsec, conf->crypto_xform);
+ if (ret)
+ return ret;
+
+ vf = crypto_dev->data->dev_private;
+ roc_cpt = &vf->cpt;
+
+ return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, &conf->ipsec,
+ conf->crypto_xform, sess);
+}
+
/* Update platform specific security ops */
void
cn10k_sec_ops_override(void)
cnxk_sec_ops.session_create = cn10k_sec_session_create;
cnxk_sec_ops.session_destroy = cn10k_sec_session_destroy;
cnxk_sec_ops.session_get_size = cn10k_sec_session_get_size;
+ cnxk_sec_ops.session_stats_get = cn10k_sec_session_stats_get;
+ cnxk_sec_ops.session_update = cn10k_sec_session_update;
}